From 51f6b6d33f5b50e16f0b04bcccd03ad370f36636 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Mon, 16 Jan 2017 10:20:20 +0100 Subject: [PATCH 01/21] common/hexutil: fix EncodeBig, Big.MarshalJSON The code was too clever and failed to include zeros on a big.Word boundary. --- common/hexutil/hexutil.go | 7 +------ common/hexutil/hexutil_test.go | 1 + common/hexutil/json.go | 9 ++------- 3 files changed, 4 insertions(+), 13 deletions(-) diff --git a/common/hexutil/hexutil.go b/common/hexutil/hexutil.go index 29e6de3335..4ec0ee8e62 100644 --- a/common/hexutil/hexutil.go +++ b/common/hexutil/hexutil.go @@ -169,12 +169,7 @@ func EncodeBig(bigint *big.Int) string { if nbits == 0 { return "0x0" } - enc := make([]byte, 2, (nbits/8)*2+2) - copy(enc, "0x") - for i := len(bigint.Bits()) - 1; i >= 0; i-- { - enc = strconv.AppendUint(enc, uint64(bigint.Bits()[i]), 16) - } - return string(enc) + return fmt.Sprintf("0x%x", bigint) } func has0xPrefix(input string) bool { diff --git a/common/hexutil/hexutil_test.go b/common/hexutil/hexutil_test.go index b58b4745c5..324e9d3481 100644 --- a/common/hexutil/hexutil_test.go +++ b/common/hexutil/hexutil_test.go @@ -46,6 +46,7 @@ var ( {referenceBig("1"), "0x1"}, {referenceBig("ff"), "0xff"}, {referenceBig("112233445566778899aabbccddeeff"), "0x112233445566778899aabbccddeeff"}, + {referenceBig("80a7f2c1bcc396c00"), "0x80a7f2c1bcc396c00"}, } encodeUint64Tests = []marshalTest{ diff --git a/common/hexutil/json.go b/common/hexutil/json.go index c36d862b5e..7e4736dd68 100644 --- a/common/hexutil/json.go +++ b/common/hexutil/json.go @@ -109,13 +109,8 @@ func (b *Big) MarshalJSON() ([]byte, error) { if nbits == 0 { return jsonZero, nil } - enc := make([]byte, 3, (nbits/8)*2+4) - copy(enc, `"0x`) - for i := len(bigint.Bits()) - 1; i >= 0; i-- { - enc = strconv.AppendUint(enc, uint64(bigint.Bits()[i]), 16) - } - enc = append(enc, '"') - return enc, nil + enc := fmt.Sprintf(`"0x%x"`, bigint) + return []byte(enc), nil } // UnmarshalJSON implements json.Unmarshaler. From 0fa9a8929c8a55d1b624479cead2bb960ada7cfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Mon, 16 Jan 2017 10:51:29 +0100 Subject: [PATCH 02/21] les: fixed transaction sending deadlock (#3568) --- les/txrelay.go | 1 - 1 file changed, 1 deletion(-) diff --git a/les/txrelay.go b/les/txrelay.go index 84d049b45d..76d416c57a 100644 --- a/les/txrelay.go +++ b/les/txrelay.go @@ -110,7 +110,6 @@ func (self *LesTxRelay) send(txs types.Transactions, count int) { for p, list := range sendTo { cost := p.GetRequestCost(SendTxMsg, len(list)) go func(p *peer, list types.Transactions, cost uint64) { - p.fcServer.SendRequest(0, cost) p.SendTxs(cost, list) }(p, list, cost) } From da2a22c384a9b621ec853fe4b1aa651d606cf42b Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Mon, 16 Jan 2017 10:57:02 +0100 Subject: [PATCH 03/21] params: stable 1.5.7 --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index 6eb920a21e..a75d5a3d10 100644 --- a/params/version.go +++ b/params/version.go @@ -19,10 +19,10 @@ package params import "fmt" const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 5 // Minor version component of the current release - VersionPatch = 7 // Patch version component of the current release - VersionMeta = "unstable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 5 // Minor version component of the current release + VersionPatch = 7 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string. From 26d385c18b5eb003d9a69ff618c78acbe594db44 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Mon, 16 Jan 2017 11:12:50 +0100 Subject: [PATCH 04/21] params, VERSION: 1.5.8 unstable --- VERSION | 2 +- params/version.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/VERSION b/VERSION index f01291b87f..1cc9c180e2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.5.7 +1.5.8 diff --git a/params/version.go b/params/version.go index a75d5a3d10..b60e105d56 100644 --- a/params/version.go +++ b/params/version.go @@ -19,10 +19,10 @@ package params import "fmt" const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 5 // Minor version component of the current release - VersionPatch = 7 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 5 // Minor version component of the current release + VersionPatch = 8 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. From 54a65e6d87800ba5ecb8a14621d4aff0bfa56ada Mon Sep 17 00:00:00 2001 From: Bas van Kervel Date: Tue, 17 Jan 2017 11:35:07 +0100 Subject: [PATCH 05/21] cmd,eth,les,internal: remove natspec support --- cmd/geth/main.go | 1 - cmd/geth/usage.go | 1 - cmd/utils/flags.go | 5 ----- common/compiler/solidity_test.go | 4 ++-- eth/backend.go | 3 --- internal/web3ext/web3ext.go | 6 ------ les/backend.go | 2 -- swarm/api/http/roundtripper.go | 4 +--- 8 files changed, 3 insertions(+), 23 deletions(-) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index a6b6331df4..bcaf2adefe 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -106,7 +106,6 @@ func init() { utils.AutoDAGFlag, utils.TargetGasLimitFlag, utils.NATFlag, - utils.NatspecEnabledFlag, utils.NoDiscoverFlag, utils.DiscoveryV5Flag, utils.NetrestrictFlag, diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 853307604b..759e0737dc 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -169,7 +169,6 @@ var AppHelpFlagGroups = []flagGroup{ Name: "EXPERIMENTAL", Flags: []cli.Flag{ utils.WhisperEnabledFlag, - utils.NatspecEnabledFlag, }, }, { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 01114a9571..ad7e6da6e0 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -130,10 +130,6 @@ var ( Name: "identity", Usage: "Custom node name", } - NatspecEnabledFlag = cli.BoolFlag{ - Name: "natspec", - Usage: "Enable NatSpec confirmation notice", - } DocRootFlag = DirectoryFlag{ Name: "docroot", Usage: "Document Root for HTTPClient file scheme", @@ -730,7 +726,6 @@ func RegisterEthService(ctx *cli.Context, stack *node.Node, extra []byte) { NetworkId: ctx.GlobalInt(NetworkIdFlag.Name), MinerThreads: ctx.GlobalInt(MinerThreadsFlag.Name), ExtraData: MakeMinerExtra(extra, ctx), - NatSpec: ctx.GlobalBool(NatspecEnabledFlag.Name), DocRoot: ctx.GlobalString(DocRootFlag.Name), GasPrice: common.String2Big(ctx.GlobalString(GasPriceFlag.Name)), GpoMinGasPrice: common.String2Big(ctx.GlobalString(GpoMinGasPriceFlag.Name)), diff --git a/common/compiler/solidity_test.go b/common/compiler/solidity_test.go index 8ba9e55d05..f166375472 100644 --- a/common/compiler/solidity_test.go +++ b/common/compiler/solidity_test.go @@ -36,7 +36,7 @@ contract test { } } ` - testInfo = `{"source":"\ncontract test {\n /// @notice Will multiply ` + "`a`" + ` by 7.\n function multiply(uint a) returns(uint d) {\n return a * 7;\n }\n}\n","language":"Solidity","languageVersion":"0.1.1","compilerVersion":"0.1.1","compilerOptions":"--binary file --json-abi file --natspec-user file --natspec-dev file --add-std 1","abiDefinition":[{"constant":false,"inputs":[{"name":"a","type":"uint256"}],"name":"multiply","outputs":[{"name":"d","type":"uint256"}],"type":"function"}],"userDoc":{"methods":{"multiply(uint256)":{"notice":"Will multiply ` + "`a`" + ` by 7."}}},"developerDoc":{"methods":{}}}` + testInfo = `{"source":"\ncontract test {\n /// @notice Will multiply ` + "`a`" + ` by 7.\n function multiply(uint a) returns(uint d) {\n return a * 7;\n }\n}\n","language":"Solidity","languageVersion":"0.1.1","compilerVersion":"0.1.1","compilerOptions":"--binary file --json-abi file --add-std 1","abiDefinition":[{"constant":false,"inputs":[{"name":"a","type":"uint256"}],"name":"multiply","outputs":[{"name":"d","type":"uint256"}],"type":"function"}],"userDoc":{"methods":{"multiply(uint256)":{"notice":"Will multiply ` + "`a`" + ` by 7."}}},"developerDoc":{"methods":{}}}` ) func skipWithoutSolc(t *testing.T) { @@ -99,7 +99,7 @@ func TestSaveInfo(t *testing.T) { if string(got) != testInfo { t.Errorf("incorrect info.json extracted, expected:\n%s\ngot\n%s", testInfo, string(got)) } - wantHash := common.HexToHash("0x9f3803735e7f16120c5a140ab3f02121fd3533a9655c69b33a10e78752cc49b0") + wantHash := common.HexToHash("0x22450a77f0c3ff7a395948d07bc1456881226a1b6325f4189cb5f1254a824080") if cinfohash != wantHash { t.Errorf("content hash for info is incorrect. expected %v, got %v", wantHash.Hex(), cinfohash.Hex()) } diff --git a/eth/backend.go b/eth/backend.go index dec8c0c6ed..a77b210266 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -77,7 +77,6 @@ type Config struct { DatabaseCache int DatabaseHandles int - NatSpec bool DocRoot string AutoDAG bool PowFake bool @@ -140,7 +139,6 @@ type Ethereum struct { etherbase common.Address solcPath string - NatSpec bool netVersionId int netRPCService *ethapi.PublicNetAPI } @@ -174,7 +172,6 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { shutdownChan: make(chan bool), stopDbUpgrade: stopDbUpgrade, netVersionId: config.NetworkId, - NatSpec: config.NatSpec, etherbase: config.Etherbase, MinerThreads: config.MinerThreads, AutoDAG: config.AutoDAG, diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index 12bdb69969..5241cda0a1 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -408,12 +408,6 @@ web3._extend({ params: 3, inputFormatter: [web3._extend.formatters.inputTransactionFormatter, web3._extend.utils.fromDecimal, web3._extend.utils.fromDecimal] }), - new web3._extend.Method({ - name: 'getNatSpec', - call: 'eth_getNatSpec', - params: 1, - inputFormatter: [web3._extend.formatters.inputTransactionFormatter] - }), new web3._extend.Method({ name: 'signTransaction', call: 'eth_signTransaction', diff --git a/les/backend.go b/les/backend.go index 3deab61f72..21ee084987 100644 --- a/les/backend.go +++ b/les/backend.go @@ -66,7 +66,6 @@ type LightEthereum struct { solcPath string solc *compiler.Solidity - NatSpec bool netVersionId int netRPCService *ethapi.PublicNetAPI } @@ -95,7 +94,6 @@ func New(ctx *node.ServiceContext, config *eth.Config) (*LightEthereum, error) { pow: pow, shutdownChan: make(chan bool), netVersionId: config.NetworkId, - NatSpec: config.NatSpec, solcPath: config.SolcPath, } diff --git a/swarm/api/http/roundtripper.go b/swarm/api/http/roundtripper.go index a3a644b730..7b5bbc8838 100644 --- a/swarm/api/http/roundtripper.go +++ b/swarm/api/http/roundtripper.go @@ -44,9 +44,7 @@ If Host is left empty, localhost is assumed. Using a public gateway, the above few lines gives you the leanest bzz-scheme aware read-only http client. You really only ever need this -if you need go-native swarm access to bzz addresses, e.g., -github.com/ethereum/go-ethereum/common/natspec - +if you need go-native swarm access to bzz addresses. */ type RoundTripper struct { From 17d92233d9e64b642fed9a992556f7ff7d6fda18 Mon Sep 17 00:00:00 2001 From: Nick Johnson Date: Tue, 17 Jan 2017 11:19:50 +0000 Subject: [PATCH 06/21] cmd/geth, core: add support for recording SHA3 preimages (#3543) --- accounts/abi/bind/backends/simulated.go | 2 +- cmd/geth/main.go | 1 + cmd/geth/usage.go | 1 + cmd/utils/flags.go | 8 ++++- core/bench_test.go | 5 +-- core/block_validator_test.go | 3 +- core/blockchain.go | 10 ++++-- core/blockchain_test.go | 26 +++++++------- core/chain_makers.go | 2 +- core/chain_makers_test.go | 3 +- core/dao_test.go | 13 +++---- core/database_util.go | 45 +++++++++++++++++++++---- core/state/journal.go | 7 ++++ core/state/statedb.go | 26 +++++++++++++- core/state_processor.go | 2 +- core/vm/instructions.go | 7 +++- core/vm/interface.go | 1 + core/vm/noop.go | 1 + core/vm/vm.go | 2 ++ eth/api.go | 6 ++++ eth/backend.go | 6 ++-- eth/handler_test.go | 3 +- eth/helper_test.go | 3 +- internal/web3ext/web3ext.go | 6 ++++ les/helper_test.go | 3 +- light/odr_test.go | 2 +- light/txpool_test.go | 3 +- light/vm_env.go | 2 ++ tests/block_test_util.go | 3 +- 29 files changed, 156 insertions(+), 46 deletions(-) diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index 7bff1a1253..c26662ea19 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -61,7 +61,7 @@ type SimulatedBackend struct { func NewSimulatedBackend(accounts ...core.GenesisAccount) *SimulatedBackend { database, _ := ethdb.NewMemDatabase() core.WriteGenesisBlockForTesting(database, accounts...) - blockchain, _ := core.NewBlockChain(database, chainConfig, new(core.FakePow), new(event.TypeMux)) + blockchain, _ := core.NewBlockChain(database, chainConfig, new(core.FakePow), new(event.TypeMux), vm.Config{}) backend := &SimulatedBackend{database: database, blockchain: blockchain} backend.rollback() return backend diff --git a/cmd/geth/main.go b/cmd/geth/main.go index a6b6331df4..ac34d30354 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -132,6 +132,7 @@ func init() { utils.VMForceJitFlag, utils.VMJitCacheFlag, utils.VMEnableJitFlag, + utils.VMEnableDebugFlag, utils.NetworkIdFlag, utils.RPCCORSDomainFlag, utils.EthStatsURLFlag, diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 853307604b..5ebfc89197 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -155,6 +155,7 @@ var AppHelpFlagGroups = []flagGroup{ utils.VMEnableJitFlag, utils.VMForceJitFlag, utils.VMJitCacheFlag, + utils.VMEnableDebugFlag, }, }, { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 01114a9571..08b4db5783 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -33,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/ethdb" @@ -230,6 +231,10 @@ var ( Name: "jitvm", Usage: "Enable the JIT VM", } + VMEnableDebugFlag = cli.BoolFlag{ + Name: "vmdebug", + Usage: "Record information useful for VM and contract debugging", + } // Logging and debug settings EthStatsURLFlag = cli.StringFlag{ Name: "ethstats", @@ -741,6 +746,7 @@ func RegisterEthService(ctx *cli.Context, stack *node.Node, extra []byte) { GpobaseCorrectionFactor: ctx.GlobalInt(GpobaseCorrectionFactorFlag.Name), SolcPath: ctx.GlobalString(SolcPathFlag.Name), AutoDAG: ctx.GlobalBool(AutoDAGFlag.Name) || ctx.GlobalBool(MiningEnabledFlag.Name), + EnablePreimageRecording: ctx.GlobalBool(VMEnableDebugFlag.Name), } // Override any default configs in dev mode or the test net @@ -912,7 +918,7 @@ func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chai if !ctx.GlobalBool(FakePoWFlag.Name) { pow = ethash.New() } - chain, err = core.NewBlockChain(chainDb, chainConfig, pow, new(event.TypeMux)) + chain, err = core.NewBlockChain(chainDb, chainConfig, pow, new(event.TypeMux), vm.Config{EnablePreimageRecording: ctx.GlobalBool(VMEnableDebugFlag.Name)}) if err != nil { Fatalf("Could not start chainmanager: %v", err) } diff --git a/core/bench_test.go b/core/bench_test.go index 5785748a14..353d217fd4 100644 --- a/core/bench_test.go +++ b/core/bench_test.go @@ -25,6 +25,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" @@ -168,7 +169,7 @@ func benchInsertChain(b *testing.B, disk bool, gen func(int, *BlockGen)) { // Time the insertion of the new chain. // State and blocks are stored in the same DB. evmux := new(event.TypeMux) - chainman, _ := NewBlockChain(db, ¶ms.ChainConfig{HomesteadBlock: new(big.Int)}, FakePow{}, evmux) + chainman, _ := NewBlockChain(db, ¶ms.ChainConfig{HomesteadBlock: new(big.Int)}, FakePow{}, evmux, vm.Config{}) defer chainman.Stop() b.ReportAllocs() b.ResetTimer() @@ -278,7 +279,7 @@ func benchReadChain(b *testing.B, full bool, count uint64) { if err != nil { b.Fatalf("error opening database at %v: %v", dir, err) } - chain, err := NewBlockChain(db, testChainConfig(), FakePow{}, new(event.TypeMux)) + chain, err := NewBlockChain(db, testChainConfig(), FakePow{}, new(event.TypeMux), vm.Config{}) if err != nil { b.Fatalf("error creating chain: %v", err) } diff --git a/core/block_validator_test.go b/core/block_validator_test.go index 413c3cc8ec..01931efd24 100644 --- a/core/block_validator_test.go +++ b/core/block_validator_test.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/params" @@ -39,7 +40,7 @@ func proc() (Validator, *BlockChain) { var mux event.TypeMux WriteTestNetGenesisBlock(db) - blockchain, err := NewBlockChain(db, testChainConfig(), thePow(), &mux) + blockchain, err := NewBlockChain(db, testChainConfig(), thePow(), &mux, vm.Config{}) if err != nil { fmt.Println(err) } diff --git a/core/blockchain.go b/core/blockchain.go index 6462c17fa5..2e522d97c4 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -107,12 +107,13 @@ type BlockChain struct { pow pow.PoW processor Processor // block processor interface validator Validator // block and state validator interface + vmConfig vm.Config } // NewBlockChain returns a fully initialised block chain using information // available in the database. It initialiser the default Ethereum Validator and // Processor. -func NewBlockChain(chainDb ethdb.Database, config *params.ChainConfig, pow pow.PoW, mux *event.TypeMux) (*BlockChain, error) { +func NewBlockChain(chainDb ethdb.Database, config *params.ChainConfig, pow pow.PoW, mux *event.TypeMux, vmConfig vm.Config) (*BlockChain, error) { bodyCache, _ := lru.New(bodyCacheLimit) bodyRLPCache, _ := lru.New(bodyCacheLimit) blockCache, _ := lru.New(blockCacheLimit) @@ -128,6 +129,7 @@ func NewBlockChain(chainDb ethdb.Database, config *params.ChainConfig, pow pow.P blockCache: blockCache, futureBlocks: futureBlocks, pow: pow, + vmConfig: vmConfig, } bc.SetValidator(NewBlockValidator(config, bc, pow)) bc.SetProcessor(NewStateProcessor(config, bc)) @@ -954,7 +956,7 @@ func (self *BlockChain) InsertChain(chain types.Blocks) (int, error) { return i, err } // Process block using the parent state as reference point. - receipts, logs, usedGas, err := self.processor.Process(block, self.stateCache, vm.Config{}) + receipts, logs, usedGas, err := self.processor.Process(block, self.stateCache, self.vmConfig) if err != nil { self.reportBlock(block, receipts, err) return i, err @@ -1004,6 +1006,10 @@ func (self *BlockChain) InsertChain(chain types.Blocks) (int, error) { if err := WriteMipmapBloom(self.chainDb, block.NumberU64(), receipts); err != nil { return i, err } + // Write hash preimages + if err := WritePreimages(self.chainDb, block.NumberU64(), self.stateCache.Preimages()); err != nil { + return i, err + } case SideStatTy: if glog.V(logger.Detail) { glog.Infof("inserted forked block #%d [%x…] (TD=%v) in %9v: %3d txs %d uncles.", block.Number(), block.Hash().Bytes()[0:4], block.Difficulty(), common.PrettyDuration(time.Since(bstart)), len(block.Transactions()), len(block.Uncles())) diff --git a/core/blockchain_test.go b/core/blockchain_test.go index a5a83ba609..8f1383acdb 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -53,7 +53,7 @@ func thePow() pow.PoW { func theBlockChain(db ethdb.Database, t *testing.T) *BlockChain { var eventMux event.TypeMux WriteTestNetGenesisBlock(db) - blockchain, err := NewBlockChain(db, testChainConfig(), thePow(), &eventMux) + blockchain, err := NewBlockChain(db, testChainConfig(), thePow(), &eventMux, vm.Config{}) if err != nil { t.Error("failed creating blockchain:", err) t.FailNow() @@ -614,7 +614,7 @@ func testReorgBadHashes(t *testing.T, full bool) { defer func() { delete(BadHashes, headers[3].Hash()) }() } // Create a new chain manager and check it rolled back the state - ncm, err := NewBlockChain(db, testChainConfig(), FakePow{}, new(event.TypeMux)) + ncm, err := NewBlockChain(db, testChainConfig(), FakePow{}, new(event.TypeMux), vm.Config{}) if err != nil { t.Fatalf("failed to create new chain manager: %v", err) } @@ -735,7 +735,7 @@ func TestFastVsFullChains(t *testing.T) { archiveDb, _ := ethdb.NewMemDatabase() WriteGenesisBlockForTesting(archiveDb, GenesisAccount{address, funds}) - archive, _ := NewBlockChain(archiveDb, testChainConfig(), FakePow{}, new(event.TypeMux)) + archive, _ := NewBlockChain(archiveDb, testChainConfig(), FakePow{}, new(event.TypeMux), vm.Config{}) if n, err := archive.InsertChain(blocks); err != nil { t.Fatalf("failed to process block %d: %v", n, err) @@ -743,7 +743,7 @@ func TestFastVsFullChains(t *testing.T) { // Fast import the chain as a non-archive node to test fastDb, _ := ethdb.NewMemDatabase() WriteGenesisBlockForTesting(fastDb, GenesisAccount{address, funds}) - fast, _ := NewBlockChain(fastDb, testChainConfig(), FakePow{}, new(event.TypeMux)) + fast, _ := NewBlockChain(fastDb, testChainConfig(), FakePow{}, new(event.TypeMux), vm.Config{}) headers := make([]*types.Header, len(blocks)) for i, block := range blocks { @@ -819,7 +819,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { archiveDb, _ := ethdb.NewMemDatabase() WriteGenesisBlockForTesting(archiveDb, GenesisAccount{address, funds}) - archive, _ := NewBlockChain(archiveDb, testChainConfig(), FakePow{}, new(event.TypeMux)) + archive, _ := NewBlockChain(archiveDb, testChainConfig(), FakePow{}, new(event.TypeMux), vm.Config{}) if n, err := archive.InsertChain(blocks); err != nil { t.Fatalf("failed to process block %d: %v", n, err) @@ -831,7 +831,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { // Import the chain as a non-archive node and ensure all pointers are updated fastDb, _ := ethdb.NewMemDatabase() WriteGenesisBlockForTesting(fastDb, GenesisAccount{address, funds}) - fast, _ := NewBlockChain(fastDb, testChainConfig(), FakePow{}, new(event.TypeMux)) + fast, _ := NewBlockChain(fastDb, testChainConfig(), FakePow{}, new(event.TypeMux), vm.Config{}) headers := make([]*types.Header, len(blocks)) for i, block := range blocks { @@ -850,7 +850,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { // Import the chain as a light node and ensure all pointers are updated lightDb, _ := ethdb.NewMemDatabase() WriteGenesisBlockForTesting(lightDb, GenesisAccount{address, funds}) - light, _ := NewBlockChain(lightDb, testChainConfig(), FakePow{}, new(event.TypeMux)) + light, _ := NewBlockChain(lightDb, testChainConfig(), FakePow{}, new(event.TypeMux), vm.Config{}) if n, err := light.InsertHeaderChain(headers, 1); err != nil { t.Fatalf("failed to insert header %d: %v", n, err) @@ -916,7 +916,7 @@ func TestChainTxReorgs(t *testing.T) { }) // Import the chain. This runs all block validation rules. evmux := &event.TypeMux{} - blockchain, _ := NewBlockChain(db, testChainConfig(), FakePow{}, evmux) + blockchain, _ := NewBlockChain(db, testChainConfig(), FakePow{}, evmux, vm.Config{}) if i, err := blockchain.InsertChain(chain); err != nil { t.Fatalf("failed to insert original chain[%d]: %v", i, err) } @@ -990,7 +990,7 @@ func TestLogReorgs(t *testing.T) { ) evmux := &event.TypeMux{} - blockchain, _ := NewBlockChain(db, testChainConfig(), FakePow{}, evmux) + blockchain, _ := NewBlockChain(db, testChainConfig(), FakePow{}, evmux, vm.Config{}) subs := evmux.Subscribe(RemovedLogsEvent{}) chain, _ := GenerateChain(params.TestChainConfig, genesis, db, 2, func(i int, gen *BlockGen) { @@ -1027,7 +1027,7 @@ func TestReorgSideEvent(t *testing.T) { ) evmux := &event.TypeMux{} - blockchain, _ := NewBlockChain(db, testChainConfig(), FakePow{}, evmux) + blockchain, _ := NewBlockChain(db, testChainConfig(), FakePow{}, evmux, vm.Config{}) chain, _ := GenerateChain(params.TestChainConfig, genesis, db, 3, func(i int, gen *BlockGen) {}) if _, err := blockchain.InsertChain(chain); err != nil { @@ -1103,7 +1103,7 @@ func TestCanonicalBlockRetrieval(t *testing.T) { ) evmux := &event.TypeMux{} - blockchain, _ := NewBlockChain(db, testChainConfig(), FakePow{}, evmux) + blockchain, _ := NewBlockChain(db, testChainConfig(), FakePow{}, evmux, vm.Config{}) chain, _ := GenerateChain(params.TestChainConfig, genesis, db, 10, func(i int, gen *BlockGen) {}) @@ -1146,7 +1146,7 @@ func TestEIP155Transition(t *testing.T) { mux event.TypeMux ) - blockchain, _ := NewBlockChain(db, config, FakePow{}, &mux) + blockchain, _ := NewBlockChain(db, config, FakePow{}, &mux, vm.Config{}) blocks, _ := GenerateChain(config, genesis, db, 4, func(i int, block *BlockGen) { var ( tx *types.Transaction @@ -1250,7 +1250,7 @@ func TestEIP161AccountRemoval(t *testing.T) { } mux event.TypeMux - blockchain, _ = NewBlockChain(db, config, FakePow{}, &mux) + blockchain, _ = NewBlockChain(db, config, FakePow{}, &mux, vm.Config{}) ) blocks, _ := GenerateChain(config, genesis, db, 3, func(i int, block *BlockGen) { var ( diff --git a/core/chain_makers.go b/core/chain_makers.go index 4a838a5aa7..8b3b015a8c 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -256,7 +256,7 @@ func newCanonical(n int, full bool) (ethdb.Database, *BlockChain, error) { // Initialize a fresh chain with only a genesis block genesis, _ := WriteTestNetGenesisBlock(db) - blockchain, _ := NewBlockChain(db, MakeChainConfig(), FakePow{}, evmux) + blockchain, _ := NewBlockChain(db, MakeChainConfig(), FakePow{}, evmux, vm.Config{}) // Create and inject the requested chain if n == 0 { return db, blockchain, nil diff --git a/core/chain_makers_test.go b/core/chain_makers_test.go index 942f4ace22..2796817c01 100644 --- a/core/chain_makers_test.go +++ b/core/chain_makers_test.go @@ -21,6 +21,7 @@ import ( "math/big" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" @@ -81,7 +82,7 @@ func ExampleGenerateChain() { // Import the chain. This runs all block validation rules. evmux := &event.TypeMux{} - blockchain, _ := NewBlockChain(db, chainConfig, FakePow{}, evmux) + blockchain, _ := NewBlockChain(db, chainConfig, FakePow{}, evmux, vm.Config{}) if i, err := blockchain.InsertChain(chain); err != nil { fmt.Printf("insert error (block %d): %v\n", chain[i].NumberU64(), err) return diff --git a/core/dao_test.go b/core/dao_test.go index f461131f49..b8b4c71cfa 100644 --- a/core/dao_test.go +++ b/core/dao_test.go @@ -20,6 +20,7 @@ import ( "math/big" "testing" + "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/params" @@ -39,12 +40,12 @@ func TestDAOForkRangeExtradata(t *testing.T) { proDb, _ := ethdb.NewMemDatabase() WriteGenesisBlockForTesting(proDb) proConf := ¶ms.ChainConfig{HomesteadBlock: big.NewInt(0), DAOForkBlock: forkBlock, DAOForkSupport: true} - proBc, _ := NewBlockChain(proDb, proConf, new(FakePow), new(event.TypeMux)) + proBc, _ := NewBlockChain(proDb, proConf, new(FakePow), new(event.TypeMux), vm.Config{}) conDb, _ := ethdb.NewMemDatabase() WriteGenesisBlockForTesting(conDb) conConf := ¶ms.ChainConfig{HomesteadBlock: big.NewInt(0), DAOForkBlock: forkBlock, DAOForkSupport: false} - conBc, _ := NewBlockChain(conDb, conConf, new(FakePow), new(event.TypeMux)) + conBc, _ := NewBlockChain(conDb, conConf, new(FakePow), new(event.TypeMux), vm.Config{}) if _, err := proBc.InsertChain(prefix); err != nil { t.Fatalf("pro-fork: failed to import chain prefix: %v", err) @@ -57,7 +58,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { // Create a pro-fork block, and try to feed into the no-fork chain db, _ = ethdb.NewMemDatabase() WriteGenesisBlockForTesting(db) - bc, _ := NewBlockChain(db, conConf, new(FakePow), new(event.TypeMux)) + bc, _ := NewBlockChain(db, conConf, new(FakePow), new(event.TypeMux), vm.Config{}) blocks := conBc.GetBlocksFromHash(conBc.CurrentBlock().Hash(), int(conBc.CurrentBlock().NumberU64()+1)) for j := 0; j < len(blocks)/2; j++ { @@ -78,7 +79,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { // Create a no-fork block, and try to feed into the pro-fork chain db, _ = ethdb.NewMemDatabase() WriteGenesisBlockForTesting(db) - bc, _ = NewBlockChain(db, proConf, new(FakePow), new(event.TypeMux)) + bc, _ = NewBlockChain(db, proConf, new(FakePow), new(event.TypeMux), vm.Config{}) blocks = proBc.GetBlocksFromHash(proBc.CurrentBlock().Hash(), int(proBc.CurrentBlock().NumberU64()+1)) for j := 0; j < len(blocks)/2; j++ { @@ -100,7 +101,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { // Verify that contra-forkers accept pro-fork extra-datas after forking finishes db, _ = ethdb.NewMemDatabase() WriteGenesisBlockForTesting(db) - bc, _ := NewBlockChain(db, conConf, new(FakePow), new(event.TypeMux)) + bc, _ := NewBlockChain(db, conConf, new(FakePow), new(event.TypeMux), vm.Config{}) blocks := conBc.GetBlocksFromHash(conBc.CurrentBlock().Hash(), int(conBc.CurrentBlock().NumberU64()+1)) for j := 0; j < len(blocks)/2; j++ { @@ -116,7 +117,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { // Verify that pro-forkers accept contra-fork extra-datas after forking finishes db, _ = ethdb.NewMemDatabase() WriteGenesisBlockForTesting(db) - bc, _ = NewBlockChain(db, proConf, new(FakePow), new(event.TypeMux)) + bc, _ = NewBlockChain(db, proConf, new(FakePow), new(event.TypeMux), vm.Config{}) blocks = proBc.GetBlocksFromHash(proBc.CurrentBlock().Hash(), int(proBc.CurrentBlock().NumberU64()+1)) for j := 0; j < len(blocks)/2; j++ { diff --git a/core/database_util.go b/core/database_util.go index 2060b8b6a4..229f21b5b7 100644 --- a/core/database_util.go +++ b/core/database_util.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" ) @@ -39,12 +40,13 @@ var ( headBlockKey = []byte("LastBlock") headFastKey = []byte("LastFast") - headerPrefix = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header - tdSuffix = []byte("t") // headerPrefix + num (uint64 big endian) + hash + tdSuffix -> td - numSuffix = []byte("n") // headerPrefix + num (uint64 big endian) + numSuffix -> hash - blockHashPrefix = []byte("H") // blockHashPrefix + hash -> num (uint64 big endian) - bodyPrefix = []byte("b") // bodyPrefix + num (uint64 big endian) + hash -> block body - blockReceiptsPrefix = []byte("r") // blockReceiptsPrefix + num (uint64 big endian) + hash -> block receipts + headerPrefix = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header + tdSuffix = []byte("t") // headerPrefix + num (uint64 big endian) + hash + tdSuffix -> td + numSuffix = []byte("n") // headerPrefix + num (uint64 big endian) + numSuffix -> hash + blockHashPrefix = []byte("H") // blockHashPrefix + hash -> num (uint64 big endian) + bodyPrefix = []byte("b") // bodyPrefix + num (uint64 big endian) + hash -> block body + blockReceiptsPrefix = []byte("r") // blockReceiptsPrefix + num (uint64 big endian) + hash -> block receipts + preimagePrefix = "secure-key-" // preimagePrefix + hash -> preimage txMetaSuffix = []byte{0x01} receiptsPrefix = []byte("receipts-") @@ -66,6 +68,9 @@ var ( ChainConfigNotFoundErr = errors.New("ChainConfig not found") // general config not found error mipmapBloomMu sync.Mutex // protect against race condition when updating mipmap blooms + + preimageCounter = metrics.NewCounter("db/preimage/total") + preimageHitCounter = metrics.NewCounter("db/preimage/hits") ) // encodeBlockNumber encodes a block number as big endian uint64 @@ -595,6 +600,34 @@ func GetMipmapBloom(db ethdb.Database, number, level uint64) types.Bloom { return types.BytesToBloom(bloomDat) } +// PreimageTable returns a Database instance with the key prefix for preimage entries. +func PreimageTable(db ethdb.Database) ethdb.Database { + return ethdb.NewTable(db, preimagePrefix) +} + +// WritePreimages writes the provided set of preimages to the database. `number` is the +// current block number, and is used for debug messages only. +func WritePreimages(db ethdb.Database, number uint64, preimages map[common.Hash][]byte) error { + table := PreimageTable(db) + batch := table.NewBatch() + hitCount := 0 + for hash, preimage := range preimages { + if _, err := table.Get(hash.Bytes()); err != nil { + batch.Put(hash.Bytes(), preimage) + hitCount += 1 + } + } + preimageCounter.Inc(int64(len(preimages))) + preimageHitCounter.Inc(int64(hitCount)) + if hitCount > 0 { + if err := batch.Write(); err != nil { + return fmt.Errorf("preimage write fail for block %d: %v", number, err) + } + glog.V(logger.Debug).Infof("%d preimages in block %d, including %d new", len(preimages), number, hitCount) + } + return nil +} + // GetBlockChainVersion reads the version number from db. func GetBlockChainVersion(db ethdb.Database) int { var vsn uint diff --git a/core/state/journal.go b/core/state/journal.go index d1e73e7d03..68d07fa03c 100644 --- a/core/state/journal.go +++ b/core/state/journal.go @@ -67,6 +67,9 @@ type ( addLogChange struct { txhash common.Hash } + addPreimageChange struct { + hash common.Hash + } touchChange struct { account *common.Address prev bool @@ -127,3 +130,7 @@ func (ch addLogChange) undo(s *StateDB) { s.logs[ch.txhash] = logs[:len(logs)-1] } } + +func (ch addPreimageChange) undo(s *StateDB) { + delete(s.preimages, ch.hash) +} diff --git a/core/state/statedb.go b/core/state/statedb.go index 063e2b4697..bbccba9fba 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -75,6 +75,8 @@ type StateDB struct { logs map[common.Hash][]*types.Log logSize uint + preimages map[common.Hash][]byte + // Journal of state modifications. This is the backbone of // Snapshot and RevertToSnapshot. journal journal @@ -99,6 +101,7 @@ func New(root common.Hash, db ethdb.Database) (*StateDB, error) { stateObjectsDirty: make(map[common.Address]struct{}), refund: new(big.Int), logs: make(map[common.Hash][]*types.Log), + preimages: make(map[common.Hash][]byte), }, nil } @@ -120,6 +123,7 @@ func (self *StateDB) New(root common.Hash) (*StateDB, error) { stateObjectsDirty: make(map[common.Address]struct{}), refund: new(big.Int), logs: make(map[common.Hash][]*types.Log), + preimages: make(map[common.Hash][]byte), }, nil } @@ -141,6 +145,7 @@ func (self *StateDB) Reset(root common.Hash) error { self.txIndex = 0 self.logs = make(map[common.Hash][]*types.Log) self.logSize = 0 + self.preimages = make(map[common.Hash][]byte) self.clearJournalAndRefund() return nil @@ -199,6 +204,21 @@ func (self *StateDB) Logs() []*types.Log { return logs } +// AddPreimage records a SHA3 preimage seen by the VM. +func (self *StateDB) AddPreimage(hash common.Hash, preimage []byte) { + if _, ok := self.preimages[hash]; !ok { + self.journal = append(self.journal, addPreimageChange{hash: hash}) + pi := make([]byte, len(preimage)) + copy(pi, preimage) + self.preimages[hash] = pi + } +} + +// Preimages returns a list of SHA3 preimages that have been submitted. +func (self *StateDB) Preimages() map[common.Hash][]byte { + return self.preimages +} + func (self *StateDB) AddRefund(gas *big.Int) { self.journal = append(self.journal, refundChange{prev: new(big.Int).Set(self.refund)}) self.refund.Add(self.refund, gas) @@ -477,8 +497,9 @@ func (self *StateDB) Copy() *StateDB { refund: new(big.Int).Set(self.refund), logs: make(map[common.Hash][]*types.Log, len(self.logs)), logSize: self.logSize, + preimages: make(map[common.Hash][]byte), } - // Copy the dirty states and logs + // Copy the dirty states, logs, and preimages for addr := range self.stateObjectsDirty { state.stateObjects[addr] = self.stateObjects[addr].deepCopy(state, state.MarkStateObjectDirty) state.stateObjectsDirty[addr] = struct{}{} @@ -487,6 +508,9 @@ func (self *StateDB) Copy() *StateDB { state.logs[hash] = make([]*types.Log, len(logs)) copy(state.logs[hash], logs) } + for hash, preimage := range self.preimages { + state.preimages[hash] = preimage + } return state } diff --git a/core/state_processor.go b/core/state_processor.go index 4f6ca651e1..6485e9abda 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -99,7 +99,7 @@ func ApplyTransaction(config *params.ChainConfig, bc *BlockChain, gp *GasPool, s context := NewEVMContext(msg, header, bc) // Create a new environment which holds all relevant information // about the transaction and calling mechanisms. - vmenv := vm.NewEVM(context, statedb, config, vm.Config{}) + vmenv := vm.NewEVM(context, statedb, config, cfg) // Apply the transaction to the current state (included in the env) _, gas, err := ApplyMessage(vmenv, msg, gp) if err != nil { diff --git a/core/vm/instructions.go b/core/vm/instructions.go index 5bfa73a30f..3b1b06cca1 100644 --- a/core/vm/instructions.go +++ b/core/vm/instructions.go @@ -247,7 +247,12 @@ func opMulmod(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *S func opSha3(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { offset, size := stack.pop(), stack.pop() - hash := crypto.Keccak256(memory.Get(offset.Int64(), size.Int64())) + data := memory.Get(offset.Int64(), size.Int64()) + hash := crypto.Keccak256(data) + + if env.vmConfig.EnablePreimageRecording { + env.StateDB.AddPreimage(common.BytesToHash(hash), data) + } stack.push(common.BytesToBig(hash)) return nil, nil diff --git a/core/vm/interface.go b/core/vm/interface.go index 8617b2d0f7..6f15112eea 100644 --- a/core/vm/interface.go +++ b/core/vm/interface.go @@ -60,6 +60,7 @@ type StateDB interface { Snapshot() int AddLog(*types.Log) + AddPreimage(common.Hash, []byte) } // Account represents a contract or basic ethereum account. diff --git a/core/vm/noop.go b/core/vm/noop.go index ef68372730..7835eeaf35 100644 --- a/core/vm/noop.go +++ b/core/vm/noop.go @@ -67,3 +67,4 @@ func (NoopStateDB) Empty(common.Address) bool { return f func (NoopStateDB) RevertToSnapshot(int) {} func (NoopStateDB) Snapshot() int { return 0 } func (NoopStateDB) AddLog(*types.Log) {} +func (NoopStateDB) AddPreimage(common.Hash, []byte) {} diff --git a/core/vm/vm.go b/core/vm/vm.go index 56081f12c3..a5f48750d1 100644 --- a/core/vm/vm.go +++ b/core/vm/vm.go @@ -44,6 +44,8 @@ type Config struct { NoRecursion bool // Disable gas metering DisableGasMetering bool + // Enable recording of SHA3/keccak preimages + EnablePreimageRecording bool // JumpTable contains the EVM instruction table. This // may me left uninitialised and will be set the default // table. diff --git a/eth/api.go b/eth/api.go index 023e9a9bba..07df0b79e3 100644 --- a/eth/api.go +++ b/eth/api.go @@ -560,3 +560,9 @@ func (api *PrivateDebugAPI) TraceTransaction(ctx context.Context, txHash common. } return nil, errors.New("database inconsistency") } + +// Preimage is a debug API function that returns the preimage for a sha3 hash, if known. +func (api *PrivateDebugAPI) Preimage(ctx context.Context, hash common.Hash) (hexutil.Bytes, error) { + db := core.PreimageTable(api.eth.ChainDb()) + return db.Get(hash.Bytes()) +} diff --git a/eth/backend.go b/eth/backend.go index dec8c0c6ed..02514c25bd 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -33,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/eth/gasprice" @@ -97,8 +98,7 @@ type Config struct { GpobaseStepUp int GpobaseCorrectionFactor int - EnableJit bool - ForceJit bool + EnablePreimageRecording bool TestGenesisBlock *types.Block // Genesis block to seed the chain database with (testing only!) TestGenesisState ethdb.Database // Genesis state to seed the database with (testing only!) @@ -218,7 +218,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { glog.V(logger.Info).Infoln("Chain config:", eth.chainConfig) - eth.blockchain, err = core.NewBlockChain(chainDb, eth.chainConfig, eth.pow, eth.EventMux()) + eth.blockchain, err = core.NewBlockChain(chainDb, eth.chainConfig, eth.pow, eth.EventMux(), vm.Config{EnablePreimageRecording: config.EnablePreimageRecording}) if err != nil { if err == core.ErrNoGenesis { return nil, fmt.Errorf(`No chain found. Please initialise a new chain using the "init" subcommand.`) diff --git a/eth/handler_test.go b/eth/handler_test.go index 22a4ddf50a..8a5d7173b3 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/ethdb" @@ -469,7 +470,7 @@ func testDAOChallenge(t *testing.T, localForked, remoteForked bool, timeout bool db, _ = ethdb.NewMemDatabase() genesis = core.WriteGenesisBlockForTesting(db) config = ¶ms.ChainConfig{DAOForkBlock: big.NewInt(1), DAOForkSupport: localForked} - blockchain, _ = core.NewBlockChain(db, config, pow, evmux) + blockchain, _ = core.NewBlockChain(db, config, pow, evmux, vm.Config{}) ) pm, err := NewProtocolManager(config, false, NetworkId, 1000, evmux, new(testTxPool), pow, blockchain, db) if err != nil { diff --git a/eth/helper_test.go b/eth/helper_test.go index a718a6d21b..0861c884bd 100644 --- a/eth/helper_test.go +++ b/eth/helper_test.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" @@ -56,7 +57,7 @@ func newTestProtocolManager(fastSync bool, blocks int, generator func(int, *core db, _ = ethdb.NewMemDatabase() genesis = core.WriteGenesisBlockForTesting(db, testBank) chainConfig = ¶ms.ChainConfig{HomesteadBlock: big.NewInt(0)} // homestead set to 0 because of chain maker - blockchain, _ = core.NewBlockChain(db, chainConfig, pow, evmux) + blockchain, _ = core.NewBlockChain(db, chainConfig, pow, evmux, vm.Config{}) ) chain, _ := core.GenerateChain(chainConfig, genesis, db, blocks, generator) if _, err := blockchain.InsertChain(chain); err != nil { diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index 12bdb69969..a265d24060 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -385,6 +385,12 @@ web3._extend({ call: 'debug_traceTransaction', params: 2, inputFormatter: [null, null] + }), + new web3._extend.Method({ + name: 'preimage', + call: 'debug_preimage', + params: 1, + inputFormatter: [null] }) ], properties: [] diff --git a/les/helper_test.go b/les/helper_test.go index 3d6bf3c29d..e0b7558eeb 100644 --- a/les/helper_test.go +++ b/les/helper_test.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" @@ -143,7 +144,7 @@ func newTestProtocolManager(lightSync bool, blocks int, generator func(int, *cor odr = NewLesOdr(db) chain, _ = light.NewLightChain(odr, chainConfig, pow, evmux) } else { - blockchain, _ := core.NewBlockChain(db, chainConfig, pow, evmux) + blockchain, _ := core.NewBlockChain(db, chainConfig, pow, evmux, vm.Config{}) gchain, _ := core.GenerateChain(chainConfig, genesis, db, blocks, generator) if _, err := blockchain.InsertChain(gchain); err != nil { panic(err) diff --git a/light/odr_test.go b/light/odr_test.go index 2dcfa40d90..a2f969acb3 100644 --- a/light/odr_test.go +++ b/light/odr_test.go @@ -251,7 +251,7 @@ func testChainOdr(t *testing.T, protocol int, expFail uint64, fn odrTestFn) { ) core.WriteGenesisBlockForTesting(ldb, core.GenesisAccount{Address: testBankAddress, Balance: testBankFunds}) // Assemble the test environment - blockchain, _ := core.NewBlockChain(sdb, testChainConfig(), pow, evmux) + blockchain, _ := core.NewBlockChain(sdb, testChainConfig(), pow, evmux, vm.Config{}) chainConfig := ¶ms.ChainConfig{HomesteadBlock: new(big.Int)} gchain, _ := core.GenerateChain(chainConfig, genesis, sdb, 4, testChainGen) if _, err := blockchain.InsertChain(gchain); err != nil { diff --git a/light/txpool_test.go b/light/txpool_test.go index e5a4670aaf..d8f04e6173 100644 --- a/light/txpool_test.go +++ b/light/txpool_test.go @@ -25,6 +25,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/params" @@ -88,7 +89,7 @@ func TestTxPool(t *testing.T) { ) core.WriteGenesisBlockForTesting(ldb, core.GenesisAccount{Address: testBankAddress, Balance: testBankFunds}) // Assemble the test environment - blockchain, _ := core.NewBlockChain(sdb, testChainConfig(), pow, evmux) + blockchain, _ := core.NewBlockChain(sdb, testChainConfig(), pow, evmux, vm.Config{}) chainConfig := ¶ms.ChainConfig{HomesteadBlock: new(big.Int)} gchain, _ := core.GenerateChain(chainConfig, genesis, sdb, poolTestBlocks, txPoolTestChainGen) if _, err := blockchain.InsertChain(gchain); err != nil { diff --git a/light/vm_env.go b/light/vm_env.go index 1b225c8de0..d2cc7e9608 100644 --- a/light/vm_env.go +++ b/light/vm_env.go @@ -45,6 +45,8 @@ func (s *VMState) Error() error { func (s *VMState) AddLog(log *types.Log) {} +func (s *VMState) AddPreimage(hash common.Hash, preimage []byte) {} + // errHandler handles and stores any state error that happens during execution. func (s *VMState) errHandler(err error) { if err != nil && s.err == nil { diff --git a/tests/block_test_util.go b/tests/block_test_util.go index ea63c99969..470eb7cb7f 100644 --- a/tests/block_test_util.go +++ b/tests/block_test_util.go @@ -31,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" @@ -172,7 +173,7 @@ func runBlockTest(homesteadBlock, daoForkBlock, gasPriceFork *big.Int, test *Blo core.WriteHeadBlockHash(db, test.Genesis.Hash()) evmux := new(event.TypeMux) config := ¶ms.ChainConfig{HomesteadBlock: homesteadBlock, DAOForkBlock: daoForkBlock, DAOForkSupport: true, EIP150Block: gasPriceFork} - chain, err := core.NewBlockChain(db, config, ethash.NewShared(), evmux) + chain, err := core.NewBlockChain(db, config, ethash.NewShared(), evmux, vm.Config{}) if err != nil { return err } From 230530f5ea71f45c5d89f62126e60d24635136d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 17 Jan 2017 13:25:36 +0200 Subject: [PATCH 07/21] accounts, mobile: make account manager API a bit more uniform --- accounts/account_manager.go | 6 +++--- accounts/accounts_test.go | 8 ++++---- mobile/accounts.go | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/accounts/account_manager.go b/accounts/account_manager.go index 6460b6e857..01dd62e25b 100644 --- a/accounts/account_manager.go +++ b/accounts/account_manager.go @@ -113,9 +113,9 @@ func (am *Manager) Accounts() []Account { return am.cache.accounts() } -// DeleteAccount deletes the key matched by account if the passphrase is correct. -// If a contains no filename, the address must match a unique key. -func (am *Manager) DeleteAccount(a Account, passphrase string) error { +// Delete deletes the key matched by account if the passphrase is correct. +// If the account contains no filename, the address must match a unique key. +func (am *Manager) Delete(a Account, passphrase string) error { // Decrypting the key isn't really necessary, but we do // it anyway to check the password and zero out the key // immediately afterwards. diff --git a/accounts/accounts_test.go b/accounts/accounts_test.go index f276059e28..b3ab87d503 100644 --- a/accounts/accounts_test.go +++ b/accounts/accounts_test.go @@ -53,14 +53,14 @@ func TestManager(t *testing.T) { if err := am.Update(a, "foo", "bar"); err != nil { t.Errorf("Update error: %v", err) } - if err := am.DeleteAccount(a, "bar"); err != nil { - t.Errorf("DeleteAccount error: %v", err) + if err := am.Delete(a, "bar"); err != nil { + t.Errorf("Delete error: %v", err) } if common.FileExist(a.File) { - t.Errorf("account file %s should be gone after DeleteAccount", a.File) + t.Errorf("account file %s should be gone after Delete", a.File) } if am.HasAddress(a.Address) { - t.Errorf("HasAccount(%x) should've returned true after DeleteAccount", a.Address) + t.Errorf("HasAccount(%x) should've returned true after Delete", a.Address) } } diff --git a/mobile/accounts.go b/mobile/accounts.go index 47c3a5c211..621be4d7a3 100644 --- a/mobile/accounts.go +++ b/mobile/accounts.go @@ -103,7 +103,7 @@ func (am *AccountManager) GetAccounts() *Accounts { // DeleteAccount deletes the key matched by account if the passphrase is correct. // If a contains no filename, the address must match a unique key. func (am *AccountManager) DeleteAccount(account *Account, passphrase string) error { - return am.manager.DeleteAccount(accounts.Account{ + return am.manager.Delete(accounts.Account{ Address: account.account.Address, File: account.account.File, }, passphrase) From 6fb76443b34ac5797c4b561ec38148eb3868a47e Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 17 Jan 2017 13:10:26 +0100 Subject: [PATCH 08/21] core/blockchain: Made logging of reorgs more structured (#3573) * core: Made logging of reorgs more structured, also always log if reorg is > 63 blocks long * core/blockchain: go fmt * core/blockchain: Minor fixes to the reorg reporting --- core/blockchain.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 2e522d97c4..90bb0b5a8e 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1088,8 +1088,6 @@ func (self *BlockChain) reorg(oldBlock, newBlock *types.Block) error { newChain types.Blocks oldChain types.Blocks commonBlock *types.Block - oldStart = oldBlock - newStart = newBlock deletedTxs types.Transactions deletedLogs []*types.Log // collectLogs collects the logs that were generated during the @@ -1130,7 +1128,6 @@ func (self *BlockChain) reorg(oldBlock, newBlock *types.Block) error { return fmt.Errorf("Invalid new chain") } - numSplit := newBlock.Number() for { if oldBlock.Hash() == newBlock.Hash() { commonBlock = oldBlock @@ -1151,9 +1148,19 @@ func (self *BlockChain) reorg(oldBlock, newBlock *types.Block) error { } } - if glog.V(logger.Debug) { - commonHash := commonBlock.Hash() - glog.Infof("Chain split detected @ %x. Reorganising chain from #%v %x to %x", commonHash[:4], numSplit, oldStart.Hash().Bytes()[:4], newStart.Hash().Bytes()[:4]) + if oldLen := len(oldChain); oldLen > 63 || glog.V(logger.Debug) { + newLen := len(newChain) + newLast := newChain[0] + newFirst := newChain[newLen-1] + oldLast := oldChain[0] + oldFirst := oldChain[oldLen-1] + glog.Infof("Chain split detected after #%v [%x…]. Reorganising chain (-%v +%v blocks), rejecting #%v-#%v [%x…/%x…] in favour of #%v-#%v [%x…/%x…]", + commonBlock.Number(), commonBlock.Hash().Bytes()[:4], + oldLen, newLen, + oldFirst.Number(), oldLast.Number(), + oldFirst.Hash().Bytes()[:4], oldLast.Hash().Bytes()[:4], + newFirst.Number(), newLast.Number(), + newFirst.Hash().Bytes()[:4], newLast.Hash().Bytes()[:4]) } var addedTxs types.Transactions From 508fdc3496542e10de1adb6d7c55030e54d66d8f Mon Sep 17 00:00:00 2001 From: Jeffrey Wilcke Date: Tue, 17 Jan 2017 21:49:08 +0100 Subject: [PATCH 09/21] core: removal of dead-code Removal of dead code that appeard as if we had a consensus issue. This however is not the case as the proper error catching happens in the vm package instead. --- core/state_transition.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/core/state_transition.go b/core/state_transition.go index 38fbebfd95..3cb7e500c5 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -233,9 +233,6 @@ func (self *StateTransition) TransitionDb() (ret []byte, requiredGas, usedGas *b ) if contractCreation { ret, _, vmerr = vmenv.Create(sender, self.data, self.gas, self.value) - if homestead && err == vm.ErrCodeStoreOutOfGas { - self.gas = Big0 - } } else { // Increment the nonce for the next transaction self.state.SetNonce(sender.Address(), self.state.GetNonce(sender.Address())+1) From ebc3d232f4b6c197d5dc5da2e1868d56b2096f1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 20 Jan 2017 02:12:14 +0200 Subject: [PATCH 10/21] eth/downloader: fix mutex regression causing panics on fail (#3591) --- eth/downloader/queue.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eth/downloader/queue.go b/eth/downloader/queue.go index dd9590b287..3318879e28 100644 --- a/eth/downloader/queue.go +++ b/eth/downloader/queue.go @@ -1129,12 +1129,13 @@ func (q *queue) deliverNodeData(results []trie.SyncResult, callback func(int, bo if err != nil { q.stateSchedLock.Unlock() callback(i, progressed, err) + return } if err = batch.Write(); err != nil { q.stateSchedLock.Unlock() callback(i, progressed, err) + return // TODO(karalabe): If a DB write fails (disk full), we ought to cancel the sync } - // Item processing succeeded, release the lock (temporarily) progressed = progressed || prog q.stateSchedLock.Unlock() From 7814a8e1315b1746d5260562de4c0b6f161a1cc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 20 Jan 2017 11:33:58 +0200 Subject: [PATCH 11/21] travis: Install Android NDK explicitly, removed from gomobile (#3593) The Android NDK was recently removed from gomobile, leading to our Android builds failing. Starting from https://go-review.googlesource.com/#/c/35173/ , gomobile requires a locally installed NDK. This PR ensures that travis installs that too before running the build steps. --- .travis.yml | 3 ++- build/ci.go | 9 ++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index baa5db7ba9..3384f3ec66 100644 --- a/.travis.yml +++ b/.travis.yml @@ -65,10 +65,11 @@ matrix: # Build the Android archive and upload it to Maven Central and Azure - brew update - - brew install android-sdk maven gpg + - travis_wait 60 brew install android-sdk android-ndk maven gpg - alias gpg="gpg2" - export ANDROID_HOME=/usr/local/opt/android-sdk + - export ANDROID_NDK=/usr/local/opt/android-ndk - echo "y" | android update sdk --no-ui --filter `android list sdk | grep "SDK Platform Android" | grep -E 'API 15|API 19|API 24' | awk '{print $1}' | cut -d '-' -f 1 | tr '\n' ','` - go run build/ci.go aar -signer ANDROID_SIGNING_KEY -deploy https://oss.sonatype.org -upload gethstore/builds diff --git a/build/ci.go b/build/ci.go index a3213e7c91..319691e8a0 100644 --- a/build/ci.go +++ b/build/ci.go @@ -700,9 +700,16 @@ func doAndroidArchive(cmdline []string) { flag.CommandLine.Parse(cmdline) env := build.Env() + // Sanity check that the SDK and NDK are installed and set + if os.Getenv("ANDROID_HOME") == "" { + log.Fatal("Please ensure ANDROID_HOME points to your Android SDK") + } + if os.Getenv("ANDROID_NDK") == "" { + log.Fatal("Please ensure ANDROID_NDK points to your Android NDK") + } // Build the Android archive and Maven resources build.MustRun(goTool("get", "golang.org/x/mobile/cmd/gomobile")) - build.MustRun(gomobileTool("init")) + build.MustRun(gomobileTool("init", "--ndk", os.Getenv("ANDROID_NDK"))) build.MustRun(gomobileTool("bind", "--target", "android", "--javapkg", "org.ethereum", "-v", "github.com/ethereum/go-ethereum/mobile")) if *local { From 946db8ba6547e5b18c5af701eb0e9e49a4298cbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 20 Jan 2017 12:50:21 +0200 Subject: [PATCH 12/21] internal/guide: initial test suite to ensure guide snippets run ok (#3582) --- internal/guide/guide.go | 18 +++++++ internal/guide/guide_test.go | 100 +++++++++++++++++++++++++++++++++++ 2 files changed, 118 insertions(+) create mode 100644 internal/guide/guide.go create mode 100644 internal/guide/guide_test.go diff --git a/internal/guide/guide.go b/internal/guide/guide.go new file mode 100644 index 0000000000..44a0bc4845 --- /dev/null +++ b/internal/guide/guide.go @@ -0,0 +1,18 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Package guide is a small test suite to ensure snippets in the dev guide work. +package guide diff --git a/internal/guide/guide_test.go b/internal/guide/guide_test.go new file mode 100644 index 0000000000..8f89037bd6 --- /dev/null +++ b/internal/guide/guide_test.go @@ -0,0 +1,100 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// This file contains the code snippets from the developer's guide embedded into +// Go tests. This ensures that any code published in out guides will not break +// accidentally via some code update. If some API changes nonetheless that needs +// modifying this file, please port any modification over into the developer's +// guide wiki pages too! + +package guide + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" + "time" + + "github.com/ethereum/go-ethereum/accounts" + "github.com/ethereum/go-ethereum/common" +) + +// Tests that the account management snippets work correctly. +func TestAccountManagement(t *testing.T) { + // Create a temporary folder to work with + workdir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("Failed to create temporary work dir: %v", err) + } + defer os.RemoveAll(workdir) + + // Create an encrypted keystore manager with standard crypto parameters + am := accounts.NewManager(filepath.Join(workdir, "keystore"), accounts.StandardScryptN, accounts.StandardScryptP) + + // Create a new account with the specified encryption passphrase + newAcc, err := am.NewAccount("Creation password") + if err != nil { + t.Fatalf("Failed to create new account: %v", err) + } + // Export the newly created account with a different passphrase. The returned + // data from this method invocation is a JSON encoded, encrypted key-file + jsonAcc, err := am.Export(newAcc, "Creation password", "Export password") + if err != nil { + t.Fatalf("Failed to export account: %v", err) + } + // Update the passphrase on the account created above inside the local keystore + if err := am.Update(newAcc, "Creation password", "Update password"); err != nil { + t.Fatalf("Failed to update account: %v", err) + } + // Delete the account updated above from the local keystore + if err := am.Delete(newAcc, "Update password"); err != nil { + t.Fatalf("Failed to delete account: %v", err) + } + // Import back the account we've exported (and then deleted) above with yet + // again a fresh passphrase + if _, err := am.Import(jsonAcc, "Export password", "Import password"); err != nil { + t.Fatalf("Failed to import account: %v", err) + } + // Create a new account to sign transactions with + signer, err := am.NewAccount("Signer password") + if err != nil { + t.Fatalf("Failed to create signer account: %v", err) + } + txHash := common.HexToHash("0x0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef") + + // Sign a transaction with a single authorization + if _, err := am.SignWithPassphrase(signer, "Signer password", txHash.Bytes()); err != nil { + t.Fatalf("Failed to sign with passphrase: %v", err) + } + // Sign a transaction with multiple manually cancelled authorizations + if err := am.Unlock(signer, "Signer password"); err != nil { + t.Fatalf("Failed to unlock account: %v", err) + } + if _, err := am.Sign(signer.Address, txHash.Bytes()); err != nil { + t.Fatalf("Failed to sign with unlocked account: %v", err) + } + if err := am.Lock(signer.Address); err != nil { + t.Fatalf("Failed to lock account: %v", err) + } + // Sign a transaction with multiple automatically cancelled authorizations + if err := am.TimedUnlock(signer, "Signer password", time.Second); err != nil { + t.Fatalf("Failed to time unlock account: %v", err) + } + if _, err := am.Sign(signer.Address, txHash.Bytes()); err != nil { + t.Fatalf("Failed to sign with time unlocked account: %v", err) + } +} From 0126d014351bc4f58e2d7e6564a054fb80f87153 Mon Sep 17 00:00:00 2001 From: bas-vk Date: Fri, 20 Jan 2017 23:32:16 +0100 Subject: [PATCH 13/21] types: bugfix invalid V derivation on tx json unmarshal (#3594) --- core/types/transaction.go | 3 ++- core/types/transaction_signing.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/core/types/transaction.go b/core/types/transaction.go index e610671d31..0b6d24268b 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -198,7 +198,8 @@ func (tx *Transaction) UnmarshalJSON(input []byte) error { var V byte if isProtectedV((*big.Int)(dec.V)) { - V = byte((new(big.Int).Sub((*big.Int)(dec.V), deriveChainId((*big.Int)(dec.V))).Uint64()) - 35) + chainId := deriveChainId((*big.Int)(dec.V)).Uint64() + V = byte(dec.V.ToInt().Uint64() - 35 - 2*chainId) } else { V = byte(((*big.Int)(dec.V)).Uint64() - 27) } diff --git a/core/types/transaction_signing.go b/core/types/transaction_signing.go index 4ebc789a59..7d7b63e9f1 100644 --- a/core/types/transaction_signing.go +++ b/core/types/transaction_signing.go @@ -160,7 +160,7 @@ func (s EIP155Signer) PublicKey(tx *Transaction) ([]byte, error) { // needs to be in the [R || S || V] format where V is 0 or 1. func (s EIP155Signer) WithSignature(tx *Transaction, sig []byte) (*Transaction, error) { if len(sig) != 65 { - panic(fmt.Sprintf("wrong size for snature: got %d, want 65", len(sig))) + panic(fmt.Sprintf("wrong size for signature: got %d, want 65", len(sig))) } cpy := &Transaction{data: tx.data} From 682875adff760a29a2bb0024190883e4b4dd5d72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Sat, 21 Jan 2017 00:39:16 +0200 Subject: [PATCH 14/21] accounts/abi/bind, internal/ethapi: binary search gas estimation (#3587) Gas estimation currently mostly works, but can underestimate for more funky refunds. This is because various ops (e.g. CALL) need more gas to run than they actually consume (e.g. 2300 stipend that is refunded if not used). With more intricate contract interplays, it becomes almost impossible to return a proper value to the user. This commit swaps out the simplistic gas estimation to a binary search approach, honing in on the correct gas use. This does mean that gas estimation needs to rerun the transaction log(max-price) times to measure whether it fails or not, but it's a price paid by the transaction issuer, and it should be worth it to support proper estimates. --- accounts/abi/bind/backends/simulated.go | 28 ++++++++++++-- accounts/abi/bind/bind_test.go | 51 +++++++++++++++++++++++-- internal/ethapi/api.go | 30 ++++++++++++++- 3 files changed, 100 insertions(+), 9 deletions(-) diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index c26662ea19..1c34ba0e79 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -201,10 +201,32 @@ func (b *SimulatedBackend) SuggestGasPrice(ctx context.Context) (*big.Int, error func (b *SimulatedBackend) EstimateGas(ctx context.Context, call ethereum.CallMsg) (*big.Int, error) { b.mu.Lock() defer b.mu.Unlock() - defer b.pendingState.RevertToSnapshot(b.pendingState.Snapshot()) - _, gas, err := b.callContract(ctx, call, b.pendingBlock, b.pendingState) - return gas, err + // Binary search the gas requirement, as it may be higher than the amount used + var lo, hi uint64 + if call.Gas != nil { + hi = call.Gas.Uint64() + } else { + hi = b.pendingBlock.GasLimit().Uint64() + } + for lo+1 < hi { + // Take a guess at the gas, and check transaction validity + mid := (hi + lo) / 2 + call.Gas = new(big.Int).SetUint64(mid) + + snapshot := b.pendingState.Snapshot() + _, gas, err := b.callContract(ctx, call, b.pendingBlock, b.pendingState) + b.pendingState.RevertToSnapshot(snapshot) + + // If the transaction became invalid or used all the gas (failed), raise the gas limit + if err != nil || gas.Cmp(call.Gas) == 0 { + lo = mid + continue + } + // Otherwise assume the transaction succeeded, lower the gas limit + hi = mid + } + return new(big.Int).SetUint64(hi), nil } // callContract implemens common code between normal and pending contract calls. diff --git a/accounts/abi/bind/bind_test.go b/accounts/abi/bind/bind_test.go index 6ebc8ea0a3..eb46bc081d 100644 --- a/accounts/abi/bind/bind_test.go +++ b/accounts/abi/bind/bind_test.go @@ -341,11 +341,11 @@ var bindTests = []struct { { `NonExistent`, ` - contract NonExistent { - function String() constant returns(string) { - return "I don't exist"; + contract NonExistent { + function String() constant returns(string) { + return "I don't exist"; + } } - } `, `6060604052609f8060106000396000f3606060405260e060020a6000350463f97a60058114601a575b005b600060605260c0604052600d60809081527f4920646f6e27742065786973740000000000000000000000000000000000000060a052602060c0908152600d60e081905281906101009060a09080838184600060046012f15050815172ffffffffffffffffffffffffffffffffffffff1916909152505060405161012081900392509050f3`, `[{"constant":true,"inputs":[],"name":"String","outputs":[{"name":"","type":"string"}],"type":"function"}]`, @@ -365,6 +365,49 @@ var bindTests = []struct { } `, }, + // Tests that gas estimation works for contracts with weird gas mechanics too. + { + `FunkyGasPattern`, + ` + contract FunkyGasPattern { + string public field; + + function SetField(string value) { + // This check will screw gas estimation! Good, good! + if (msg.gas < 100000) { + throw; + } + field = value; + } + } + `, + `606060405261021c806100126000396000f3606060405260e060020a600035046323fcf32a81146100265780634f28bf0e1461007b575b005b6040805160206004803580820135601f8101849004840285018401909552848452610024949193602493909291840191908190840183828082843750949650505050505050620186a05a101561014e57610002565b6100db60008054604080516020601f600260001961010060018816150201909516949094049384018190048102820181019092528281529291908301828280156102145780601f106101e957610100808354040283529160200191610214565b60405180806020018281038252838181518152602001915080519060200190808383829060006004602084601f0104600302600f01f150905090810190601f16801561013b5780820380516001836020036101000a031916815260200191505b509250505060405180910390f35b505050565b8060006000509080519060200190828054600181600116156101000203166002900490600052602060002090601f016020900481019282601f106101b557805160ff19168380011785555b506101499291505b808211156101e557600081556001016101a1565b82800160010185558215610199579182015b828111156101995782518260005055916020019190600101906101c7565b5090565b820191906000526020600020905b8154815290600101906020018083116101f757829003601f168201915b50505050508156`, + `[{"constant":false,"inputs":[{"name":"value","type":"string"}],"name":"SetField","outputs":[],"type":"function"},{"constant":true,"inputs":[],"name":"field","outputs":[{"name":"","type":"string"}],"type":"function"}]`, + ` + // Generate a new random account and a funded simulator + key, _ := crypto.GenerateKey() + auth := bind.NewKeyedTransactor(key) + sim := backends.NewSimulatedBackend(core.GenesisAccount{Address: auth.From, Balance: big.NewInt(10000000000)}) + + // Deploy a funky gas pattern contract + _, _, limiter, err := DeployFunkyGasPattern(auth, sim) + if err != nil { + t.Fatalf("Failed to deploy funky contract: %v", err) + } + sim.Commit() + + // Set the field with automatic estimation and check that it succeeds + auth.GasLimit = nil + if _, err := limiter.SetField(auth, "automatic"); err != nil { + t.Fatalf("Failed to call automatically gased transaction: %v", err) + } + sim.Commit() + + if field, _ := limiter.Field(nil); field != "automatic" { + t.Fatalf("Field mismatch: have %v, want %v", field, "automatic") + } + `, + }, } // Tests that packages generated by the binder can be successfully compiled and diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 4359181c87..925f547b60 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -559,8 +559,34 @@ func (s *PublicBlockChainAPI) Call(ctx context.Context, args CallArgs, blockNr r // EstimateGas returns an estimate of the amount of gas needed to execute the given transaction. func (s *PublicBlockChainAPI) EstimateGas(ctx context.Context, args CallArgs) (*hexutil.Big, error) { - _, gas, err := s.doCall(ctx, args, rpc.PendingBlockNumber) - return (*hexutil.Big)(gas), err + // Binary search the gas requirement, as it may be higher than the amount used + var lo, hi uint64 + if (*big.Int)(&args.Gas).BitLen() > 0 { + hi = (*big.Int)(&args.Gas).Uint64() + } else { + // Retrieve the current pending block to act as the gas ceiling + block, err := s.b.BlockByNumber(ctx, rpc.PendingBlockNumber) + if err != nil { + return nil, err + } + hi = block.GasLimit().Uint64() + } + for lo+1 < hi { + // Take a guess at the gas, and check transaction validity + mid := (hi + lo) / 2 + (*big.Int)(&args.Gas).SetUint64(mid) + + _, gas, err := s.doCall(ctx, args, rpc.PendingBlockNumber) + + // If the transaction became invalid or used all the gas (failed), raise the gas limit + if err != nil || gas.Cmp((*big.Int)(&args.Gas)) == 0 { + lo = mid + continue + } + // Otherwise assume the transaction succeeded, lower the gas limit + hi = mid + } + return (*hexutil.Big)(new(big.Int).SetUint64(hi)), nil } // ExecutionResult groups all structured logs emitted by the EVM From 935d891e9db870edde6b37ad84bb21d7df19c383 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Sun, 22 Jan 2017 22:14:09 +0100 Subject: [PATCH 15/21] cmd/evm: added debug flag (back) (#3554) * evm: added debug flag (back) * cmd/evm: gofmt --- cmd/evm/main.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/evm/main.go b/cmd/evm/main.go index 035cf1c545..9f67e66286 100644 --- a/cmd/evm/main.go +++ b/cmd/evm/main.go @@ -161,6 +161,7 @@ func run(ctx *cli.Context) error { Value: common.Big(ctx.GlobalString(ValueFlag.Name)), EVMConfig: vm.Config{ Tracer: logger, + Debug: ctx.GlobalBool(DebugFlag.Name), DisableGasMetering: ctx.GlobalBool(DisableGasMeteringFlag.Name), }, }) @@ -176,6 +177,7 @@ func run(ctx *cli.Context) error { Value: common.Big(ctx.GlobalString(ValueFlag.Name)), EVMConfig: vm.Config{ Tracer: logger, + Debug: ctx.GlobalBool(DebugFlag.Name), DisableGasMetering: ctx.GlobalBool(DisableGasMeteringFlag.Name), }, }) From 96778a1c216f7d0d987dd8ea6474b2d3eebe9cfc Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Sun, 22 Jan 2017 23:28:47 +0100 Subject: [PATCH 16/21] crypto/secp256k1: sign with deterministic K (rfc6979) (#3561) --- crypto/secp256k1/secp256.go | 12 ++++-------- crypto/secp256k1/secp256_test.go | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/crypto/secp256k1/secp256.go b/crypto/secp256k1/secp256.go index 070e0d902f..4284115e2d 100644 --- a/crypto/secp256k1/secp256.go +++ b/crypto/secp256k1/secp256.go @@ -40,8 +40,6 @@ import ( "errors" "math/big" "unsafe" - - "github.com/ethereum/go-ethereum/crypto/randentropy" ) var ( @@ -89,13 +87,11 @@ func Sign(msg []byte, seckey []byte) ([]byte, error) { } var ( - msgdata = (*C.uchar)(unsafe.Pointer(&msg[0])) - nonce = randentropy.GetEntropyCSPRNG(32) - noncefunc = &(*C.secp256k1_nonce_function_default) - noncefuncData = unsafe.Pointer(&nonce[0]) - sigstruct C.secp256k1_ecdsa_recoverable_signature + msgdata = (*C.uchar)(unsafe.Pointer(&msg[0])) + noncefunc = C.secp256k1_nonce_function_rfc6979 + sigstruct C.secp256k1_ecdsa_recoverable_signature ) - if C.secp256k1_ecdsa_sign_recoverable(context, &sigstruct, msgdata, seckeydata, noncefunc, noncefuncData) == 0 { + if C.secp256k1_ecdsa_sign_recoverable(context, &sigstruct, msgdata, seckeydata, noncefunc, nil) == 0 { return nil, ErrSignFailed } diff --git a/crypto/secp256k1/secp256_test.go b/crypto/secp256k1/secp256_test.go index ec28b8e39e..287ab512ec 100644 --- a/crypto/secp256k1/secp256_test.go +++ b/crypto/secp256k1/secp256_test.go @@ -112,6 +112,24 @@ func TestSignAndRecover(t *testing.T) { } } +func TestSignDeterministic(t *testing.T) { + _, seckey := generateKeyPair() + msg := make([]byte, 32) + copy(msg, "hi there") + + sig1, err := Sign(msg, seckey) + if err != nil { + t.Fatal(err) + } + sig2, err := Sign(msg, seckey) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(sig1, sig2) { + t.Fatal("signatures not equal") + } +} + func TestRandomMessagesWithSameKey(t *testing.T) { pubkey, seckey := generateKeyPair() keys := func() ([]byte, []byte) { From c04598f2b09ab66204b30b1c9242b4dfadcd6bdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 23 Jan 2017 11:46:15 +0200 Subject: [PATCH 17/21] containers/docker: update to alpine 3.5, add CA certificates --- containers/docker/develop-alpine/Dockerfile | 4 ++-- containers/docker/master-alpine/Dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/containers/docker/develop-alpine/Dockerfile b/containers/docker/develop-alpine/Dockerfile index 3393c43374..a8d85bc63a 100644 --- a/containers/docker/develop-alpine/Dockerfile +++ b/containers/docker/develop-alpine/Dockerfile @@ -1,7 +1,7 @@ -FROM alpine:3.4 +FROM alpine:3.5 RUN \ - apk add --update go git make gcc musl-dev && \ + apk add --update go git make gcc musl-dev ca-certificates && \ git clone --depth 1 https://github.com/ethereum/go-ethereum && \ (cd go-ethereum && make geth) && \ cp go-ethereum/build/bin/geth /geth && \ diff --git a/containers/docker/master-alpine/Dockerfile b/containers/docker/master-alpine/Dockerfile index 5131c473a6..0db583a439 100644 --- a/containers/docker/master-alpine/Dockerfile +++ b/containers/docker/master-alpine/Dockerfile @@ -1,7 +1,7 @@ -FROM alpine:3.4 +FROM alpine:3.5 RUN \ - apk add --update go git make gcc musl-dev && \ + apk add --update go git make gcc musl-dev ca-certificates && \ git clone --depth 1 --branch release/1.5 https://github.com/ethereum/go-ethereum && \ (cd go-ethereum && make geth) && \ cp go-ethereum/build/bin/geth /geth && \ From 0b9070fe019a79740af3271109c6ce550911cb60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 23 Jan 2017 12:12:38 +0200 Subject: [PATCH 18/21] containers/docker: update ubuntu images to build, not pull --- containers/docker/develop-ubuntu/Dockerfile | 22 ++++++++++----------- containers/docker/master-ubuntu/Dockerfile | 22 ++++++++++----------- 2 files changed, 20 insertions(+), 24 deletions(-) diff --git a/containers/docker/develop-ubuntu/Dockerfile b/containers/docker/develop-ubuntu/Dockerfile index 98b4aadf8f..c79becb55a 100644 --- a/containers/docker/develop-ubuntu/Dockerfile +++ b/containers/docker/develop-ubuntu/Dockerfile @@ -1,17 +1,15 @@ -FROM ubuntu:wily -MAINTAINER caktux +FROM ubuntu:xenial -ENV DEBIAN_FRONTEND noninteractive - -RUN apt-get update && \ - apt-get upgrade -q -y && \ - apt-get dist-upgrade -q -y && \ - apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 923F6CA9 && \ - echo "deb http://ppa.launchpad.net/ethereum/ethereum-dev/ubuntu wily main" | tee -a /etc/apt/sources.list.d/ethereum.list && \ - apt-get update && \ - apt-get install -q -y geth +RUN \ + apt-get update && apt-get upgrade -q -y && \ + apt-get install -y --no-install-recommends golang git make gcc libc-dev ca-certificates && \ + git clone --depth 1 https://github.com/ethereum/go-ethereum && \ + (cd go-ethereum && make geth) && \ + cp go-ethereum/build/bin/geth /geth && \ + apt-get remove -y golang git make gcc libc-dev && apt autoremove -y && apt-get clean && \ + rm -rf /go-ethereum EXPOSE 8545 EXPOSE 30303 -ENTRYPOINT ["/usr/bin/geth"] +ENTRYPOINT ["/geth"] diff --git a/containers/docker/master-ubuntu/Dockerfile b/containers/docker/master-ubuntu/Dockerfile index 2c6de28c92..877ae94e99 100644 --- a/containers/docker/master-ubuntu/Dockerfile +++ b/containers/docker/master-ubuntu/Dockerfile @@ -1,17 +1,15 @@ -FROM ubuntu:wily -MAINTAINER caktux +FROM ubuntu:xenial -ENV DEBIAN_FRONTEND noninteractive - -RUN apt-get update && \ - apt-get upgrade -q -y && \ - apt-get dist-upgrade -q -y && \ - apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 923F6CA9 && \ - echo "deb http://ppa.launchpad.net/ethereum/ethereum/ubuntu wily main" | tee -a /etc/apt/sources.list.d/ethereum.list && \ - apt-get update && \ - apt-get install -q -y geth +RUN \ + apt-get update && apt-get upgrade -q -y && \ + apt-get install -y --no-install-recommends golang git make gcc libc-dev ca-certificates && \ + git clone --depth 1 --branch release/1.5 https://github.com/ethereum/go-ethereum && \ + (cd go-ethereum && make geth) && \ + cp go-ethereum/build/bin/geth /geth && \ + apt-get remove -y golang git make gcc libc-dev && apt autoremove -y && apt-get clean && \ + rm -rf /go-ethereum EXPOSE 8545 EXPOSE 30303 -ENTRYPOINT ["/usr/bin/geth"] +ENTRYPOINT ["/geth"] From fc52f2c007fb4a9065cd1b0ec402f7cbbae4dc5e Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Mon, 23 Jan 2017 18:51:02 +0100 Subject: [PATCH 19/21] core/types: make Transaction zero value printable (#3595) --- core/types/transaction.go | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/core/types/transaction.go b/core/types/transaction.go index 0b6d24268b..9382acb70b 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -134,7 +134,7 @@ func (tx *Transaction) ChainId() *big.Int { return deriveChainId(tx.data.V) } -// Protected returns whether the transaction is pretected from replay protection +// Protected returns whether the transaction is protected from replay protection. func (tx *Transaction) Protected() bool { return isProtectedV(tx.data.V) } @@ -311,16 +311,20 @@ func (tx *Transaction) RawSignatureValues() (*big.Int, *big.Int, *big.Int) { } func (tx *Transaction) String() string { - // make a best guess about the signer and use that to derive - // the sender. - signer := deriveSigner(tx.data.V) - var from, to string - if f, err := Sender(signer, tx); err != nil { // derive but don't cache - from = "[invalid sender: invalid sig]" + if tx.data.V != nil { + // make a best guess about the signer and use that to derive + // the sender. + signer := deriveSigner(tx.data.V) + if f, err := Sender(signer, tx); err != nil { // derive but don't cache + from = "[invalid sender: invalid sig]" + } else { + from = fmt.Sprintf("%x", f[:]) + } } else { - from = fmt.Sprintf("%x", f[:]) + from = "[invalid sender: nil V field]" } + if tx.data.Recipient == nil { to = "[contract creation]" } else { @@ -333,13 +337,13 @@ func (tx *Transaction) String() string { From: %s To: %s Nonce: %v - GasPrice: %v - GasLimit %v - Value: %v + GasPrice: %#x + GasLimit %#x + Value: %#x Data: 0x%x - V: 0x%x - R: 0x%x - S: 0x%x + V: %#x + R: %#x + S: %#x Hex: %x `, tx.Hash(), From f1069a30b9ca13769ace154ff25dfabc83f25485 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 24 Jan 2017 13:20:37 +0100 Subject: [PATCH 20/21] eth/downloader: improve deliverNodeData (#3588) Commit d3b751e accidentally deleted a crucial 'return' statement, leading to a crash in case of an issue with node data. This change improves the fix in PR #3591 by removing the lock entirely. --- eth/downloader/queue.go | 124 +++++++++++++++++----------------------- 1 file changed, 52 insertions(+), 72 deletions(-) diff --git a/eth/downloader/queue.go b/eth/downloader/queue.go index 3318879e28..5be09f37dc 100644 --- a/eth/downloader/queue.go +++ b/eth/downloader/queue.go @@ -23,7 +23,6 @@ import ( "errors" "fmt" "sync" - "sync/atomic" "time" "github.com/ethereum/go-ethereum/common" @@ -101,10 +100,9 @@ type queue struct { stateTaskQueue *prque.Prque // [eth/63] Priority queue of the hashes to fetch the node data for statePendPool map[string]*fetchRequest // [eth/63] Currently pending node data retrieval operations - stateDatabase ethdb.Database // [eth/63] Trie database to populate during state reassembly - stateScheduler *state.StateSync // [eth/63] State trie synchronisation scheduler and integrator - stateProcessors int32 // [eth/63] Number of currently running state processors - stateSchedLock sync.RWMutex // [eth/63] Lock serialising access to the state scheduler + stateDatabase ethdb.Database // [eth/63] Trie database to populate during state reassembly + stateScheduler *state.StateSync // [eth/63] State trie synchronisation scheduler and integrator + stateWriters int // [eth/63] Number of running state DB writer goroutines resultCache []*fetchResult // Downloaded but not yet delivered fetch results resultOffset uint64 // Offset of the first cached fetch result in the block chain @@ -143,9 +141,6 @@ func (q *queue) Reset() { q.lock.Lock() defer q.lock.Unlock() - q.stateSchedLock.Lock() - defer q.stateSchedLock.Unlock() - q.closed = false q.mode = FullSync q.fastSyncPivot = 0 @@ -209,13 +204,24 @@ func (q *queue) PendingReceipts() int { // PendingNodeData retrieves the number of node data entries pending for retrieval. func (q *queue) PendingNodeData() int { - q.stateSchedLock.RLock() - defer q.stateSchedLock.RUnlock() + q.lock.Lock() + defer q.lock.Unlock() + return q.pendingNodeDataLocked() +} + +// pendingNodeDataLocked retrieves the number of node data entries pending for retrieval. +// The caller must hold q.lock. +func (q *queue) pendingNodeDataLocked() int { + var n int if q.stateScheduler != nil { - return q.stateScheduler.Pending() + n = q.stateScheduler.Pending() } - return 0 + // Ensure that PendingNodeData doesn't return 0 until all state is written. + if q.stateWriters > 0 { + n++ + } + return n } // InFlightHeaders retrieves whether there are header fetch requests currently @@ -251,7 +257,7 @@ func (q *queue) InFlightNodeData() bool { q.lock.Lock() defer q.lock.Unlock() - return len(q.statePendPool)+int(atomic.LoadInt32(&q.stateProcessors)) > 0 + return len(q.statePendPool)+q.stateWriters > 0 } // Idle returns if the queue is fully idle or has some data still inside. This @@ -264,12 +270,9 @@ func (q *queue) Idle() bool { pending := len(q.blockPendPool) + len(q.receiptPendPool) + len(q.statePendPool) cached := len(q.blockDonePool) + len(q.receiptDonePool) - q.stateSchedLock.RLock() if q.stateScheduler != nil { queued += q.stateScheduler.Pending() } - q.stateSchedLock.RUnlock() - return (queued + pending + cached) == 0 } @@ -398,9 +401,7 @@ func (q *queue) Schedule(headers []*types.Header, from uint64) []*types.Header { req.Hashes = make(map[common.Hash]int) // Make sure executing requests fail, but don't disappear } - q.stateSchedLock.Lock() q.stateScheduler = state.NewStateSync(header.Root, q.stateDatabase) - q.stateSchedLock.Unlock() } inserts = append(inserts, header) q.headerHead = hash @@ -459,7 +460,7 @@ func (q *queue) countProcessableItems() int { // resultCache has space for fsHeaderForceVerify items. Not // doing this could leave us unable to download the required // amount of headers. - if i > 0 || len(q.stateTaskPool) > 0 || q.PendingNodeData() > 0 { + if i > 0 || len(q.stateTaskPool) > 0 || q.pendingNodeDataLocked() > 0 { return i } for j := 0; j < fsHeaderForceVerify; j++ { @@ -524,9 +525,6 @@ func (q *queue) ReserveHeaders(p *peer, count int) *fetchRequest { func (q *queue) ReserveNodeData(p *peer, count int) *fetchRequest { // Create a task generator to fetch status-fetch tasks if all schedules ones are done generator := func(max int) { - q.stateSchedLock.Lock() - defer q.stateSchedLock.Unlock() - if q.stateScheduler != nil { for _, hash := range q.stateScheduler.Missing(max) { q.stateTaskPool[hash] = q.stateTaskIndex @@ -1068,7 +1066,7 @@ func (q *queue) DeliverNodeData(id string, data [][]byte, callback func(int, boo } } // Iterate over the downloaded data and verify each of them - accepted, errs := 0, make([]error, 0) + errs := make([]error, 0) process := []trie.SyncResult{} for _, blob := range data { // Skip any state trie entries that were not requested @@ -1079,70 +1077,52 @@ func (q *queue) DeliverNodeData(id string, data [][]byte, callback func(int, boo } // Inject the next state trie item into the processing queue process = append(process, trie.SyncResult{Hash: hash, Data: blob}) - accepted++ - delete(request.Hashes, hash) delete(q.stateTaskPool, hash) } - // Start the asynchronous node state data injection - atomic.AddInt32(&q.stateProcessors, 1) - go func() { - defer atomic.AddInt32(&q.stateProcessors, -1) - q.deliverNodeData(process, callback) - }() // Return all failed or missing fetches to the queue for hash, index := range request.Hashes { q.stateTaskQueue.Push(hash, float32(index)) } + if q.stateScheduler == nil { + return 0, errNoFetchesPending + } + + // Run valid nodes through the trie download scheduler. It writes completed nodes to a + // batch, which is committed asynchronously. This may lead to over-fetches because the + // scheduler treats everything as written after Process has returned, but it's + // unlikely to be an issue in practice. + batch := q.stateDatabase.NewBatch() + progressed, nproc, procerr := q.stateScheduler.Process(process, batch) + q.stateWriters += 1 + go func() { + if procerr == nil { + nproc = len(process) + procerr = batch.Write() + } + // Return processing errors through the callback so the sync gets canceled. The + // number of writers is decremented prior to the call so PendingNodeData will + // return zero when the callback runs. + q.lock.Lock() + q.stateWriters -= 1 + q.lock.Unlock() + callback(nproc, progressed, procerr) + // Wake up WaitResults after the state has been written because it might be + // waiting for completion of the pivot block's state download. + q.active.Signal() + }() + // If none of the data items were good, it's a stale delivery switch { case len(errs) == 0: - return accepted, nil + return len(process), nil case len(errs) == len(request.Hashes): - return accepted, errStaleDelivery + return len(process), errStaleDelivery default: - return accepted, fmt.Errorf("multiple failures: %v", errs) + return len(process), fmt.Errorf("multiple failures: %v", errs) } } -// deliverNodeData is the asynchronous node data processor that injects a batch -// of sync results into the state scheduler. -func (q *queue) deliverNodeData(results []trie.SyncResult, callback func(int, bool, error)) { - // Wake up WaitResults after the state has been written because it - // might be waiting for the pivot block state to get completed. - defer q.active.Signal() - - // Process results one by one to permit task fetches in between - progressed := false - for i, result := range results { - q.stateSchedLock.Lock() - - if q.stateScheduler == nil { - // Syncing aborted since this async delivery started, bail out - q.stateSchedLock.Unlock() - callback(i, progressed, errNoFetchesPending) - return - } - - batch := q.stateDatabase.NewBatch() - prog, _, err := q.stateScheduler.Process([]trie.SyncResult{result}, batch) - if err != nil { - q.stateSchedLock.Unlock() - callback(i, progressed, err) - return - } - if err = batch.Write(); err != nil { - q.stateSchedLock.Unlock() - callback(i, progressed, err) - return // TODO(karalabe): If a DB write fails (disk full), we ought to cancel the sync - } - // Item processing succeeded, release the lock (temporarily) - progressed = progressed || prog - q.stateSchedLock.Unlock() - } - callback(len(results), progressed, nil) -} - // Prepare configures the result cache to allow accepting and caching inbound // fetch results. func (q *queue) Prepare(offset uint64, mode SyncMode, pivot uint64, head *types.Header) { From 1886d03faa9b7d8cdf335da84c297d30c213bb69 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 25 Jan 2017 16:29:40 +0100 Subject: [PATCH 21/21] console, internal/web3ext: remove bzz and ens extensions (#3602) web3.js includes bzz methods and throws an error when the extension module is reregistered. The ENS RPC API is deprecated and not exposed by anything. --- console/console.go | 4 ++ internal/web3ext/web3ext.go | 97 ------------------------------------- 2 files changed, 4 insertions(+), 97 deletions(-) diff --git a/console/console.go b/console/console.go index 9bb3df9265..389d528580 100644 --- a/console/console.go +++ b/console/console.go @@ -137,10 +137,14 @@ func (c *Console) init(preload []string) error { continue // manually mapped or ignore } if file, ok := web3ext.Modules[api]; ok { + // Load our extension for the module. if err = c.jsre.Compile(fmt.Sprintf("%s.js", api), file); err != nil { return fmt.Errorf("%s.js: %v", api, err) } flatten += fmt.Sprintf("var %s = web3.%s; ", api, api) + } else if obj, err := c.jsre.Run("web3." + api); err == nil && obj.IsObject() { + // Enable web3.js built-in extension if available. + flatten += fmt.Sprintf("var %s = web3.%s; ", api, api) } } if _, err = c.jsre.Run(flatten); err != nil { diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index e10e660450..edbe45fa34 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -19,10 +19,8 @@ package web3ext var Modules = map[string]string{ "admin": Admin_JS, - "bzz": Bzz_JS, "chequebook": Chequebook_JS, "debug": Debug_JS, - "ens": ENS_JS, "eth": Eth_JS, "miner": Miner_JS, "net": Net_JS, @@ -32,101 +30,6 @@ var Modules = map[string]string{ "txpool": TxPool_JS, } -const Bzz_JS = ` -web3._extend({ - property: 'bzz', - methods: - [ - new web3._extend.Method({ - name: 'syncEnabled', - call: 'bzz_syncEnabled', - params: 1, - inputFormatter: [null] - }), - new web3._extend.Method({ - name: 'swapEnabled', - call: 'bzz_swapEnabled', - params: 1, - inputFormatter: [null] - }), - new web3._extend.Method({ - name: 'download', - call: 'bzz_download', - params: 2, - inputFormatter: [null, null] - }), - new web3._extend.Method({ - name: 'upload', - call: 'bzz_upload', - params: 2, - inputFormatter: [null, null] - }), - new web3._extend.Method({ - name: 'resolve', - call: 'bzz_resolve', - params: 1, - inputFormatter: [null] - }), - new web3._extend.Method({ - name: 'get', - call: 'bzz_get', - params: 1, - inputFormatter: [null] - }), - new web3._extend.Method({ - name: 'put', - call: 'bzz_put', - params: 2, - inputFormatter: [null, null] - }), - new web3._extend.Method({ - name: 'modify', - call: 'bzz_modify', - params: 4, - inputFormatter: [null, null, null, null] - }) - ], - properties: - [ - new web3._extend.Property({ - name: 'hive', - getter: 'bzz_hive' - }), - new web3._extend.Property({ - name: 'info', - getter: 'bzz_info', - }), - ] -}); -` - -const ENS_JS = ` -web3._extend({ - property: 'ens', - methods: - [ - new web3._extend.Method({ - name: 'register', - call: 'ens_register', - params: 1, - inputFormatter: [null] - }), - new web3._extend.Method({ - name: 'setContentHash', - call: 'ens_setContentHash', - params: 2, - inputFormatter: [null, null] - }), - new web3._extend.Method({ - name: 'resolve', - call: 'ens_resolve', - params: 1, - inputFormatter: [null] - }), - ] -}) -` - const Chequebook_JS = ` web3._extend({ property: 'chequebook',