diff --git a/accounts/hd.go b/accounts/hd.go index 277f688e48..6ed6318078 100644 --- a/accounts/hd.go +++ b/accounts/hd.go @@ -30,8 +30,8 @@ import ( var DefaultRootDerivationPath = DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0} // DefaultBaseDerivationPath is the base path from which custom derivation endpoints -// are incremented. As such, the first account will be at m/44'/60'/0'/0, the second -// at m/44'/60'/0'/1, etc. +// are incremented. As such, the first account will be at m/44'/60'/0'/0/0, the second +// at m/44'/60'/0'/0/1, etc. var DefaultBaseDerivationPath = DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0, 0} // DefaultLedgerBaseDerivationPath is the base path from which custom derivation endpoints diff --git a/cmd/bootnode/main.go b/cmd/bootnode/main.go index 346523ddb6..32f7d63bea 100644 --- a/cmd/bootnode/main.go +++ b/cmd/bootnode/main.go @@ -38,7 +38,7 @@ func main() { var ( listenAddr = flag.String("addr", ":30301", "listen address") genKey = flag.String("genkey", "", "generate a node key") - writeAddr = flag.Bool("writeaddress", false, "write out the node's pubkey hash and quit") + writeAddr = flag.Bool("writeaddress", false, "write out the node's public key and quit") nodeKeyFile = flag.String("nodekey", "", "private key filename") nodeKeyHex = flag.String("nodekeyhex", "", "private key as hex (for testing)") natdesc = flag.String("nat", "none", "port mapping mechanism (any|none|upnp|pmp|extip:)") @@ -86,7 +86,7 @@ func main() { } if *writeAddr { - fmt.Printf("%v\n", enode.PubkeyToIDV4(&nodeKey.PublicKey)) + fmt.Printf("%x\n", crypto.FromECDSAPub(&nodeKey.PublicKey)[1:]) os.Exit(0) } diff --git a/common/bytes.go b/common/bytes.go index 0c257a1ee0..c82e616241 100644 --- a/common/bytes.go +++ b/common/bytes.go @@ -31,6 +31,15 @@ func ToHex(b []byte) string { return "0x" + hex } +// ToHexArray creates a array of hex-string based on []byte +func ToHexArray(b [][]byte) []string { + r := make([]string, len(b)) + for i := range b { + r[i] = ToHex(b[i]) + } + return r +} + // FromHex returns the bytes represented by the hexadecimal string s. // s may be prefixed with "0x". func FromHex(s string) []byte { diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index da54328326..6660e17de1 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -278,7 +278,7 @@ func ReadReceipts(db DatabaseReader, hash common.Hash, number uint64) types.Rece if len(data) == 0 { return nil } - // Convert the revceipts from their storage form to their internal representation + // Convert the receipts from their storage form to their internal representation storageReceipts := []*types.ReceiptForStorage{} if err := rlp.DecodeBytes(data, &storageReceipts); err != nil { log.Error("Invalid receipt array RLP", "hash", hash, "err", err) diff --git a/core/state/statedb.go b/core/state/statedb.go index 216667ce98..b1bb53ed04 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -18,6 +18,7 @@ package state import ( + "errors" "fmt" "math/big" "sort" @@ -25,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" @@ -256,6 +258,24 @@ func (self *StateDB) GetState(addr common.Address, hash common.Hash) common.Hash return common.Hash{} } +// GetProof returns the MerkleProof for a given Account +func (self *StateDB) GetProof(a common.Address) (vm.ProofList, error) { + var proof vm.ProofList + err := self.trie.Prove(crypto.Keccak256(a.Bytes()), 0, &proof) + return proof, err +} + +// GetProof returns the StorageProof for given key +func (self *StateDB) GetStorageProof(a common.Address, key common.Hash) (vm.ProofList, error) { + var proof vm.ProofList + trie := self.StorageTrie(a) + if trie == nil { + return proof, errors.New("storage trie for requested address does not exist") + } + err := trie.Prove(crypto.Keccak256(key.Bytes()), 0, &proof) + return proof, err +} + // GetCommittedState retrieves a value from the given account's committed storage trie. func (self *StateDB) GetCommittedState(addr common.Address, hash common.Hash) common.Hash { stateObject := self.getStateObject(addr) diff --git a/core/vm/interface.go b/core/vm/interface.go index fc15082f18..1ae98cf7a2 100644 --- a/core/vm/interface.go +++ b/core/vm/interface.go @@ -35,6 +35,8 @@ type StateDB interface { SetNonce(common.Address, uint64) GetCodeHash(common.Address) common.Hash + GetProof(common.Address) (ProofList, error) + GetStorageProof(common.Address, common.Hash) (ProofList, error) GetCode(common.Address) []byte SetCode(common.Address, []byte) GetCodeSize(common.Address) int @@ -78,3 +80,11 @@ type CallContext interface { // Create a new contract Create(env *EVM, me ContractRef, data []byte, gas, value *big.Int) ([]byte, common.Address, error) } + +// MerkleProof +type ProofList [][]byte + +func (n *ProofList) Put(key []byte, value []byte) error { + *n = append(*n, value) + return nil +} diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 196be43e03..0c751c3282 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -508,6 +508,72 @@ func (s *PublicBlockChainAPI) GetBalance(ctx context.Context, address common.Add return (*hexutil.Big)(state.GetBalance(address)), state.Error() } +// Result structs for GetProof +type AccountResult struct { + Address common.Address `json:"address"` + AccountProof []string `json:"accountProof"` + Balance *hexutil.Big `json:"balance"` + CodeHash common.Hash `json:"codeHash"` + Nonce hexutil.Uint64 `json:"nonce"` + StorageHash common.Hash `json:"storageHash"` + StorageProof []StorageResult `json:"storageProof"` +} +type StorageResult struct { + Key string `json:"key"` + Value *hexutil.Big `json:"value"` + Proof []string `json:"proof"` +} + +// GetProof returns the Merkle-proof for a given account and optionally some storage keys. +func (s *PublicBlockChainAPI) GetProof(ctx context.Context, address common.Address, storageKeys []string, blockNr rpc.BlockNumber) (*AccountResult, error) { + state, _, err := s.b.StateAndHeaderByNumber(ctx, blockNr) + if state == nil || err != nil { + return nil, err + } + + storageTrie := state.StorageTrie(address) + storageHash := types.EmptyRootHash + codeHash := state.GetCodeHash(address) + storageProof := make([]StorageResult, len(storageKeys)) + + // if we have a storageTrie, (which means the account exists), we can update the storagehash + if storageTrie != nil { + storageHash = storageTrie.Hash() + } else { + // no storageTrie means the account does not exist, so the codeHash is the hash of an empty bytearray. + codeHash = crypto.Keccak256Hash(nil) + } + + // create the proof for the storageKeys + for i, key := range storageKeys { + if storageTrie != nil { + proof, storageError := state.GetStorageProof(address, common.HexToHash(key)) + if storageError != nil { + return nil, storageError + } + storageProof[i] = StorageResult{key, (*hexutil.Big)(state.GetState(address, common.HexToHash(key)).Big()), common.ToHexArray(proof)} + } else { + storageProof[i] = StorageResult{key, &hexutil.Big{}, []string{}} + } + } + + // create the accountProof + accountProof, proofErr := state.GetProof(address) + if proofErr != nil { + return nil, proofErr + } + + return &AccountResult{ + Address: address, + AccountProof: common.ToHexArray(accountProof), + Balance: (*hexutil.Big)(state.GetBalance(address)), + CodeHash: codeHash, + Nonce: hexutil.Uint64(state.GetNonce(address)), + StorageHash: storageHash, + StorageProof: storageProof, + }, state.Error() +} + // GetBlockByNumber returns the requested block. When blockNr is -1 the chain head is returned. When fullTx is true all // transactions in the block are returned in full detail, otherwise only the transaction hash is returned. func (s *PublicBlockChainAPI) GetBlockByNumber(ctx context.Context, blockNr rpc.BlockNumber, fullTx bool) (map[string]interface{}, error) { diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 370aca5a75..b4294b0581 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -409,7 +409,6 @@ func (s *Server) handleMultipartUpload(r *http.Request, boundary string, mw *api Path: path, ContentType: part.Header.Get("Content-Type"), Size: size, - ModTime: time.Now(), } log.Debug("adding path to new manifest", "ruid", ruid, "bytes", entry.Size, "path", entry.Path) contentKey, err := mw.AddEntry(r.Context(), reader, entry) @@ -428,7 +427,6 @@ func (s *Server) handleDirectUpload(r *http.Request, mw *api.ManifestWriter) err ContentType: r.Header.Get("Content-Type"), Mode: 0644, Size: r.ContentLength, - ModTime: time.Now(), }) if err != nil { return err diff --git a/swarm/network/stream/delivery.go b/swarm/network/stream/delivery.go index 0429c4dffc..9092ffe3ed 100644 --- a/swarm/network/stream/delivery.go +++ b/swarm/network/stream/delivery.go @@ -173,7 +173,8 @@ func (d *Delivery) handleRetrieveRequestMsg(ctx context.Context, sp *Peer, req * return } if req.SkipCheck { - err = sp.Deliver(ctx, chunk, s.priority) + syncing := false + err = sp.Deliver(ctx, chunk, s.priority, syncing) if err != nil { log.Warn("ERROR in handleRetrieveRequestMsg", "err", err) } @@ -189,12 +190,22 @@ func (d *Delivery) handleRetrieveRequestMsg(ctx context.Context, sp *Peer, req * return nil } +//Chunk delivery always uses the same message type.... type ChunkDeliveryMsg struct { Addr storage.Address SData []byte // the stored chunk Data (incl size) peer *Peer // set in handleChunkDeliveryMsg } +//...but swap accounting needs to disambiguate if it is a delivery for syncing or for retrieval +//as it decides based on message type if it needs to account for this message or not + +//defines a chunk delivery for retrieval (with accounting) +type ChunkDeliveryMsgRetrieval ChunkDeliveryMsg + +//defines a chunk delivery for syncing (without accounting) +type ChunkDeliveryMsgSyncing ChunkDeliveryMsg + // TODO: Fix context SNAFU func (d *Delivery) handleChunkDeliveryMsg(ctx context.Context, sp *Peer, req *ChunkDeliveryMsg) error { var osp opentracing.Span diff --git a/swarm/network/stream/delivery_test.go b/swarm/network/stream/delivery_test.go index ec6148a0bf..949645558b 100644 --- a/swarm/network/stream/delivery_test.go +++ b/swarm/network/stream/delivery_test.go @@ -416,7 +416,7 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck return fmt.Errorf("No registry") } registry := item.(*Registry) - err = registry.Subscribe(sid, NewStream(swarmChunkServerStreamName, "", true), NewRange(0, 0), Top) + err = registry.Subscribe(sid, NewStream(swarmChunkServerStreamName, "", true), nil, Top) if err != nil { return err } diff --git a/swarm/network/stream/messages.go b/swarm/network/stream/messages.go index 68503fe1f0..eb1b2983e1 100644 --- a/swarm/network/stream/messages.go +++ b/swarm/network/stream/messages.go @@ -158,6 +158,7 @@ type SubscribeErrorMsg struct { } func (p *Peer) handleSubscribeErrorMsg(req *SubscribeErrorMsg) (err error) { + //TODO the error should be channeled to whoever calls the subscribe return fmt.Errorf("subscribe to peer %s: %v", p.ID(), req.Error) } @@ -356,7 +357,8 @@ func (p *Peer) handleWantedHashesMsg(ctx context.Context, req *WantedHashesMsg) return fmt.Errorf("handleWantedHashesMsg get data %x: %v", hash, err) } chunk := storage.NewChunk(hash, data) - if err := p.Deliver(ctx, chunk, s.priority); err != nil { + syncing := true + if err := p.Deliver(ctx, chunk, s.priority, syncing); err != nil { return err } } diff --git a/swarm/network/stream/peer.go b/swarm/network/stream/peer.go index 89d135ad52..4bccf56f5d 100644 --- a/swarm/network/stream/peer.go +++ b/swarm/network/stream/peer.go @@ -128,17 +128,34 @@ func NewPeer(peer *protocols.Peer, streamer *Registry) *Peer { } // Deliver sends a storeRequestMsg protocol message to the peer -func (p *Peer) Deliver(ctx context.Context, chunk storage.Chunk, priority uint8) error { +// Depending on the `syncing` parameter we send different message types +func (p *Peer) Deliver(ctx context.Context, chunk storage.Chunk, priority uint8, syncing bool) error { var sp opentracing.Span + var msg interface{} + + spanName := "send.chunk.delivery" + + //we send different types of messages if delivery is for syncing or retrievals, + //even if handling and content of the message are the same, + //because swap accounting decides which messages need accounting based on the message type + if syncing { + msg = &ChunkDeliveryMsgSyncing{ + Addr: chunk.Address(), + SData: chunk.Data(), + } + spanName += ".syncing" + } else { + msg = &ChunkDeliveryMsgRetrieval{ + Addr: chunk.Address(), + SData: chunk.Data(), + } + spanName += ".retrieval" + } ctx, sp = spancontext.StartSpan( ctx, - "send.chunk.delivery") + spanName) defer sp.Finish() - msg := &ChunkDeliveryMsg{ - Addr: chunk.Address(), - SData: chunk.Data(), - } return p.SendPriority(ctx, msg, priority) } diff --git a/swarm/network/stream/snapshot_retrieval_test.go b/swarm/network/stream/snapshot_retrieval_test.go index 7625e02d7a..b81cfc0ca0 100644 --- a/swarm/network/stream/snapshot_retrieval_test.go +++ b/swarm/network/stream/snapshot_retrieval_test.go @@ -218,7 +218,7 @@ func runFileRetrievalTest(nodeCount int) error { reader, _ := fileStore.Retrieve(context.TODO(), hash) //check that we can read the file size and that it corresponds to the generated file size if s, err := reader.Size(ctx, nil); err != nil || s != int64(len(randomFiles[i])) { - log.Warn("Retrieve error", "err", err, "hash", hash, "nodeId", id) + log.Debug("Retrieve error", "err", err, "hash", hash, "nodeId", id) time.Sleep(500 * time.Millisecond) continue REPEAT } @@ -309,7 +309,7 @@ func runRetrievalTest(chunkCount int, nodeCount int) error { reader, _ := fileStore.Retrieve(context.TODO(), hash) //check that we can read the chunk size and that it corresponds to the generated chunk size if s, err := reader.Size(ctx, nil); err != nil || s != int64(chunkSize) { - log.Warn("Retrieve error", "err", err, "hash", hash, "nodeId", id, "size", s) + log.Debug("Retrieve error", "err", err, "hash", hash, "nodeId", id, "size", s) time.Sleep(500 * time.Millisecond) continue REPEAT } diff --git a/swarm/network/stream/snapshot_sync_test.go b/swarm/network/stream/snapshot_sync_test.go index 96e92c5cf1..8d89f28cba 100644 --- a/swarm/network/stream/snapshot_sync_test.go +++ b/swarm/network/stream/snapshot_sync_test.go @@ -310,7 +310,7 @@ func runSim(conf *synctestConfig, ctx context.Context, sim *simulation.Simulatio _, err = lstore.Get(ctx, chunk) } if err != nil { - log.Warn(fmt.Sprintf("Chunk %s NOT found for id %s", chunk, id)) + log.Debug(fmt.Sprintf("Chunk %s NOT found for id %s", chunk, id)) // Do not get crazy with logging the warn message time.Sleep(500 * time.Millisecond) continue REPEAT @@ -514,7 +514,7 @@ func testSyncingViaDirectSubscribe(t *testing.T, chunkCount int, nodeCount int) _, err = lstore.Get(ctx, chunk) } if err != nil { - log.Warn(fmt.Sprintf("Chunk %s NOT found for id %s", chunk, id)) + log.Debug(fmt.Sprintf("Chunk %s NOT found for id %s", chunk, id)) // Do not get crazy with logging the warn message time.Sleep(500 * time.Millisecond) continue REPEAT diff --git a/swarm/network/stream/stream.go b/swarm/network/stream/stream.go index 1dc2a8cba8..0ac374def1 100644 --- a/swarm/network/stream/stream.go +++ b/swarm/network/stream/stream.go @@ -18,6 +18,7 @@ package stream import ( "context" + "errors" "fmt" "math" "sync" @@ -96,7 +97,10 @@ func NewRegistry(localID enode.ID, delivery *Delivery, syncChunkStore storage.Sy delivery.getPeer = streamer.getPeer if options.DoServeRetrieve { - streamer.RegisterServerFunc(swarmChunkServerStreamName, func(_ *Peer, _ string, _ bool) (Server, error) { + streamer.RegisterServerFunc(swarmChunkServerStreamName, func(_ *Peer, _ string, live bool) (Server, error) { + if !live { + return nil, errors.New("only live retrieval requests supported") + } return NewSwarmChunkServer(delivery.chunkStore), nil }) } @@ -279,7 +283,6 @@ func (r *Registry) Subscribe(peerId enode.ID, s Stream, h *Range, priority uint8 if err != nil { return err } - if s.Live && h != nil { if err := peer.setClientParams( getHistoryStream(s), @@ -486,8 +489,13 @@ func (p *Peer) HandleMsg(ctx context.Context, msg interface{}) error { case *WantedHashesMsg: return p.handleWantedHashesMsg(ctx, msg) - case *ChunkDeliveryMsg: - return p.streamer.delivery.handleChunkDeliveryMsg(ctx, p, msg) + case *ChunkDeliveryMsgRetrieval: + //handling chunk delivery is the same for retrieval and syncing, so let's cast the msg + return p.streamer.delivery.handleChunkDeliveryMsg(ctx, p, ((*ChunkDeliveryMsg)(msg))) + + case *ChunkDeliveryMsgSyncing: + //handling chunk delivery is the same for retrieval and syncing, so let's cast the msg + return p.streamer.delivery.handleChunkDeliveryMsg(ctx, p, ((*ChunkDeliveryMsg)(msg))) case *RetrieveRequestMsg: return p.streamer.delivery.handleRetrieveRequestMsg(ctx, p, msg) @@ -678,7 +686,7 @@ func (c *clientParams) clientCreated() { // Spec is the spec of the streamer protocol var Spec = &protocols.Spec{ Name: "stream", - Version: 7, + Version: 8, MaxMsgSize: 10 * 1024 * 1024, Messages: []interface{}{ UnsubscribeMsg{}, @@ -687,10 +695,11 @@ var Spec = &protocols.Spec{ TakeoverProofMsg{}, SubscribeMsg{}, RetrieveRequestMsg{}, - ChunkDeliveryMsg{}, + ChunkDeliveryMsgRetrieval{}, SubscribeErrorMsg{}, RequestSubscriptionMsg{}, QuitMsg{}, + ChunkDeliveryMsgSyncing{}, }, }