From 5bd72779d5a5cacbae7af37faf8efb9f312dcf34 Mon Sep 17 00:00:00 2001 From: atvanguard <93arpit@gmail.com> Date: Tue, 5 May 2020 09:45:57 +0530 Subject: [PATCH 1/4] new: Add GetRootHash bor api method --- consensus/bor/api.go | 44 +++++++++++++++++++++++++++++ consensus/bor/bor.go | 2 +- consensus/bor/bor_test/helper.go | 4 +-- consensus/bor/merkle.go | 48 ++++++++++++++++++++++++++++++++ go.mod | 1 + go.sum | 2 ++ internal/web3ext/web3ext.go | 5 ++++ 7 files changed, 103 insertions(+), 3 deletions(-) create mode 100644 consensus/bor/merkle.go diff --git a/consensus/bor/api.go b/consensus/bor/api.go index 98f7c46b6e..7ef9cdcd47 100644 --- a/consensus/bor/api.go +++ b/consensus/bor/api.go @@ -17,10 +17,16 @@ package bor import ( + "math/big" + "sync" + "github.com/maticnetwork/bor/common" "github.com/maticnetwork/bor/consensus" "github.com/maticnetwork/bor/core/types" + "github.com/maticnetwork/bor/crypto" "github.com/maticnetwork/bor/rpc" + "github.com/xsleonard/go-merkle" + "golang.org/x/crypto/sha3" ) // API is a user facing RPC API to allow controlling the signer and voting @@ -105,3 +111,41 @@ func (api *API) GetCurrentValidators() ([]*Validator, error) { } return snap.ValidatorSet.Validators, nil } + +// GetRootHash gets the current validators +func (api *API) GetRootHash(start uint64, end uint64) ([]byte, error) { + blockHeaders := make([]*types.Header, end-start+1) + wg := new(sync.WaitGroup) + // do we want to limit the # of concurrent go routines? + for i := start; i <= end; i++ { + wg.Add(1) + go func(number uint64) { + blockHeaders[number-start] = api.chain.GetHeaderByNumber(number) + wg.Done() + }(i) + } + wg.Wait() + + expectedLength := nextPowerOfTwo(end - start + 1) + headers := make([][32]byte, expectedLength) + for i := 0; i < len(blockHeaders); i++ { + blockHeader := blockHeaders[i] + header := crypto.Keccak256(appendBytes32( + blockHeader.Number.Bytes(), + new(big.Int).SetUint64(blockHeader.Time).Bytes(), + blockHeader.TxHash.Bytes(), + blockHeader.ReceiptHash.Bytes(), + )) + + var arr [32]byte + copy(arr[:], header) + headers[i] = arr + } + + tree := merkle.NewTreeWithOpts(merkle.TreeOptions{EnableHashSorting: false, DisableHashLeaves: true}) + if err := tree.Generate(convert(headers), sha3.NewLegacyKeccak256()); err != nil { + return nil, err + } + + return tree.Root().Hash, nil +} diff --git a/consensus/bor/bor.go b/consensus/bor/bor.go index 84fabf3336..945395dd6b 100644 --- a/consensus/bor/bor.go +++ b/consensus/bor/bor.go @@ -441,7 +441,7 @@ func (c *Bor) verifyCascadingFields(chain consensus.ChainReader, header *types.H copy(validatorsBytes[i*validatorHeaderBytesLength:], validator.HeaderBytes()) } // len(header.Extra) >= extraVanity+extraSeal has already been validated in validateHeaderExtraField, so this won't result in a panic - if !bytes.Equal(header.Extra[extraVanity : len(header.Extra)-extraSeal], validatorsBytes) { + if !bytes.Equal(header.Extra[extraVanity:len(header.Extra)-extraSeal], validatorsBytes) { return errMismatchingSprintValidators } } diff --git a/consensus/bor/bor_test/helper.go b/consensus/bor/bor_test/helper.go index 1ead54ed88..48735a7c5f 100644 --- a/consensus/bor/bor_test/helper.go +++ b/consensus/bor/bor_test/helper.go @@ -107,13 +107,13 @@ func buildMinimalNextHeader(t *testing.T, block *types.Block, borConfig *params. header.Time += bor.CalcProducerDelay(header.Number.Uint64(), borConfig.Period, borConfig.Sprint, borConfig.ProducerDelay) isSprintEnd := (header.Number.Uint64()+1)%borConfig.Sprint == 0 if isSprintEnd { - header.Extra = make([]byte, 32 + 40 + 65) // vanity + validatorBytes + extraSeal + header.Extra = make([]byte, 32+40+65) // vanity + validatorBytes + extraSeal // the genesis file was initialized with a validator 0x71562b71999873db5b286df957af199ec94617f7 with power 10 // So, if you change ./genesis.json, do change the following as well validatorBytes, _ := hex.DecodeString("71562b71999873db5b286df957af199ec94617f7000000000000000000000000000000000000000a") copy(header.Extra[32:72], validatorBytes) } else { - header.Extra = make([]byte, 32 + 65) // vanity + extraSeal + header.Extra = make([]byte, 32+65) // vanity + extraSeal } _key, _ := hex.DecodeString(privKey) sig, err := secp256k1.Sign(crypto.Keccak256(bor.BorRLP(header)), _key) diff --git a/consensus/bor/merkle.go b/consensus/bor/merkle.go new file mode 100644 index 0000000000..bdfbaba983 --- /dev/null +++ b/consensus/bor/merkle.go @@ -0,0 +1,48 @@ +package bor + +func appendBytes32(data ...[]byte) []byte { + var result []byte + for _, v := range data { + paddedV, err := convertTo32(v) + if err == nil { + result = append(result, paddedV[:]...) + } + } + return result +} + +func nextPowerOfTwo(n uint64) uint64 { + if n == 0 { + return 1 + } + // http://graphics.stanford.edu/~seander/bithacks.html#RoundUpPowerOf2 + n-- + n |= n >> 1 + n |= n >> 2 + n |= n >> 4 + n |= n >> 8 + n |= n >> 16 + n |= n >> 32 + n++ + return n +} + +func convertTo32(input []byte) (output [32]byte, err error) { + l := len(input) + if l > 32 || l == 0 { + return + } + copy(output[32-l:], input[:]) + return +} + +func convert(input []([32]byte)) [][]byte { + var output [][]byte + for _, in := range input { + newInput := make([]byte, len(in[:])) + copy(newInput, in[:]) + output = append(output, newInput) + + } + return output +} diff --git a/go.mod b/go.mod index ca433a71c6..efce11a0bf 100644 --- a/go.mod +++ b/go.mod @@ -60,6 +60,7 @@ require ( github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208 + github.com/xsleonard/go-merkle v1.1.0 golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4 golang.org/x/net v0.0.0-20200301022130-244492dfa37a golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e diff --git a/go.sum b/go.sum index c51ed2d8e9..c15b30bb97 100644 --- a/go.sum +++ b/go.sum @@ -176,6 +176,8 @@ github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef h1:wHSqTBrZ github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208 h1:1cngl9mPEoITZG8s8cVcUy5CeIBYhEESkOB7m6Gmkrk= github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208/go.mod h1:IotVbo4F+mw0EzQ08zFqg7pK3FebNXpaMsRy2RT+Ees= +github.com/xsleonard/go-merkle v1.1.0 h1:fHe1fuhJjGH22ZzVTAH0jqHLhTGhOq3wQjJN+8P0jQg= +github.com/xsleonard/go-merkle v1.1.0/go.mod h1:cW4z+UZ/4f2n9IJgIiyDCdYguchoDyDAPmpuOWGxdGg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4 h1:QmwruyY+bKbDDL0BaglrbZABEali68eoMFhTZpCjYVA= diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index d3fea7b50c..9ee4b4f3a7 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -149,6 +149,11 @@ web3._extend({ call: 'bor_getCurrentValidators', params: 0 }), + new web3._extend.Method({ + name: 'getRootHash', + call: 'bor_getRootHash', + params: 2, + }), ] }); ` From bbe6522f3575e7f3df65ccdd2cd03110e008920f Mon Sep 17 00:00:00 2001 From: atvanguard <93arpit@gmail.com> Date: Tue, 5 May 2020 14:45:56 +0530 Subject: [PATCH 2/4] new: Add GetRootHash to api backend --- consensus/bor/api.go | 65 +++++++++++++++++++++++++++++++------- consensus/bor/errors.go | 28 ++++++++++++++++ eth/api_backend.go | 14 ++++++++ internal/ethapi/api.go | 8 +++++ internal/ethapi/backend.go | 1 + les/api_backend.go | 4 +++ 6 files changed, 109 insertions(+), 11 deletions(-) diff --git a/consensus/bor/api.go b/consensus/bor/api.go index 7ef9cdcd47..ca64cd5521 100644 --- a/consensus/bor/api.go +++ b/consensus/bor/api.go @@ -17,9 +17,12 @@ package bor import ( + "math" "math/big" + "strconv" "sync" + lru "github.com/hashicorp/golang-lru" "github.com/maticnetwork/bor/common" "github.com/maticnetwork/bor/consensus" "github.com/maticnetwork/bor/core/types" @@ -29,11 +32,17 @@ import ( "golang.org/x/crypto/sha3" ) +var ( + // MaxCheckpointLength is the maximum number of blocks that can be requested for constructing a checkpoint root hash + MaxCheckpointLength = uint64(math.Pow(2, 15)) +) + // API is a user facing RPC API to allow controlling the signer and voting // mechanisms of the proof-of-authority scheme. type API struct { - chain consensus.ChainReader - bor *Bor + chain consensus.ChainReader + bor *Bor + rootHashCache *lru.ARCCache } // GetSnapshot retrieves the state snapshot at a given block. @@ -112,22 +121,38 @@ func (api *API) GetCurrentValidators() ([]*Validator, error) { return snap.ValidatorSet.Validators, nil } -// GetRootHash gets the current validators -func (api *API) GetRootHash(start uint64, end uint64) ([]byte, error) { +// GetRootHash returns the merkle root of the start to end block headers +func (api *API) GetRootHash(start int64, end int64) ([]byte, error) { + key := getKey(start, end) + if root, err := api.lookupCache(key); err != nil { + return nil, err + } else if root != nil { + return root, nil + } + length := uint64(end - start + 1) + if length > MaxCheckpointLength { + return nil, &MaxCheckpointLengthExceededError{start, end} + } + currentHeaderNumber := api.chain.CurrentHeader().Number.Int64() + if start > end || end > currentHeaderNumber { + return nil, &InvalidStartEndBlockError{start, end, currentHeaderNumber} + } blockHeaders := make([]*types.Header, end-start+1) wg := new(sync.WaitGroup) - // do we want to limit the # of concurrent go routines? + concurrent := make(chan bool, 20) for i := start; i <= end; i++ { wg.Add(1) - go func(number uint64) { - blockHeaders[number-start] = api.chain.GetHeaderByNumber(number) + concurrent <- true + go func(number int64) { + blockHeaders[number-start] = api.chain.GetHeaderByNumber(uint64(number)) + <-concurrent wg.Done() }(i) } wg.Wait() + close(concurrent) - expectedLength := nextPowerOfTwo(end - start + 1) - headers := make([][32]byte, expectedLength) + headers := make([][32]byte, nextPowerOfTwo(length)) for i := 0; i < len(blockHeaders); i++ { blockHeader := blockHeaders[i] header := crypto.Keccak256(appendBytes32( @@ -146,6 +171,24 @@ func (api *API) GetRootHash(start uint64, end uint64) ([]byte, error) { if err := tree.Generate(convert(headers), sha3.NewLegacyKeccak256()); err != nil { return nil, err } - - return tree.Root().Hash, nil + root := tree.Root().Hash + api.rootHashCache.Add(key, root) + return root, nil +} + +func (api *API) lookupCache(key string) ([]byte, error) { + var err error + if api.rootHashCache == nil { + if api.rootHashCache, err = lru.NewARC(10); err != nil { + return nil, err + } + } + if root, known := api.rootHashCache.Get(key); known { + return root.([]byte), nil + } + return nil, nil +} + +func getKey(start int64, end int64) string { + return strconv.FormatInt(start, 10) + "-" + strconv.FormatInt(end, 10) } diff --git a/consensus/bor/errors.go b/consensus/bor/errors.go index a3b0dc656d..ae7da982a7 100644 --- a/consensus/bor/errors.go +++ b/consensus/bor/errors.go @@ -40,3 +40,31 @@ func (e *TotalVotingPowerExceededError) Error() string { e.Validators, ) } + +type InvalidStartEndBlockError struct { + Start int64 + End int64 + CurrentHeader int64 +} + +func (e *InvalidStartEndBlockError) Error() string { + return fmt.Sprintf( + "Invalid parameters start: %d and end block: %d params", + e.Start, + e.End, + ) +} + +type MaxCheckpointLengthExceededError struct { + Start int64 + End int64 +} + +func (e *MaxCheckpointLengthExceededError) Error() string { + return fmt.Sprintf( + "Start: %d and end block: %d exceed max allowed checkpoint length: %d", + e.Start, + e.End, + MaxCheckpointLength, + ) +} diff --git a/eth/api_backend.go b/eth/api_backend.go index ff03d1a91e..7958f86b37 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -248,3 +248,17 @@ func (b *EthAPIBackend) ServiceFilter(ctx context.Context, session *bloombits.Ma go session.Multiplex(bloomRetrievalBatch, bloomRetrievalWait, b.eth.bloomRequests) } } + +func (b *EthAPIBackend) GetRootHash(ctx context.Context, starBlockNr rpc.BlockNumber, endBlockNr rpc.BlockNumber) ([]byte, error) { + var api *bor.API + for _, _api := range b.eth.Engine().APIs(b.eth.BlockChain()) { + if _api.Namespace == "bor" { + api = _api.Service.(*bor.API) + } + } + root, err := api.GetRootHash(starBlockNr.Int64(), endBlockNr.Int64()) + if err != nil { + return nil, err + } + return root, nil +} diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 2a939ed2c4..0ad624e7ed 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -716,6 +716,14 @@ func (s *PublicBlockChainAPI) GetStorageAt(ctx context.Context, address common.A return res[:], state.Error() } +func (s *PublicBlockChainAPI) GetRootHash(ctx context.Context, starBlockNr rpc.BlockNumber, endBlockNr rpc.BlockNumber) ([]byte, error) { + root, err := s.b.GetRootHash(ctx, starBlockNr, endBlockNr) + if err != nil { + return nil, err + } + return root, nil +} + // CallArgs represents the arguments for a call. type CallArgs struct { From *common.Address `json:"from"` diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index e0deeab81f..21cef13f44 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -62,6 +62,7 @@ type Backend interface { SubscribeStateEvent(ch chan<- core.NewStateChangeEvent) event.Subscription SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription SubscribeChainSideEvent(ch chan<- core.ChainSideEvent) event.Subscription + GetRootHash(ctx context.Context, starBlockNr rpc.BlockNumber, endBlockNr rpc.BlockNumber) ([]byte, error) // Transaction pool API SendTx(ctx context.Context, signedTx *types.Transaction) error diff --git a/les/api_backend.go b/les/api_backend.go index 5c8ec82d88..4739abcdeb 100644 --- a/les/api_backend.go +++ b/les/api_backend.go @@ -223,3 +223,7 @@ func (b *LesApiBackend) ServiceFilter(ctx context.Context, session *bloombits.Ma go session.Multiplex(bloomRetrievalBatch, bloomRetrievalWait, b.eth.bloomRequests) } } + +func (b *LesApiBackend) GetRootHash(ctx context.Context, starBlockNr rpc.BlockNumber, endBlockNr rpc.BlockNumber) ([]byte, error) { + return nil, errors.New("Not implemented") +} From 54bbdff1e4a26ae65760cbddfa9cd886e8e03613 Mon Sep 17 00:00:00 2001 From: atvanguard <93arpit@gmail.com> Date: Wed, 6 May 2020 16:35:12 +0530 Subject: [PATCH 3/4] Initialize apiCache in once.do --- consensus/bor/api.go | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/consensus/bor/api.go b/consensus/bor/api.go index ca64cd5521..d5de0a61cb 100644 --- a/consensus/bor/api.go +++ b/consensus/bor/api.go @@ -35,6 +35,7 @@ import ( var ( // MaxCheckpointLength is the maximum number of blocks that can be requested for constructing a checkpoint root hash MaxCheckpointLength = uint64(math.Pow(2, 15)) + once sync.Once ) // API is a user facing RPC API to allow controlling the signer and voting @@ -123,11 +124,18 @@ func (api *API) GetCurrentValidators() ([]*Validator, error) { // GetRootHash returns the merkle root of the start to end block headers func (api *API) GetRootHash(start int64, end int64) ([]byte, error) { - key := getKey(start, end) - if root, err := api.lookupCache(key); err != nil { + var err error + once.Do(func() { + if api.rootHashCache == nil { + api.rootHashCache, err = lru.NewARC(10) + } + }) + if err != nil { return nil, err - } else if root != nil { - return root, nil + } + key := getRootHashKey(start, end) + if root, known := api.rootHashCache.Get(key); known { + return root.([]byte), nil } length := uint64(end - start + 1) if length > MaxCheckpointLength { @@ -176,19 +184,6 @@ func (api *API) GetRootHash(start int64, end int64) ([]byte, error) { return root, nil } -func (api *API) lookupCache(key string) ([]byte, error) { - var err error - if api.rootHashCache == nil { - if api.rootHashCache, err = lru.NewARC(10); err != nil { - return nil, err - } - } - if root, known := api.rootHashCache.Get(key); known { - return root.([]byte), nil - } - return nil, nil -} - -func getKey(start int64, end int64) string { +func getRootHashKey(start int64, end int64) string { return strconv.FormatInt(start, 10) + "-" + strconv.FormatInt(end, 10) } From 50c0b662cfeeee8b8f14ff8131429bfd1e8eaf16 Mon Sep 17 00:00:00 2001 From: atvanguard <93arpit@gmail.com> Date: Wed, 6 May 2020 17:14:53 +0530 Subject: [PATCH 4/4] Remove once.do --- consensus/bor/api.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/consensus/bor/api.go b/consensus/bor/api.go index d5de0a61cb..672d378215 100644 --- a/consensus/bor/api.go +++ b/consensus/bor/api.go @@ -35,7 +35,6 @@ import ( var ( // MaxCheckpointLength is the maximum number of blocks that can be requested for constructing a checkpoint root hash MaxCheckpointLength = uint64(math.Pow(2, 15)) - once sync.Once ) // API is a user facing RPC API to allow controlling the signer and voting @@ -124,13 +123,7 @@ func (api *API) GetCurrentValidators() ([]*Validator, error) { // GetRootHash returns the merkle root of the start to end block headers func (api *API) GetRootHash(start int64, end int64) ([]byte, error) { - var err error - once.Do(func() { - if api.rootHashCache == nil { - api.rootHashCache, err = lru.NewARC(10) - } - }) - if err != nil { + if err := api.initializeRootHashCache(); err != nil { return nil, err } key := getRootHashKey(start, end) @@ -184,6 +177,14 @@ func (api *API) GetRootHash(start int64, end int64) ([]byte, error) { return root, nil } +func (api *API) initializeRootHashCache() error { + var err error + if api.rootHashCache == nil { + api.rootHashCache, err = lru.NewARC(10) + } + return err +} + func getRootHashKey(start int64, end int64) string { return strconv.FormatInt(start, 10) + "-" + strconv.FormatInt(end, 10) }