From 92c980f355aafff1ca2210b496eec758fb4aa862 Mon Sep 17 00:00:00 2001 From: jsvisa Date: Wed, 7 Jun 2023 14:23:09 +0800 Subject: [PATCH 01/70] eth/filters: wip Signed-off-by: jsvisa --- eth/filters/api.go | 78 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 64 insertions(+), 14 deletions(-) diff --git a/eth/filters/api.go b/eth/filters/api.go index c929810a12..0661c57adf 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -273,26 +273,76 @@ func (api *FilterAPI) Logs(ctx context.Context, crit FilterCriteria) (*rpc.Subsc matchedLogs = make(chan []*types.Log) ) - logsSub, err := api.events.SubscribeLogs(ethereum.FilterQuery(crit), matchedLogs) - if err != nil { - return nil, err + syncHistLogs := func(n int64) error { + for { + // Get latest head block num + head := api.sys.backend.CurrentHeader().Number.Int64() + if n >= head { + return nil + } + // Do historical sync from n to head + f := api.sys.NewRangeFilter(n, head, crit.Addresses, crit.Topics) + logChan, errChan := f.rangeLogsAsync(ctx) + select { + case <-notifier.Closed(): + return errors.New("connection dropped") + case log := <-logChan: + notifier.Notify(rpcSub.ID, &log) + case err := <-errChan: + if err != nil { + return err + } + } + + // TODO: Check if any logs from n to head have been reorged + // send the reorged logs + + // Head is stable, move n to current head + n = head + } } - go func() { - defer logsSub.Unsubscribe() - for { - select { - case logs := <-matchedLogs: - for _, log := range logs { - notifier.Notify(rpcSub.ID, &log) + syncLiveLogs := func() error { + logsSub, err := api.events.SubscribeLogs(ethereum.FilterQuery(crit), matchedLogs) + if err != nil { + return err + } + + go func() { + for { + select { + case logs := <-matchedLogs: + for _, log := range logs { + log := log + notifier.Notify(rpcSub.ID, &log) + } + case <-rpcSub.Err(): // client send an unsubscribe request + logsSub.Unsubscribe() + return + case <-notifier.Closed(): // connection dropped + logsSub.Unsubscribe() + return } - case <-rpcSub.Err(): // client send an unsubscribe request + } + }() + return nil + } + + if crit.FromBlock != nil { + n := crit.FromBlock.Int64() + go func() { + // do historical sync first + if err := syncHistLogs(n); err != nil { return } - } - }() + // subscribe from latest + syncLiveLogs() + }() + return rpcSub, nil + } - return rpcSub, nil + err := syncLiveLogs() + return rpcSub, err } // FilterCriteria represents a request to create a new filter. From dc6a5b1b3d6ecbf86ce5e6430fe70bcf3a54dfda Mon Sep 17 00:00:00 2001 From: jsvisa Date: Wed, 7 Jun 2023 15:13:11 +0800 Subject: [PATCH 02/70] eth/filters: do hist sync Signed-off-by: jsvisa --- eth/filters/api.go | 52 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 39 insertions(+), 13 deletions(-) diff --git a/eth/filters/api.go b/eth/filters/api.go index 0661c57adf..9dd084fc93 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -271,34 +271,60 @@ func (api *FilterAPI) Logs(ctx context.Context, crit FilterCriteria) (*rpc.Subsc var ( rpcSub = notifier.CreateSubscription() matchedLogs = make(chan []*types.Log) + rmLogsCh = make(chan []*types.Log) ) syncHistLogs := func(n int64) error { for { - // Get latest head block num + // get the latest block header head := api.sys.backend.CurrentHeader().Number.Int64() if n >= head { return nil } - // Do historical sync from n to head + + // do historical sync from n to head f := api.sys.NewRangeFilter(n, head, crit.Addresses, crit.Topics) logChan, errChan := f.rangeLogsAsync(ctx) - select { - case <-notifier.Closed(): - return errors.New("connection dropped") - case log := <-logChan: - notifier.Notify(rpcSub.ID, &log) - case err := <-errChan: - if err != nil { - return err + + // subscribe rmLogs + query := ethereum.FilterQuery{FromBlock: big.NewInt(n), ToBlock: big.NewInt(head), Addresses: crit.Addresses, Topics: crit.Topics} + rmLogsSub, err := api.events.SubscribeLogs(query, rmLogsCh) + if err != nil { + return err + } + + var rmLogs []*types.Log + FORLOOP: + for { + select { + case <-notifier.Closed(): + rmLogsSub.Unsubscribe() + return errors.New("connection dropped") + case log := <-logChan: + notifier.Notify(rpcSub.ID, &log) + case logs := <-rmLogsCh: + for _, log := range logs { + if log.Removed { + rmLogs = append(rmLogs, log) + } + } + case err := <-errChan: + if err != nil { + return err + } + // range filter is done, let's also stop the rmLogs subscribe + rmLogsSub.Unsubscribe() + break FORLOOP } } - // TODO: Check if any logs from n to head have been reorged // send the reorged logs + for _, log := range rmLogs { + notifier.Notify(rpcSub.ID, &log) + } - // Head is stable, move n to current head - n = head + // move forward to the next batch + n = head + 1 } } From 870bf640aa6d575f99f93b4ad95787df78b14b1d Mon Sep 17 00:00:00 2001 From: jsvisa Date: Thu, 8 Jun 2023 07:19:48 +0000 Subject: [PATCH 03/70] eth/filters: use a sole context Signed-off-by: jsvisa --- eth/filters/api.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/eth/filters/api.go b/eth/filters/api.go index 9dd084fc93..0eb977cadf 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -275,6 +275,9 @@ func (api *FilterAPI) Logs(ctx context.Context, crit FilterCriteria) (*rpc.Subsc ) syncHistLogs := func(n int64) error { + // the ctx will be canceled after this function is called + // and we are run in a background goroutine, use a new context instead + cctx := context.Background() for { // get the latest block header head := api.sys.backend.CurrentHeader().Number.Int64() @@ -284,7 +287,7 @@ func (api *FilterAPI) Logs(ctx context.Context, crit FilterCriteria) (*rpc.Subsc // do historical sync from n to head f := api.sys.NewRangeFilter(n, head, crit.Addresses, crit.Topics) - logChan, errChan := f.rangeLogsAsync(ctx) + logChan, errChan := f.rangeLogsAsync(cctx) // subscribe rmLogs query := ethereum.FilterQuery{FromBlock: big.NewInt(n), ToBlock: big.NewInt(head), Addresses: crit.Addresses, Topics: crit.Topics} From 290904803e68e8160addef6e3ab40386b5b67ca7 Mon Sep 17 00:00:00 2001 From: jsvisa Date: Thu, 8 Jun 2023 07:48:54 +0000 Subject: [PATCH 04/70] eth/filters: restruct Signed-off-by: jsvisa --- eth/filters/api.go | 91 ++++++++++++++++++++++++---------------------- 1 file changed, 47 insertions(+), 44 deletions(-) diff --git a/eth/filters/api.go b/eth/filters/api.go index 0eb977cadf..ebb2a45a2f 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -271,13 +271,47 @@ func (api *FilterAPI) Logs(ctx context.Context, crit FilterCriteria) (*rpc.Subsc var ( rpcSub = notifier.CreateSubscription() matchedLogs = make(chan []*types.Log) - rmLogsCh = make(chan []*types.Log) ) - syncHistLogs := func(n int64) error { - // the ctx will be canceled after this function is called + filterLiveLogs := func() error { + logsSub, err := api.events.SubscribeLogs(ethereum.FilterQuery(crit), matchedLogs) + if err != nil { + return err + } + + go func() { + for { + select { + case logs := <-matchedLogs: + for _, log := range logs { + log := log + notifier.Notify(rpcSub.ID, &log) + } + case <-rpcSub.Err(): // client send an unsubscribe request + logsSub.Unsubscribe() + return + case <-notifier.Closed(): // connection dropped + logsSub.Unsubscribe() + return + } + } + }() + return nil + } + + // do live filter only + if crit.FromBlock == nil { + err := filterLiveLogs() + return rpcSub, err + } + + filterHistLogs := func(n int64) error { + // the ctx will be canceled after this callback(filter.Logs) is called // and we are run in a background goroutine, use a new context instead - cctx := context.Background() + var ( + cctx = context.Background() + rmLogsCh = make(chan []*types.Log) + ) for { // get the latest block header head := api.sys.backend.CurrentHeader().Number.Int64() @@ -331,47 +365,16 @@ func (api *FilterAPI) Logs(ctx context.Context, crit FilterCriteria) (*rpc.Subsc } } - syncLiveLogs := func() error { - logsSub, err := api.events.SubscribeLogs(ethereum.FilterQuery(crit), matchedLogs) - if err != nil { - return err + n := crit.FromBlock.Int64() + go func() { + // do historical sync first + if err := filterHistLogs(n); err != nil { + return } - - go func() { - for { - select { - case logs := <-matchedLogs: - for _, log := range logs { - log := log - notifier.Notify(rpcSub.ID, &log) - } - case <-rpcSub.Err(): // client send an unsubscribe request - logsSub.Unsubscribe() - return - case <-notifier.Closed(): // connection dropped - logsSub.Unsubscribe() - return - } - } - }() - return nil - } - - if crit.FromBlock != nil { - n := crit.FromBlock.Int64() - go func() { - // do historical sync first - if err := syncHistLogs(n); err != nil { - return - } - // subscribe from latest - syncLiveLogs() - }() - return rpcSub, nil - } - - err := syncLiveLogs() - return rpcSub, err + // then subscribe from the header + filterLiveLogs() + }() + return rpcSub, nil } // FilterCriteria represents a request to create a new filter. From 390a4b301828cbcfbc3e1ef6641d5b7a193490c5 Mon Sep 17 00:00:00 2001 From: jsvisa Date: Thu, 8 Jun 2023 08:20:24 +0000 Subject: [PATCH 05/70] eth/filters: more accurate check of live or hist Signed-off-by: jsvisa --- eth/filters/api.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/eth/filters/api.go b/eth/filters/api.go index ebb2a45a2f..fa78629c95 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -299,8 +299,12 @@ func (api *FilterAPI) Logs(ctx context.Context, crit FilterCriteria) (*rpc.Subsc return nil } + isLiveFilter := true + if crit.FromBlock != nil { + isLiveFilter = crit.FromBlock.Sign() < 0 + } // do live filter only - if crit.FromBlock == nil { + if isLiveFilter { err := filterLiveLogs() return rpcSub, err } @@ -346,11 +350,11 @@ func (api *FilterAPI) Logs(ctx context.Context, crit FilterCriteria) (*rpc.Subsc } } case err := <-errChan: + // range filter is done or error, let's also stop the rmLogs subscribe + rmLogsSub.Unsubscribe() if err != nil { return err } - // range filter is done, let's also stop the rmLogs subscribe - rmLogsSub.Unsubscribe() break FORLOOP } } From d30d0f0a15dc16c8d2048501c065d719aa27b962 Mon Sep 17 00:00:00 2001 From: jsvisa Date: Thu, 8 Jun 2023 08:29:05 +0000 Subject: [PATCH 06/70] eth/filters: handle toBlock Signed-off-by: jsvisa --- eth/filters/api.go | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/eth/filters/api.go b/eth/filters/api.go index fa78629c95..4e3782f54d 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -309,7 +309,7 @@ func (api *FilterAPI) Logs(ctx context.Context, crit FilterCriteria) (*rpc.Subsc return rpcSub, err } - filterHistLogs := func(n int64) error { + filterHistLogs := func(n int64) (int64, error) { // the ctx will be canceled after this callback(filter.Logs) is called // and we are run in a background goroutine, use a new context instead var ( @@ -320,18 +320,22 @@ func (api *FilterAPI) Logs(ctx context.Context, crit FilterCriteria) (*rpc.Subsc // get the latest block header head := api.sys.backend.CurrentHeader().Number.Int64() if n >= head { - return nil + return head, nil } - // do historical sync from n to head - f := api.sys.NewRangeFilter(n, head, crit.Addresses, crit.Topics) + to := big.NewInt(head) + if toBlock := crit.ToBlock; toBlock != nil && toBlock.Sign() > 0 && toBlock.Cmp(to) < 0 { + to = toBlock + } + // do historical sync from n to min(head, to) + f := api.sys.NewRangeFilter(n, to.Int64(), crit.Addresses, crit.Topics) logChan, errChan := f.rangeLogsAsync(cctx) // subscribe rmLogs - query := ethereum.FilterQuery{FromBlock: big.NewInt(n), ToBlock: big.NewInt(head), Addresses: crit.Addresses, Topics: crit.Topics} + query := ethereum.FilterQuery{FromBlock: big.NewInt(n), ToBlock: to, Addresses: crit.Addresses, Topics: crit.Topics} rmLogsSub, err := api.events.SubscribeLogs(query, rmLogsCh) if err != nil { - return err + return 0, err } var rmLogs []*types.Log @@ -340,7 +344,7 @@ func (api *FilterAPI) Logs(ctx context.Context, crit FilterCriteria) (*rpc.Subsc select { case <-notifier.Closed(): rmLogsSub.Unsubscribe() - return errors.New("connection dropped") + return 0, errors.New("connection dropped") case log := <-logChan: notifier.Notify(rpcSub.ID, &log) case logs := <-rmLogsCh: @@ -353,7 +357,7 @@ func (api *FilterAPI) Logs(ctx context.Context, crit FilterCriteria) (*rpc.Subsc // range filter is done or error, let's also stop the rmLogs subscribe rmLogsSub.Unsubscribe() if err != nil { - return err + return 0, err } break FORLOOP } @@ -372,7 +376,12 @@ func (api *FilterAPI) Logs(ctx context.Context, crit FilterCriteria) (*rpc.Subsc n := crit.FromBlock.Int64() go func() { // do historical sync first - if err := filterHistLogs(n); err != nil { + head, err := filterHistLogs(n) + if err != nil { + return + } + // if toBlock is limited and handled, no need to subscribe live logs anymore + if toBlock := crit.ToBlock; toBlock != nil && toBlock.Sign() > 0 && toBlock.Cmp(big.NewInt(head)) <= 0 { return } // then subscribe from the header From 1380a479d25513664f4637a6cb222b1eb5efadf4 Mon Sep 17 00:00:00 2001 From: jsvisa Date: Thu, 8 Jun 2023 08:55:30 +0000 Subject: [PATCH 07/70] eth/filters: notify with error check Signed-off-by: jsvisa --- eth/filters/api.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/eth/filters/api.go b/eth/filters/api.go index 4e3782f54d..6a48a9413c 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -345,6 +345,9 @@ func (api *FilterAPI) Logs(ctx context.Context, crit FilterCriteria) (*rpc.Subsc case <-notifier.Closed(): rmLogsSub.Unsubscribe() return 0, errors.New("connection dropped") + case err := <-rpcSub.Err(): // client send an unsubscribe request + rmLogsSub.Unsubscribe() + return 0, err case log := <-logChan: notifier.Notify(rpcSub.ID, &log) case logs := <-rmLogsCh: @@ -365,7 +368,14 @@ func (api *FilterAPI) Logs(ctx context.Context, crit FilterCriteria) (*rpc.Subsc // send the reorged logs for _, log := range rmLogs { - notifier.Notify(rpcSub.ID, &log) + select { + case <-notifier.Closed(): + return head, errors.New("connection dropped") + case err := <-rpcSub.Err(): // client send an unsubscribe request + return head, err + default: + notifier.Notify(rpcSub.ID, &log) + } } // move forward to the next batch From c93423eb603c5e3c93eea8b685e1b90b1dd102fa Mon Sep 17 00:00:00 2001 From: jsvisa Date: Fri, 9 Jun 2023 15:39:06 +0800 Subject: [PATCH 08/70] eth/filters: local variable Signed-off-by: jsvisa --- eth/filters/api.go | 1 + 1 file changed, 1 insertion(+) diff --git a/eth/filters/api.go b/eth/filters/api.go index 6a48a9413c..2cd27c31da 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -352,6 +352,7 @@ func (api *FilterAPI) Logs(ctx context.Context, crit FilterCriteria) (*rpc.Subsc notifier.Notify(rpcSub.ID, &log) case logs := <-rmLogsCh: for _, log := range logs { + log := log if log.Removed { rmLogs = append(rmLogs, log) } From 964e801961487380b396c119084d322fe14f8a75 Mon Sep 17 00:00:00 2001 From: jsvisa Date: Fri, 9 Jun 2023 17:31:42 +0800 Subject: [PATCH 09/70] eth/filters: wrap Notify, for unit test Signed-off-by: jsvisa --- eth/filters/api.go | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/eth/filters/api.go b/eth/filters/api.go index 2cd27c31da..973d22801a 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -261,19 +261,25 @@ func (api *FilterAPI) NewHeads(ctx context.Context) (*rpc.Subscription, error) { return rpcSub, nil } +type notify interface { + Notify(id rpc.ID, data interface{}) error + Closed() <-chan interface{} +} + // Logs creates a subscription that fires for all new log that match the given filter criteria. func (api *FilterAPI) Logs(ctx context.Context, crit FilterCriteria) (*rpc.Subscription, error) { notifier, supported := rpc.NotifierFromContext(ctx) if !supported { return &rpc.Subscription{}, rpc.ErrNotificationsUnsupported } + rpcSub := notifier.CreateSubscription() + err := api.logs(ctx, notifier, rpcSub, crit) + return rpcSub, err +} - var ( - rpcSub = notifier.CreateSubscription() - matchedLogs = make(chan []*types.Log) - ) - +func (api *FilterAPI) logs(ctx context.Context, notifier notify, rpcSub *rpc.Subscription, crit FilterCriteria) error { filterLiveLogs := func() error { + matchedLogs := make(chan []*types.Log) logsSub, err := api.events.SubscribeLogs(ethereum.FilterQuery(crit), matchedLogs) if err != nil { return err @@ -306,7 +312,7 @@ func (api *FilterAPI) Logs(ctx context.Context, crit FilterCriteria) (*rpc.Subsc // do live filter only if isLiveFilter { err := filterLiveLogs() - return rpcSub, err + return err } filterHistLogs := func(n int64) (int64, error) { @@ -398,7 +404,7 @@ func (api *FilterAPI) Logs(ctx context.Context, crit FilterCriteria) (*rpc.Subsc // then subscribe from the header filterLiveLogs() }() - return rpcSub, nil + return nil } // FilterCriteria represents a request to create a new filter. From 2c0ca046b59a9611930bec29440da008804918fe Mon Sep 17 00:00:00 2001 From: jsvisa Date: Sun, 11 Jun 2023 16:58:34 +0800 Subject: [PATCH 10/70] eth/filters: move inner function as private method Signed-off-by: jsvisa --- eth/filters/api.go | 217 +++++++++++++++++++++++---------------------- 1 file changed, 110 insertions(+), 107 deletions(-) diff --git a/eth/filters/api.go b/eth/filters/api.go index 973d22801a..eccdf9fcae 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -261,7 +261,7 @@ func (api *FilterAPI) NewHeads(ctx context.Context) (*rpc.Subscription, error) { return rpcSub, nil } -type notify interface { +type notifier interface { Notify(id rpc.ID, data interface{}) error Closed() <-chan interface{} } @@ -277,123 +277,126 @@ func (api *FilterAPI) Logs(ctx context.Context, crit FilterCriteria) (*rpc.Subsc return rpcSub, err } -func (api *FilterAPI) logs(ctx context.Context, notifier notify, rpcSub *rpc.Subscription, crit FilterCriteria) error { - filterLiveLogs := func() error { - matchedLogs := make(chan []*types.Log) - logsSub, err := api.events.SubscribeLogs(ethereum.FilterQuery(crit), matchedLogs) - if err != nil { - return err - } - - go func() { - for { - select { - case logs := <-matchedLogs: - for _, log := range logs { - log := log - notifier.Notify(rpcSub.ID, &log) - } - case <-rpcSub.Err(): // client send an unsubscribe request - logsSub.Unsubscribe() - return - case <-notifier.Closed(): // connection dropped - logsSub.Unsubscribe() - return - } - } - }() - return nil +// liveLogs only retrieves live logs +func (api *FilterAPI) liveLogs(ctx context.Context, notify notifier, rpcSub *rpc.Subscription, crit FilterCriteria) error { + matchedLogs := make(chan []*types.Log) + logsSub, err := api.events.SubscribeLogs(ethereum.FilterQuery(crit), matchedLogs) + if err != nil { + return err } + go func() { + for { + select { + case logs := <-matchedLogs: + for _, log := range logs { + log := log + notify.Notify(rpcSub.ID, &log) + } + case <-rpcSub.Err(): // client send an unsubscribe request + logsSub.Unsubscribe() + return + case <-notify.Closed(): // connection dropped + logsSub.Unsubscribe() + return + } + } + }() + return nil +} + +// histLogs only retrieves the logs older than current header +func (api *FilterAPI) histLogs(ctx context.Context, notifier notifier, rpcSub *rpc.Subscription, crit FilterCriteria) (int64, error) { + // the ctx will be canceled after this callback(filter.Logs) is called + // and we are run in a background goroutine, use a new context instead + var ( + cctx = context.Background() + rmLogsCh = make(chan []*types.Log) + n = crit.FromBlock.Int64() + ) + for { + // get the latest block header + head := api.sys.backend.CurrentHeader().Number.Int64() + if n >= head { + return head, nil + } + + to := big.NewInt(head) + if toBlock := crit.ToBlock; toBlock != nil && toBlock.Sign() > 0 && toBlock.Cmp(to) < 0 { + to = toBlock + } + // do historical sync from n to min(head, to) + f := api.sys.NewRangeFilter(n, to.Int64(), crit.Addresses, crit.Topics) + logChan, errChan := f.rangeLogsAsync(cctx) + + // subscribe rmLogs + query := ethereum.FilterQuery{FromBlock: big.NewInt(n), ToBlock: to, Addresses: crit.Addresses, Topics: crit.Topics} + rmLogsSub, err := api.events.SubscribeLogs(query, rmLogsCh) + if err != nil { + return 0, err + } + + var rmLogs []*types.Log + FORLOOP: + for { + select { + case <-notifier.Closed(): + rmLogsSub.Unsubscribe() + return 0, errors.New("connection dropped") + case err := <-rpcSub.Err(): // client send an unsubscribe request + rmLogsSub.Unsubscribe() + return 0, err + case log := <-logChan: + notifier.Notify(rpcSub.ID, &log) + case logs := <-rmLogsCh: + for _, log := range logs { + log := log + if log.Removed { + rmLogs = append(rmLogs, log) + } + } + case err := <-errChan: + // range filter is done or error, let's also stop the rmLogs subscribe + rmLogsSub.Unsubscribe() + if err != nil { + return 0, err + } + break FORLOOP + } + } + + // send the reorged logs + for _, log := range rmLogs { + select { + case <-notifier.Closed(): + return head, errors.New("connection dropped") + case err := <-rpcSub.Err(): // client send an unsubscribe request + return head, err + default: + notifier.Notify(rpcSub.ID, &log) + } + } + + // move forward to the next batch + n = head + 1 + } +} + +// logs is the inner implmention of logs filter +func (api *FilterAPI) logs(ctx context.Context, notifier notifier, rpcSub *rpc.Subscription, crit FilterCriteria) error { isLiveFilter := true if crit.FromBlock != nil { isLiveFilter = crit.FromBlock.Sign() < 0 } + // do live filter only if isLiveFilter { - err := filterLiveLogs() - return err + return api.liveLogs(ctx, notifier, rpcSub, crit) } - filterHistLogs := func(n int64) (int64, error) { - // the ctx will be canceled after this callback(filter.Logs) is called - // and we are run in a background goroutine, use a new context instead - var ( - cctx = context.Background() - rmLogsCh = make(chan []*types.Log) - ) - for { - // get the latest block header - head := api.sys.backend.CurrentHeader().Number.Int64() - if n >= head { - return head, nil - } - - to := big.NewInt(head) - if toBlock := crit.ToBlock; toBlock != nil && toBlock.Sign() > 0 && toBlock.Cmp(to) < 0 { - to = toBlock - } - // do historical sync from n to min(head, to) - f := api.sys.NewRangeFilter(n, to.Int64(), crit.Addresses, crit.Topics) - logChan, errChan := f.rangeLogsAsync(cctx) - - // subscribe rmLogs - query := ethereum.FilterQuery{FromBlock: big.NewInt(n), ToBlock: to, Addresses: crit.Addresses, Topics: crit.Topics} - rmLogsSub, err := api.events.SubscribeLogs(query, rmLogsCh) - if err != nil { - return 0, err - } - - var rmLogs []*types.Log - FORLOOP: - for { - select { - case <-notifier.Closed(): - rmLogsSub.Unsubscribe() - return 0, errors.New("connection dropped") - case err := <-rpcSub.Err(): // client send an unsubscribe request - rmLogsSub.Unsubscribe() - return 0, err - case log := <-logChan: - notifier.Notify(rpcSub.ID, &log) - case logs := <-rmLogsCh: - for _, log := range logs { - log := log - if log.Removed { - rmLogs = append(rmLogs, log) - } - } - case err := <-errChan: - // range filter is done or error, let's also stop the rmLogs subscribe - rmLogsSub.Unsubscribe() - if err != nil { - return 0, err - } - break FORLOOP - } - } - - // send the reorged logs - for _, log := range rmLogs { - select { - case <-notifier.Closed(): - return head, errors.New("connection dropped") - case err := <-rpcSub.Err(): // client send an unsubscribe request - return head, err - default: - notifier.Notify(rpcSub.ID, &log) - } - } - - // move forward to the next batch - n = head + 1 - } - } - - n := crit.FromBlock.Int64() go func() { // do historical sync first - head, err := filterHistLogs(n) + head, err := api.histLogs(ctx, notifier, rpcSub, crit) if err != nil { return } @@ -402,7 +405,7 @@ func (api *FilterAPI) logs(ctx context.Context, notifier notify, rpcSub *rpc.Sub return } // then subscribe from the header - filterLiveLogs() + api.liveLogs(ctx, notifier, rpcSub, crit) }() return nil } From 6f672fa997f84fc99cb680f0facf464e7e95f2c9 Mon Sep 17 00:00:00 2001 From: jsvisa Date: Mon, 12 Jun 2023 07:30:04 +0800 Subject: [PATCH 11/70] eth/filters: add the first logs testcase Signed-off-by: Delweng --- eth/filters/filter_system_test.go | 150 ++++++++++++++++++++++++++++++ 1 file changed, 150 insertions(+) diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index 013c1ae527..4e009bee15 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -17,8 +17,10 @@ package filters import ( + "bytes" "context" "errors" + "fmt" "math/big" "reflect" "runtime" @@ -31,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/core/filtermaps" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/internal/ethapi" @@ -694,3 +697,150 @@ func TestPendingTxFilterDeadlock(t *testing.T) { } } } + +func flattenLogs(pl [][]*types.Log) []*types.Log { + var logs []*types.Log + for _, l := range pl { + logs = append(logs, l...) + } + return logs +} + +type mockNotifier struct { + c chan interface{} +} + +func newMockNotifier() *mockNotifier { + return &mockNotifier{c: make(chan interface{})} +} + +func (n *mockNotifier) Notify(id rpc.ID, data interface{}) error { + n.c <- data + return nil +} + +func (n *mockNotifier) Closed() <-chan interface{} { + return nil +} + +// TestLogsSubscription tests if a rpc subscription receives the correct logs +func TestLogsSubscription(t *testing.T) { + t.Parallel() + + var ( + signer = types.HomesteadSigner{} + key, _ = crypto.GenerateKey() + addr = crypto.PubkeyToAddress(key.PublicKey) + contract = common.HexToAddress("0000000000000000000000000000000000031ec7") + genesis = &core.Genesis{ + Config: params.TestChainConfig, + Alloc: core.GenesisAlloc{ + // // SPDX-License-Identifier: GPL-3.0 + // pragma solidity >=0.7.0 <0.9.0; + // + // contract Token { + // event Transfer(address indexed from, address indexed to, uint256 value); + // function transfer(address to, uint256 value) public returns (bool) { + // emit Transfer(msg.sender, to, value); + // return true; + // } + // } + contract: {Balance: big.NewInt(params.Ether), Code: common.FromHex("0x608060405234801561001057600080fd5b506004361061002b5760003560e01c8063a9059cbb14610030575b600080fd5b61004a6004803603810190610045919061016a565b610060565b60405161005791906101c5565b60405180910390f35b60008273ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff167fddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef846040516100bf91906101ef565b60405180910390a36001905092915050565b600080fd5b60073ffffffffffffffffffffffffffffffffffffffff82169050919050565b6000610101826100d6565b9050919050565b610111816100f6565b811461011c57600080fd5b50565b60008135905061012e81610108565b92915050565b6000819050919050565b61014781610134565b811461015257600080fd5b50565b6000813590506101648161013e565b92915050565b60008060408385031215610181576101806100d1565b5b600061018f8582860161011f565b92505060206101a085828601610155565b9150509250929050565b60008115159050919050565b6101bf816101aa565b82525050565b60006020820190506101da60008301846101b6565b92915050565b6101e981610134565b82525050565b600060208201905061020460008301846101e0565b9291505056fea2646970667358221220b469033f4b77b9565ee84e0a2f04d496b18160d26034d54f9487e57788fd36d564736f6c63430008120033")}, + addr: {Balance: big.NewInt(params.Ether)}, + }, + } + + db, blocks, receipts = core.GenerateChainWithGenesis(genesis, ethash.NewFaker(), 4, func(i int, b *core.BlockGen) { + // transfer(address to, uint256 value) + data := fmt.Sprintf("0xa9059cbb%s%s", common.HexToHash(common.BigToAddress(big.NewInt(int64(i + 1))).Hex()).String()[2:], common.BytesToHash([]byte{byte(i + 11)}).String()[2:]) + tx, _ := types.SignTx(types.NewTx(&types.LegacyTx{Nonce: uint64(i), To: &contract, Value: big.NewInt(0), Gas: 46000, GasPrice: b.BaseFee(), Data: common.FromHex(data)}), signer, key) + b.AddTx(tx) + }) + ) + + for i, block := range blocks { + rawdb.WriteBlock(db, block) + rawdb.WriteCanonicalHash(db, block.Hash(), block.NumberU64()) + rawdb.WriteHeadBlockHash(db, block.Hash()) + rawdb.WriteReceipts(db, block.Hash(), block.NumberU64(), receipts[i]) + } + + var ( + _, sys = newTestFilterSystem(t, db, Config{}) + api = NewFilterAPI(sys, false) + // Transfer(address indexed from, address indexed to, uint256 value); + topic = common.HexToHash("0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef") + ) + + allLogs := []*types.Log{ + {Topics: []common.Hash{topic, common.HexToHash(addr.Hex()), common.BigToHash(big.NewInt(1))}, Data: common.BigToHash(big.NewInt(11)).Bytes(), BlockNumber: 1}, + {Topics: []common.Hash{topic, common.HexToHash(addr.Hex()), common.BigToHash(big.NewInt(2))}, Data: common.BigToHash(big.NewInt(12)).Bytes(), BlockNumber: 2}, + {Topics: []common.Hash{topic, common.HexToHash(addr.Hex()), common.BigToHash(big.NewInt(3))}, Data: common.BigToHash(big.NewInt(13)).Bytes(), BlockNumber: 3}, + {Topics: []common.Hash{topic, common.HexToHash(addr.Hex()), common.BigToHash(big.NewInt(4))}, Data: common.BigToHash(big.NewInt(14)).Bytes(), BlockNumber: 4}, + } + + // pendingBlockNumber := big.NewInt(rpc.PendingBlockNumber.Int64()) + testCases := []struct { + crit FilterCriteria + expected []*types.Log + notifier *mockNotifier + sub *rpc.Subscription + err chan error + }{ + // from 1 to latest + { + FilterCriteria{FromBlock: big.NewInt(0), ToBlock: big.NewInt(5)}, + allLogs, newMockNotifier(), &rpc.Subscription{ID: rpc.NewID()}, nil, + }, + } + + // subscribe logs + for i, tc := range testCases { + testCases[i].err = make(chan error) + err := api.logs(context.Background(), tc.notifier, tc.sub, tc.crit) + if err != nil { + t.Fatalf("SubscribeLogs %d failed: %v\n", i, err) + } + } + + // receive logs + for n, test := range testCases { + i := n + tt := test + go func() { + var fetched []*types.Log + + timeout := time.After(3 * time.Second) + fetchLoop: + for { + select { + case log := <-tt.notifier.c: + fetched = append(fetched, *log.(**types.Log)) + case <-timeout: + break fetchLoop + } + } + + if len(fetched) != len(tt.expected) { + tt.err <- fmt.Errorf("invalid number of logs for case %d, want %d log(s), got %d", i, len(tt.expected), len(fetched)) + return + } + + for l := range fetched { + have, want := fetched[l], tt.expected[l] + if have.Address != contract || len(have.Topics) != len(want.Topics) || have.Topics[2] != want.Topics[2] || bytes.Compare(have.Data, want.Data) != 0 || have.BlockNumber != want.BlockNumber { + tt.err <- fmt.Errorf("invalid log on index %d for case %d have: %+v want: %+v\n", l, i, have, want) + return + } + } + tt.err <- nil + }() + } + + for i := range testCases { + err := <-testCases[i].err + if err != nil { + t.Fatalf("test %d failed: %v", i, err) + } + } +} From 88ac691d01bcab016f2f3035bdc8ccb2764b9940 Mon Sep 17 00:00:00 2001 From: Delweng Date: Tue, 13 Jun 2023 17:11:41 +0800 Subject: [PATCH 12/70] eth/filters: more testcase Signed-off-by: Delweng --- eth/filters/filter_system_test.go | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index 4e009bee15..cac36c0ef9 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -772,14 +772,16 @@ func TestLogsSubscription(t *testing.T) { topic = common.HexToHash("0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef") ) + i2h := func(i int) common.Hash { return common.BigToHash(big.NewInt(int64(i))) } + allLogs := []*types.Log{ - {Topics: []common.Hash{topic, common.HexToHash(addr.Hex()), common.BigToHash(big.NewInt(1))}, Data: common.BigToHash(big.NewInt(11)).Bytes(), BlockNumber: 1}, - {Topics: []common.Hash{topic, common.HexToHash(addr.Hex()), common.BigToHash(big.NewInt(2))}, Data: common.BigToHash(big.NewInt(12)).Bytes(), BlockNumber: 2}, - {Topics: []common.Hash{topic, common.HexToHash(addr.Hex()), common.BigToHash(big.NewInt(3))}, Data: common.BigToHash(big.NewInt(13)).Bytes(), BlockNumber: 3}, - {Topics: []common.Hash{topic, common.HexToHash(addr.Hex()), common.BigToHash(big.NewInt(4))}, Data: common.BigToHash(big.NewInt(14)).Bytes(), BlockNumber: 4}, + {Topics: []common.Hash{topic, common.HexToHash(addr.Hex()), i2h(1)}, Data: i2h(11).Bytes(), BlockNumber: 1}, + {Topics: []common.Hash{topic, common.HexToHash(addr.Hex()), i2h(2)}, Data: i2h(12).Bytes(), BlockNumber: 2}, + {Topics: []common.Hash{topic, common.HexToHash(addr.Hex()), i2h(3)}, Data: i2h(13).Bytes(), BlockNumber: 3}, + {Topics: []common.Hash{topic, common.HexToHash(addr.Hex()), i2h(4)}, Data: i2h(14).Bytes(), BlockNumber: 4}, } - // pendingBlockNumber := big.NewInt(rpc.PendingBlockNumber.Int64()) + pendingBlockNumber := big.NewInt(rpc.PendingBlockNumber.Int64()) testCases := []struct { crit FilterCriteria expected []*types.Log @@ -787,11 +789,21 @@ func TestLogsSubscription(t *testing.T) { sub *rpc.Subscription err chan error }{ - // from 1 to latest + // from 0 to latest { FilterCriteria{FromBlock: big.NewInt(0), ToBlock: big.NewInt(5)}, allLogs, newMockNotifier(), &rpc.Subscription{ID: rpc.NewID()}, nil, }, + // from 1 to latest + { + FilterCriteria{FromBlock: big.NewInt(1), ToBlock: pendingBlockNumber}, + allLogs, newMockNotifier(), &rpc.Subscription{ID: rpc.NewID()}, nil, + }, + // from 1 to 3 + { + FilterCriteria{FromBlock: big.NewInt(1), ToBlock: big.NewInt(3)}, + allLogs[0:3], newMockNotifier(), &rpc.Subscription{ID: rpc.NewID()}, nil, + }, } // subscribe logs From f50087ebc763747250af6a835f683cbf6cc09bbe Mon Sep 17 00:00:00 2001 From: Delweng Date: Tue, 13 Jun 2023 17:13:51 +0800 Subject: [PATCH 13/70] eth/filters: header is nullable Signed-off-by: Delweng --- eth/filters/api.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/eth/filters/api.go b/eth/filters/api.go index eccdf9fcae..589abed99b 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -316,7 +316,11 @@ func (api *FilterAPI) histLogs(ctx context.Context, notifier notifier, rpcSub *r ) for { // get the latest block header - head := api.sys.backend.CurrentHeader().Number.Int64() + header := api.sys.backend.CurrentHeader() + if header == nil { + return 0, errors.New("header is null") + } + head := header.Number.Int64() if n >= head { return head, nil } From 8c429c95b34cdbb311197e1b78aa261c8724a4a1 Mon Sep 17 00:00:00 2001 From: Delweng Date: Tue, 13 Jun 2023 18:11:14 +0800 Subject: [PATCH 14/70] eth/filters: check hist only mode Signed-off-by: Delweng --- eth/filters/api.go | 13 ++++++++----- eth/filters/filter_system_test.go | 10 +++++----- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/eth/filters/api.go b/eth/filters/api.go index 589abed99b..5791a8e703 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -398,16 +398,19 @@ func (api *FilterAPI) logs(ctx context.Context, notifier notifier, rpcSub *rpc.S return api.liveLogs(ctx, notifier, rpcSub, crit) } + // if toBlock is limited and handled, no need to subscribe live logs anymore + if toBlock := crit.ToBlock; toBlock != nil { + if header := api.sys.backend.CurrentHeader(); header != nil && toBlock.Sign() > 0 && toBlock.Cmp(header.Number) <= 0 { + return errors.New("historical only log subscription is not supported") + } + } + go func() { // do historical sync first - head, err := api.histLogs(ctx, notifier, rpcSub, crit) + _, err := api.histLogs(ctx, notifier, rpcSub, crit) if err != nil { return } - // if toBlock is limited and handled, no need to subscribe live logs anymore - if toBlock := crit.ToBlock; toBlock != nil && toBlock.Sign() > 0 && toBlock.Cmp(big.NewInt(head)) <= 0 { - return - } // then subscribe from the header api.liveLogs(ctx, notifier, rpcSub, crit) }() diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index cac36c0ef9..f5b4f2565b 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -799,11 +799,11 @@ func TestLogsSubscription(t *testing.T) { FilterCriteria{FromBlock: big.NewInt(1), ToBlock: pendingBlockNumber}, allLogs, newMockNotifier(), &rpc.Subscription{ID: rpc.NewID()}, nil, }, - // from 1 to 3 - { - FilterCriteria{FromBlock: big.NewInt(1), ToBlock: big.NewInt(3)}, - allLogs[0:3], newMockNotifier(), &rpc.Subscription{ID: rpc.NewID()}, nil, - }, + // // from 1 to 3 + // { + // FilterCriteria{FromBlock: big.NewInt(1), ToBlock: big.NewInt(3)}, + // allLogs[0:3], newMockNotifier(), &rpc.Subscription{ID: rpc.NewID()}, nil, + // }, } // subscribe logs From 554662b3d745ce6785da60d78cab5c16f17b606e Mon Sep 17 00:00:00 2001 From: Delweng Date: Tue, 13 Jun 2023 20:42:39 +0800 Subject: [PATCH 15/70] eth/filters: go lint Signed-off-by: jsvisa Signed-off-by: jsvisa --- eth/filters/filter_system_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index f5b4f2565b..55c4537e36 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -840,7 +840,7 @@ func TestLogsSubscription(t *testing.T) { for l := range fetched { have, want := fetched[l], tt.expected[l] - if have.Address != contract || len(have.Topics) != len(want.Topics) || have.Topics[2] != want.Topics[2] || bytes.Compare(have.Data, want.Data) != 0 || have.BlockNumber != want.BlockNumber { + if have.Address != contract || len(have.Topics) != len(want.Topics) || !bytes.Equal(have.Topics[2].Bytes(), want.Topics[2].Bytes()) || !bytes.Equal(have.Data, want.Data) || have.BlockNumber != want.BlockNumber { tt.err <- fmt.Errorf("invalid log on index %d for case %d have: %+v want: %+v\n", l, i, have, want) return } From c7bfb244df09d0dac69bed3e1f7532a881b630fb Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Wed, 14 Jun 2023 17:51:51 +0200 Subject: [PATCH 16/70] fix minor bug Signed-off-by: jsvisa --- eth/filters/filter_system.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/eth/filters/filter_system.go b/eth/filters/filter_system.go index 751cd417e8..2ce12f2e94 100644 --- a/eth/filters/filter_system.go +++ b/eth/filters/filter_system.go @@ -327,6 +327,10 @@ func (es *EventSystem) SubscribeLogs(crit ethereum.FilterQuery, logs chan []*typ if from >= 0 && to >= 0 && to >= from { return es.subscribeLogs(crit, logs), nil } + // interested in mined logs from a specific block number, new logs and pending logs + if (from == rpc.LatestBlockNumber || from >= 0) && to == rpc.PendingBlockNumber { + return es.subscribeMinedPendingLogs(crit, logs), nil + } // interested in logs from a specific block number to new mined blocks if from >= 0 && to == rpc.LatestBlockNumber { return es.subscribeLogs(crit, logs), nil From 89cce937ab3ef5d8781bc24b8ce75a5c0b42895a Mon Sep 17 00:00:00 2001 From: jsvisa Date: Tue, 27 Jun 2023 00:00:51 +0800 Subject: [PATCH 17/70] eth/filters: add notifier interface doc Signed-off-by: jsvisa --- eth/filters/api.go | 64 ++++++++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 31 deletions(-) diff --git a/eth/filters/api.go b/eth/filters/api.go index 5791a8e703..076117296b 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -261,6 +261,8 @@ func (api *FilterAPI) NewHeads(ctx context.Context) (*rpc.Subscription, error) { return rpcSub, nil } +// notifier is used for broadcasting data(eg: logs) to rpc receivers +// used in unit testing type notifier interface { Notify(id rpc.ID, data interface{}) error Closed() <-chan interface{} @@ -277,6 +279,37 @@ func (api *FilterAPI) Logs(ctx context.Context, crit FilterCriteria) (*rpc.Subsc return rpcSub, err } +// logs is the inner implmention of logs filter +func (api *FilterAPI) logs(ctx context.Context, notifier notifier, rpcSub *rpc.Subscription, crit FilterCriteria) error { + isLiveFilter := true + if crit.FromBlock != nil { + isLiveFilter = crit.FromBlock.Sign() < 0 + } + + // do live filter only + if isLiveFilter { + return api.liveLogs(ctx, notifier, rpcSub, crit) + } + + // if toBlock is limited and handled, no need to subscribe live logs anymore + if toBlock := crit.ToBlock; toBlock != nil { + if header := api.sys.backend.CurrentHeader(); header != nil && toBlock.Sign() > 0 && toBlock.Cmp(header.Number) <= 0 { + return errors.New("historical only log subscription is not supported") + } + } + + go func() { + // do historical sync first + _, err := api.histLogs(ctx, notifier, rpcSub, crit) + if err != nil { + return + } + // then subscribe from the header + api.liveLogs(ctx, notifier, rpcSub, crit) + }() + return nil +} + // liveLogs only retrieves live logs func (api *FilterAPI) liveLogs(ctx context.Context, notify notifier, rpcSub *rpc.Subscription, crit FilterCriteria) error { matchedLogs := make(chan []*types.Log) @@ -386,37 +419,6 @@ func (api *FilterAPI) histLogs(ctx context.Context, notifier notifier, rpcSub *r } } -// logs is the inner implmention of logs filter -func (api *FilterAPI) logs(ctx context.Context, notifier notifier, rpcSub *rpc.Subscription, crit FilterCriteria) error { - isLiveFilter := true - if crit.FromBlock != nil { - isLiveFilter = crit.FromBlock.Sign() < 0 - } - - // do live filter only - if isLiveFilter { - return api.liveLogs(ctx, notifier, rpcSub, crit) - } - - // if toBlock is limited and handled, no need to subscribe live logs anymore - if toBlock := crit.ToBlock; toBlock != nil { - if header := api.sys.backend.CurrentHeader(); header != nil && toBlock.Sign() > 0 && toBlock.Cmp(header.Number) <= 0 { - return errors.New("historical only log subscription is not supported") - } - } - - go func() { - // do historical sync first - _, err := api.histLogs(ctx, notifier, rpcSub, crit) - if err != nil { - return - } - // then subscribe from the header - api.liveLogs(ctx, notifier, rpcSub, crit) - }() - return nil -} - // FilterCriteria represents a request to create a new filter. // Same as ethereum.FilterQuery but with UnmarshalJSON() method. type FilterCriteria ethereum.FilterQuery From 5cb56120b4637269b7da7cc066ca628528425439 Mon Sep 17 00:00:00 2001 From: jsvisa Date: Tue, 27 Jun 2023 00:25:25 +0800 Subject: [PATCH 18/70] eth/filters: logs subscribe donot support to criteria Signed-off-by: jsvisa --- eth/filters/api.go | 92 ++++++++++++++++++++++------------------------ 1 file changed, 44 insertions(+), 48 deletions(-) diff --git a/eth/filters/api.go b/eth/filters/api.go index 076117296b..112abf51f6 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -35,23 +35,10 @@ import ( ) var ( - errInvalidTopic = errors.New("invalid topic(s)") - errFilterNotFound = errors.New("filter not found") - errInvalidBlockRange = errors.New("invalid block range params") - errUnknownBlock = errors.New("unknown block") - errBlockHashWithRange = errors.New("can't specify fromBlock/toBlock with blockHash") - errPendingLogsUnsupported = errors.New("pending logs are not supported") - errExceedMaxTopics = errors.New("exceed max topics") - errExceedMaxAddresses = errors.New("exceed max addresses") -) - -const ( - // The maximum number of addresses allowed in a filter criteria - maxAddresses = 1000 - // The maximum number of topic criteria allowed, vm.LOG4 - vm.LOG0 - maxTopics = 4 - // The maximum number of allowed topics within a topic criteria - maxSubTopics = 1000 + errInvalidTopic = errors.New("invalid topic(s)") + errFilterNotFound = errors.New("filter not found") + errInvalidToBlock = errors.New("log subscription does not support history block range") + errInvalidFromBlock = errors.New("invalid from block") ) // filter is a helper struct that holds meta information over the filter type @@ -280,34 +267,48 @@ func (api *FilterAPI) Logs(ctx context.Context, crit FilterCriteria) (*rpc.Subsc } // logs is the inner implmention of logs filter +// from: nil, to: nil -> only live +// from: blockNum | safe | finalized, to: nil -> historical from from and then toggle live +// every other case should fail with an error func (api *FilterAPI) logs(ctx context.Context, notifier notifier, rpcSub *rpc.Subscription, crit FilterCriteria) error { - isLiveFilter := true - if crit.FromBlock != nil { - isLiveFilter = crit.FromBlock.Sign() < 0 + var from, to rpc.BlockNumber + if crit.FromBlock == nil { + from = rpc.LatestBlockNumber + } else { + from = rpc.BlockNumber(crit.FromBlock.Int64()) + } + if crit.ToBlock == nil { + to = rpc.LatestBlockNumber + } else { + to = rpc.BlockNumber(crit.ToBlock.Int64()) } - // do live filter only - if isLiveFilter { + if to != rpc.LatestBlockNumber { + return errInvalidToBlock + } + + switch from { + case rpc.LatestBlockNumber: + // do live filter only return api.liveLogs(ctx, notifier, rpcSub, crit) - } - - // if toBlock is limited and handled, no need to subscribe live logs anymore - if toBlock := crit.ToBlock; toBlock != nil { - if header := api.sys.backend.CurrentHeader(); header != nil && toBlock.Sign() > 0 && toBlock.Cmp(header.Number) <= 0 { - return errors.New("historical only log subscription is not supported") - } - } - - go func() { - // do historical sync first - _, err := api.histLogs(ctx, notifier, rpcSub, crit) + case rpc.SafeBlockNumber, rpc.FinalizedBlockNumber: + header, err := api.sys.backend.HeaderByNumber(ctx, from) if err != nil { - return + return err } - // then subscribe from the header - api.liveLogs(ctx, notifier, rpcSub, crit) - }() - return nil + go func() { + // do historical sync first + _, err := api.histLogs(ctx, notifier, rpcSub, header.Number.Int64(), crit) + if err != nil { + return + } + // then subscribe from the header + api.liveLogs(ctx, notifier, rpcSub, crit) + }() + return nil + default: + return errInvalidFromBlock + } } // liveLogs only retrieves live logs @@ -339,13 +340,12 @@ func (api *FilterAPI) liveLogs(ctx context.Context, notify notifier, rpcSub *rpc } // histLogs only retrieves the logs older than current header -func (api *FilterAPI) histLogs(ctx context.Context, notifier notifier, rpcSub *rpc.Subscription, crit FilterCriteria) (int64, error) { +func (api *FilterAPI) histLogs(ctx context.Context, notifier notifier, rpcSub *rpc.Subscription, n int64, crit FilterCriteria) (int64, error) { // the ctx will be canceled after this callback(filter.Logs) is called // and we are run in a background goroutine, use a new context instead var ( cctx = context.Background() rmLogsCh = make(chan []*types.Log) - n = crit.FromBlock.Int64() ) for { // get the latest block header @@ -358,16 +358,12 @@ func (api *FilterAPI) histLogs(ctx context.Context, notifier notifier, rpcSub *r return head, nil } - to := big.NewInt(head) - if toBlock := crit.ToBlock; toBlock != nil && toBlock.Sign() > 0 && toBlock.Cmp(to) < 0 { - to = toBlock - } - // do historical sync from n to min(head, to) - f := api.sys.NewRangeFilter(n, to.Int64(), crit.Addresses, crit.Topics) + // do historical sync from n to head + f := api.sys.NewRangeFilter(n, head, crit.Addresses, crit.Topics) logChan, errChan := f.rangeLogsAsync(cctx) // subscribe rmLogs - query := ethereum.FilterQuery{FromBlock: big.NewInt(n), ToBlock: to, Addresses: crit.Addresses, Topics: crit.Topics} + query := ethereum.FilterQuery{FromBlock: big.NewInt(n), ToBlock: big.NewInt(head), Addresses: crit.Addresses, Topics: crit.Topics} rmLogsSub, err := api.events.SubscribeLogs(query, rmLogsCh) if err != nil { return 0, err From a43785332f4e03cf49980ab201f8958dedc5a839 Mon Sep 17 00:00:00 2001 From: jsvisa Date: Tue, 27 Jun 2023 00:26:45 +0800 Subject: [PATCH 19/70] eth/filters: dry Signed-off-by: jsvisa --- eth/filters/api.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/eth/filters/api.go b/eth/filters/api.go index 112abf51f6..f11a5f551a 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -37,6 +37,7 @@ import ( var ( errInvalidTopic = errors.New("invalid topic(s)") errFilterNotFound = errors.New("filter not found") + errConnectDropped = errors.New("connection dropped") errInvalidToBlock = errors.New("log subscription does not support history block range") errInvalidFromBlock = errors.New("invalid from block") ) @@ -375,7 +376,7 @@ func (api *FilterAPI) histLogs(ctx context.Context, notifier notifier, rpcSub *r select { case <-notifier.Closed(): rmLogsSub.Unsubscribe() - return 0, errors.New("connection dropped") + return 0, errConnectDropped case err := <-rpcSub.Err(): // client send an unsubscribe request rmLogsSub.Unsubscribe() return 0, err @@ -402,7 +403,7 @@ func (api *FilterAPI) histLogs(ctx context.Context, notifier notifier, rpcSub *r for _, log := range rmLogs { select { case <-notifier.Closed(): - return head, errors.New("connection dropped") + return head, errConnectDropped case err := <-rpcSub.Err(): // client send an unsubscribe request return head, err default: From a374b1b6ea2d754a493b87a4e2d292bb71965bfc Mon Sep 17 00:00:00 2001 From: jsvisa Date: Tue, 27 Jun 2023 00:46:39 +0800 Subject: [PATCH 20/70] eth/filters: fix the issue of wrong check on from block Signed-off-by: jsvisa --- eth/filters/api.go | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/eth/filters/api.go b/eth/filters/api.go index f11a5f551a..31252dcf04 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -288,28 +288,32 @@ func (api *FilterAPI) logs(ctx context.Context, notifier notifier, rpcSub *rpc.S return errInvalidToBlock } - switch from { - case rpc.LatestBlockNumber: - // do live filter only + // do live filter only + if from == rpc.LatestBlockNumber { return api.liveLogs(ctx, notifier, rpcSub, crit) - case rpc.SafeBlockNumber, rpc.FinalizedBlockNumber: + } + + if from == rpc.SafeBlockNumber || from == rpc.FinalizedBlockNumber { header, err := api.sys.backend.HeaderByNumber(ctx, from) if err != nil { return err } - go func() { - // do historical sync first - _, err := api.histLogs(ctx, notifier, rpcSub, header.Number.Int64(), crit) - if err != nil { - return - } - // then subscribe from the header - api.liveLogs(ctx, notifier, rpcSub, crit) - }() - return nil - default: + from = rpc.BlockNumber(header.Number.Int64()) + } + if from < 0 { return errInvalidFromBlock } + + go func() { + // do historical sync first + _, err := api.histLogs(ctx, notifier, rpcSub, int64(from), crit) + if err != nil { + return + } + // then subscribe from the header + api.liveLogs(ctx, notifier, rpcSub, crit) + }() + return nil } // liveLogs only retrieves live logs From 223f05ff76dabb4ac5b4008a9bbcacab517ecdee Mon Sep 17 00:00:00 2001 From: jsvisa Date: Tue, 27 Jun 2023 00:47:59 +0800 Subject: [PATCH 21/70] eth/filters: more testcase Signed-off-by: jsvisa --- eth/filters/filter_system_test.go | 55 ++++++++++++++++++++++--------- 1 file changed, 40 insertions(+), 15 deletions(-) diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index 55c4537e36..eb6746aed2 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -781,35 +781,60 @@ func TestLogsSubscription(t *testing.T) { {Topics: []common.Hash{topic, common.HexToHash(addr.Hex()), i2h(4)}, Data: i2h(14).Bytes(), BlockNumber: 4}, } - pendingBlockNumber := big.NewInt(rpc.PendingBlockNumber.Int64()) testCases := []struct { - crit FilterCriteria - expected []*types.Log - notifier *mockNotifier - sub *rpc.Subscription - err chan error + crit FilterCriteria + expected []*types.Log + notifier *mockNotifier + sub *rpc.Subscription + expectErr error + err chan error }{ // from 0 to latest { - FilterCriteria{FromBlock: big.NewInt(0), ToBlock: big.NewInt(5)}, - allLogs, newMockNotifier(), &rpc.Subscription{ID: rpc.NewID()}, nil, + FilterCriteria{FromBlock: big.NewInt(0)}, + allLogs, newMockNotifier(), &rpc.Subscription{ID: rpc.NewID()}, nil, nil, }, // from 1 to latest { - FilterCriteria{FromBlock: big.NewInt(1), ToBlock: pendingBlockNumber}, - allLogs, newMockNotifier(), &rpc.Subscription{ID: rpc.NewID()}, nil, + FilterCriteria{FromBlock: big.NewInt(1), ToBlock: big.NewInt(rpc.LatestBlockNumber.Int64())}, + allLogs, newMockNotifier(), &rpc.Subscription{ID: rpc.NewID()}, nil, nil, + }, + // from 2 to latest + { + FilterCriteria{FromBlock: big.NewInt(2)}, + allLogs[1:], newMockNotifier(), &rpc.Subscription{ID: rpc.NewID()}, nil, nil, + }, + // from latest to latest + { + FilterCriteria{}, + nil, newMockNotifier(), &rpc.Subscription{ID: rpc.NewID()}, nil, nil, + }, + // from 1 to 3 + { + FilterCriteria{FromBlock: big.NewInt(1), ToBlock: big.NewInt(3)}, + nil, newMockNotifier(), &rpc.Subscription{ID: rpc.NewID()}, errInvalidToBlock, nil, + }, + // from -101 to latest + { + FilterCriteria{FromBlock: big.NewInt(-101)}, + nil, newMockNotifier(), &rpc.Subscription{ID: rpc.NewID()}, errInvalidFromBlock, nil, }, - // // from 1 to 3 - // { - // FilterCriteria{FromBlock: big.NewInt(1), ToBlock: big.NewInt(3)}, - // allLogs[0:3], newMockNotifier(), &rpc.Subscription{ID: rpc.NewID()}, nil, - // }, } // subscribe logs for i, tc := range testCases { testCases[i].err = make(chan error) err := api.logs(context.Background(), tc.notifier, tc.sub, tc.crit) + if tc.expectErr != nil { + if err == nil { + t.Errorf("test %d: want error %v, have nothing", i, tc.expectErr) + continue + } + if !errors.Is(err, tc.expectErr) { + t.Errorf("test %d: error mismatch, want %v, have %v", i, tc.expectErr, err) + } + continue + } if err != nil { t.Fatalf("SubscribeLogs %d failed: %v\n", i, err) } From 7f3b72fe10cae4ed70020516f211b1419d049fcb Mon Sep 17 00:00:00 2001 From: jsvisa Date: Tue, 27 Jun 2023 01:04:09 +0800 Subject: [PATCH 22/70] eth/filters: subscribe pending logs Signed-off-by: jsvisa --- eth/filters/api.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eth/filters/api.go b/eth/filters/api.go index 31252dcf04..1f82f981d7 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -284,12 +284,12 @@ func (api *FilterAPI) logs(ctx context.Context, notifier notifier, rpcSub *rpc.S to = rpc.BlockNumber(crit.ToBlock.Int64()) } - if to != rpc.LatestBlockNumber { + if to != rpc.LatestBlockNumber && to != rpc.PendingBlockNumber { return errInvalidToBlock } // do live filter only - if from == rpc.LatestBlockNumber { + if from == rpc.LatestBlockNumber || from == rpc.PendingBlockNumber { return api.liveLogs(ctx, notifier, rpcSub, crit) } From 69a14cb0aeb4e8023d89df557318a8affe75dbfe Mon Sep 17 00:00:00 2001 From: jsvisa Date: Tue, 27 Jun 2023 01:04:26 +0800 Subject: [PATCH 23/70] eth/filters: testing pending with latest Signed-off-by: jsvisa --- eth/filters/filter_system.go | 23 ++++++++++++++++++++++- eth/filters/filter_system_test.go | 15 +++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/eth/filters/filter_system.go b/eth/filters/filter_system.go index 2ce12f2e94..c5f8318826 100644 --- a/eth/filters/filter_system.go +++ b/eth/filters/filter_system.go @@ -39,6 +39,10 @@ import ( "github.com/ethereum/go-ethereum/rpc" ) +var ( + errInvalidFromTo = errors.New("invalid from and to block combination: from > to") +) + // Config represents the configuration of the filter system. type Config struct { LogCacheSize int // maximum number of cached blocks (default: 32) @@ -335,7 +339,24 @@ func (es *EventSystem) SubscribeLogs(crit ethereum.FilterQuery, logs chan []*typ if from >= 0 && to == rpc.LatestBlockNumber { return es.subscribeLogs(crit, logs), nil } - return nil, errInvalidBlockRange + return nil, errInvalidFromTo +} + +// subscribeMinedPendingLogs creates a subscription that returned mined and +// pending logs that match the given criteria. +func (es *EventSystem) subscribeMinedPendingLogs(crit ethereum.FilterQuery, logs chan []*types.Log) *Subscription { + sub := &subscription{ + id: rpc.NewID(), + typ: MinedAndPendingLogsSubscription, + logsCrit: crit, + created: time.Now(), + logs: logs, + txs: make(chan []*types.Transaction), + headers: make(chan *types.Header), + installed: make(chan struct{}), + err: make(chan error), + } + return es.subscribe(sub) } // subscribeLogs creates a subscription that will write all logs matching the diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index eb6746aed2..1d15c7dd98 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -809,6 +809,21 @@ func TestLogsSubscription(t *testing.T) { FilterCriteria{}, nil, newMockNotifier(), &rpc.Subscription{ID: rpc.NewID()}, nil, nil, }, + // from pending to pending + { + FilterCriteria{FromBlock: big.NewInt(rpc.PendingBlockNumber.Int64()), ToBlock: big.NewInt(rpc.PendingBlockNumber.Int64())}, + nil, newMockNotifier(), &rpc.Subscription{ID: rpc.NewID()}, nil, nil, + }, + // from latest to pending + { + FilterCriteria{FromBlock: big.NewInt(rpc.LatestBlockNumber.Int64()), ToBlock: big.NewInt(rpc.PendingBlockNumber.Int64())}, + nil, newMockNotifier(), &rpc.Subscription{ID: rpc.NewID()}, nil, nil, + }, + // from pending to latest + { + FilterCriteria{FromBlock: big.NewInt(rpc.PendingBlockNumber.Int64()), ToBlock: big.NewInt(rpc.LatestBlockNumber.Int64())}, + nil, newMockNotifier(), &rpc.Subscription{ID: rpc.NewID()}, errInvalidFromTo, nil, + }, // from 1 to 3 { FilterCriteria{FromBlock: big.NewInt(1), ToBlock: big.NewInt(3)}, From 6419fa7f38ae463cc7ab9e01fe4dfccecf3b7ad5 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Thu, 29 Jun 2023 15:34:43 +0200 Subject: [PATCH 24/70] stricter range validation Signed-off-by: jsvisa --- eth/filters/api.go | 36 ++++++++++++------------------------ 1 file changed, 12 insertions(+), 24 deletions(-) diff --git a/eth/filters/api.go b/eth/filters/api.go index 1f82f981d7..5fc8aa513b 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -39,7 +39,7 @@ var ( errFilterNotFound = errors.New("filter not found") errConnectDropped = errors.New("connection dropped") errInvalidToBlock = errors.New("log subscription does not support history block range") - errInvalidFromBlock = errors.New("invalid from block") + errInvalidFromBlock = errors.New("from block can be only a number, or \"safe\", or \"finalized\"") ) // filter is a helper struct that holds meta information over the filter type @@ -267,33 +267,22 @@ func (api *FilterAPI) Logs(ctx context.Context, crit FilterCriteria) (*rpc.Subsc return rpcSub, err } -// logs is the inner implmention of logs filter -// from: nil, to: nil -> only live -// from: blockNum | safe | finalized, to: nil -> historical from from and then toggle live -// every other case should fail with an error +// logs is the inner implementation of logs filtering. +// from: nil, to: nil -> only live mode. +// from: blockNum | safe | finalized, to: nil -> historical beginning at `from` to head, then live mode. +// Every other case should fail with an error. func (api *FilterAPI) logs(ctx context.Context, notifier notifier, rpcSub *rpc.Subscription, crit FilterCriteria) error { - var from, to rpc.BlockNumber - if crit.FromBlock == nil { - from = rpc.LatestBlockNumber - } else { - from = rpc.BlockNumber(crit.FromBlock.Int64()) - } - if crit.ToBlock == nil { - to = rpc.LatestBlockNumber - } else { - to = rpc.BlockNumber(crit.ToBlock.Int64()) - } - - if to != rpc.LatestBlockNumber && to != rpc.PendingBlockNumber { + if crit.ToBlock != nil { return errInvalidToBlock } - - // do live filter only - if from == rpc.LatestBlockNumber || from == rpc.PendingBlockNumber { + if crit.FromBlock == nil { return api.liveLogs(ctx, notifier, rpcSub, crit) } - - if from == rpc.SafeBlockNumber || from == rpc.FinalizedBlockNumber { + from := rpc.BlockNumber(crit.FromBlock.Int64()) + switch from { + case rpc.LatestBlockNumber, rpc.PendingBlockNumber: + return errInvalidFromBlock + case rpc.SafeBlockNumber, rpc.FinalizedBlockNumber: header, err := api.sys.backend.HeaderByNumber(ctx, from) if err != nil { return err @@ -303,7 +292,6 @@ func (api *FilterAPI) logs(ctx context.Context, notifier notifier, rpcSub *rpc.S if from < 0 { return errInvalidFromBlock } - go func() { // do historical sync first _, err := api.histLogs(ctx, notifier, rpcSub, int64(from), crit) From 8f71da4d062e570471f79f0a537f30eaa933c142 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Thu, 29 Jun 2023 18:04:05 +0200 Subject: [PATCH 25/70] minor Signed-off-by: jsvisa --- eth/filters/api.go | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/eth/filters/api.go b/eth/filters/api.go index 5fc8aa513b..ec8cbd183f 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -276,7 +276,7 @@ func (api *FilterAPI) logs(ctx context.Context, notifier notifier, rpcSub *rpc.S return errInvalidToBlock } if crit.FromBlock == nil { - return api.liveLogs(ctx, notifier, rpcSub, crit) + return api.liveLogs(notifier, rpcSub, crit) } from := rpc.BlockNumber(crit.FromBlock.Int64()) switch from { @@ -294,18 +294,18 @@ func (api *FilterAPI) logs(ctx context.Context, notifier notifier, rpcSub *rpc.S } go func() { // do historical sync first - _, err := api.histLogs(ctx, notifier, rpcSub, int64(from), crit) + _, err := api.histLogs(notifier, rpcSub, int64(from), crit) if err != nil { return } // then subscribe from the header - api.liveLogs(ctx, notifier, rpcSub, crit) + api.liveLogs(notifier, rpcSub, crit) }() return nil } // liveLogs only retrieves live logs -func (api *FilterAPI) liveLogs(ctx context.Context, notify notifier, rpcSub *rpc.Subscription, crit FilterCriteria) error { +func (api *FilterAPI) liveLogs(notify notifier, rpcSub *rpc.Subscription, crit FilterCriteria) error { matchedLogs := make(chan []*types.Log) logsSub, err := api.events.SubscribeLogs(ethereum.FilterQuery(crit), matchedLogs) if err != nil { @@ -332,31 +332,31 @@ func (api *FilterAPI) liveLogs(ctx context.Context, notify notifier, rpcSub *rpc return nil } -// histLogs only retrieves the logs older than current header -func (api *FilterAPI) histLogs(ctx context.Context, notifier notifier, rpcSub *rpc.Subscription, n int64, crit FilterCriteria) (int64, error) { - // the ctx will be canceled after this callback(filter.Logs) is called - // and we are run in a background goroutine, use a new context instead +// histLogs retrieves the logs older than current header. +func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from int64, crit FilterCriteria) (int64, error) { + // The original request ctx will be canceled as soon as the parent goroutine + // returns a subscription. Use a new context instead. var ( - cctx = context.Background() + ctx = context.Background() rmLogsCh = make(chan []*types.Log) ) for { - // get the latest block header + // Get the latest block header. header := api.sys.backend.CurrentHeader() if header == nil { - return 0, errors.New("header is null") + return 0, errors.New("unexpected error: no header block found") } head := header.Number.Int64() - if n >= head { + if from >= head { return head, nil } - // do historical sync from n to head - f := api.sys.NewRangeFilter(n, head, crit.Addresses, crit.Topics) - logChan, errChan := f.rangeLogsAsync(cctx) + // Do historical sync beginning at `from` to head. + f := api.sys.NewRangeFilter(from, head, crit.Addresses, crit.Topics) + logChan, errChan := f.rangeLogsAsync(ctx) // subscribe rmLogs - query := ethereum.FilterQuery{FromBlock: big.NewInt(n), ToBlock: big.NewInt(head), Addresses: crit.Addresses, Topics: crit.Topics} + query := ethereum.FilterQuery{FromBlock: big.NewInt(from), ToBlock: big.NewInt(head), Addresses: crit.Addresses, Topics: crit.Topics} rmLogsSub, err := api.events.SubscribeLogs(query, rmLogsCh) if err != nil { return 0, err @@ -404,7 +404,7 @@ func (api *FilterAPI) histLogs(ctx context.Context, notifier notifier, rpcSub *r } // move forward to the next batch - n = head + 1 + from = head + 1 } } From 222c3190261caed72d99b9c23d821d6d14e03281 Mon Sep 17 00:00:00 2001 From: jsvisa Date: Tue, 4 Jul 2023 11:52:45 +0800 Subject: [PATCH 26/70] eth/filters: reorg logs Signed-off-by: jsvisa --- eth/filters/api.go | 63 +++++++++++++++++++++++++++++++++------------- 1 file changed, 46 insertions(+), 17 deletions(-) diff --git a/eth/filters/api.go b/eth/filters/api.go index ec8cbd183f..0c4a351f40 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -337,9 +337,11 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from // The original request ctx will be canceled as soon as the parent goroutine // returns a subscription. Use a new context instead. var ( - ctx = context.Background() - rmLogsCh = make(chan []*types.Log) + ctx, cancel = context.WithCancel(context.Background()) + reorgLogsCh = make(chan []*types.Log) ) + defer cancel() + for { // Get the latest block header. header := api.sys.backend.CurrentHeader() @@ -355,35 +357,62 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from f := api.sys.NewRangeFilter(from, head, crit.Addresses, crit.Topics) logChan, errChan := f.rangeLogsAsync(ctx) - // subscribe rmLogs + // Subscribe the ChainReorg logs + // if an ChainReorg occured, + // we will first recv the old chain's deleted logs in descending order, + // and then the new chain's added logs in desecnding order + // see core/blockchain.go#reorg(oldHead *types.Header, newHead *types.Block) for more details + // if an reorg happened between `from` and `to`, we will need to think about some senarios: + // 1. if a reorg occurs after the currently delivered block, then because this is happening in the future, has nothing to do with the current historical sync, we can just ignore it. + // 2. if a reorg occurs before the currently delivered block, then we need to stop the historical delivery, and send all replaced logs instead query := ethereum.FilterQuery{FromBlock: big.NewInt(from), ToBlock: big.NewInt(head), Addresses: crit.Addresses, Topics: crit.Topics} - rmLogsSub, err := api.events.SubscribeLogs(query, rmLogsCh) + reorgLogsSub, err := api.events.SubscribeLogs(query, reorgLogsCh) if err != nil { return 0, err } - var rmLogs []*types.Log + var ( + delivered uint64 + reorged bool + reorgBlock uint64 + reorgLogs []*types.Log + ) FORLOOP: for { select { case <-notifier.Closed(): - rmLogsSub.Unsubscribe() + reorgLogsSub.Unsubscribe() return 0, errConnectDropped case err := <-rpcSub.Err(): // client send an unsubscribe request - rmLogsSub.Unsubscribe() + reorgLogsSub.Unsubscribe() return 0, err case log := <-logChan: - notifier.Notify(rpcSub.ID, &log) - case logs := <-rmLogsCh: + // We transmit all data that meets the following conditions: + // 1. reorg not happened + // 2. reorg happened but the the log is in the remainder of the currently delivered block + if !reorged || log.BlockNumber < reorgBlock || log.BlockNumber <= delivered { + notifier.Notify(rpcSub.ID, &log) + delivered = log.BlockNumber + } else { + // No more rangelogs are needed, let's stop the rangeLogsAsync + cancel() + } + case logs := <-reorgLogsCh: for _, log := range logs { - log := log - if log.Removed { - rmLogs = append(rmLogs, log) + reorgLogs = append(reorgLogs, log) + // Update the reorgBlock to the last removed log's block number or the first added log's + if reorgBlock == 0 || log.Removed { + reorgBlock = log.BlockNumber } } + reorged = reorgBlock < delivered + // Reorg happens in the future + if !reorged { + reorgLogs = reorgLogs[:] + } case err := <-errChan: - // range filter is done or error, let's also stop the rmLogs subscribe - rmLogsSub.Unsubscribe() + // Range filter is done or error, let's also stop the reorgLogs subscribe + reorgLogsSub.Unsubscribe() if err != nil { return 0, err } @@ -391,8 +420,8 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from } } - // send the reorged logs - for _, log := range rmLogs { + // Send the reorged logs + for _, log := range reorgLogs { select { case <-notifier.Closed(): return head, errConnectDropped @@ -403,7 +432,7 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from } } - // move forward to the next batch + // Move forward to the next batch from = head + 1 } } From 97e0942f12f819ca8fdd1ebc1479f91b71262fc6 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Tue, 4 Jul 2023 14:37:13 +0200 Subject: [PATCH 27/70] fix tests Signed-off-by: jsvisa --- eth/filters/filter_system_test.go | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index 1d15c7dd98..3b90127830 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -797,7 +797,7 @@ func TestLogsSubscription(t *testing.T) { // from 1 to latest { FilterCriteria{FromBlock: big.NewInt(1), ToBlock: big.NewInt(rpc.LatestBlockNumber.Int64())}, - allLogs, newMockNotifier(), &rpc.Subscription{ID: rpc.NewID()}, nil, nil, + nil, newMockNotifier(), &rpc.Subscription{ID: rpc.NewID()}, errInvalidToBlock, nil, }, // from 2 to latest { @@ -809,21 +809,6 @@ func TestLogsSubscription(t *testing.T) { FilterCriteria{}, nil, newMockNotifier(), &rpc.Subscription{ID: rpc.NewID()}, nil, nil, }, - // from pending to pending - { - FilterCriteria{FromBlock: big.NewInt(rpc.PendingBlockNumber.Int64()), ToBlock: big.NewInt(rpc.PendingBlockNumber.Int64())}, - nil, newMockNotifier(), &rpc.Subscription{ID: rpc.NewID()}, nil, nil, - }, - // from latest to pending - { - FilterCriteria{FromBlock: big.NewInt(rpc.LatestBlockNumber.Int64()), ToBlock: big.NewInt(rpc.PendingBlockNumber.Int64())}, - nil, newMockNotifier(), &rpc.Subscription{ID: rpc.NewID()}, nil, nil, - }, - // from pending to latest - { - FilterCriteria{FromBlock: big.NewInt(rpc.PendingBlockNumber.Int64()), ToBlock: big.NewInt(rpc.LatestBlockNumber.Int64())}, - nil, newMockNotifier(), &rpc.Subscription{ID: rpc.NewID()}, errInvalidFromTo, nil, - }, // from 1 to 3 { FilterCriteria{FromBlock: big.NewInt(1), ToBlock: big.NewInt(3)}, From 5600bbf03e32f88e23327f5a839c12233d3532c7 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Tue, 4 Jul 2023 20:17:40 +0200 Subject: [PATCH 28/70] add a live log to the test Signed-off-by: jsvisa --- eth/filters/api.go | 1 + eth/filters/filter_system_test.go | 36 ++++++++++++++++++++----------- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/eth/filters/api.go b/eth/filters/api.go index 0c4a351f40..3086c68e9d 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -435,6 +435,7 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from // Move forward to the next batch from = head + 1 } + return 0, nil } // FilterCriteria represents a request to create a new filter. diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index 3b90127830..e3bc7fca97 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -17,7 +17,6 @@ package filters import ( - "bytes" "context" "errors" "fmt" @@ -765,9 +764,18 @@ func TestLogsSubscription(t *testing.T) { rawdb.WriteReceipts(db, block.Hash(), block.NumberU64(), receipts[i]) } + // Generate pending block, logs for which + // will be sent to subscription feed. + _, preceipts := core.GenerateChain(genesis.Config, blocks[len(blocks)-1], ethash.NewFaker(), db, 1, func(i int, gen *core.BlockGen) { + // transfer(address to, uint256 value) + data := fmt.Sprintf("0xa9059cbb%s%s", common.HexToHash(common.BigToAddress(big.NewInt(int64(i + 1))).Hex()).String()[2:], common.BytesToHash([]byte{byte(i + 11)}).String()[2:]) + tx, _ := types.SignTx(types.NewTx(&types.LegacyTx{Nonce: uint64(len(blocks) + i), To: &contract, Value: big.NewInt(0), Gas: 46000, GasPrice: gen.BaseFee(), Data: common.FromHex(data)}), signer, key) + gen.AddTx(tx) + }) + liveLogs := preceipts[0][0].Logs var ( - _, sys = newTestFilterSystem(t, db, Config{}) - api = NewFilterAPI(sys, false) + backend, sys = newTestFilterSystem(t, db, Config{}) + api = NewFilterAPI(sys, false) // Transfer(address indexed from, address indexed to, uint256 value); topic = common.HexToHash("0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef") ) @@ -775,12 +783,11 @@ func TestLogsSubscription(t *testing.T) { i2h := func(i int) common.Hash { return common.BigToHash(big.NewInt(int64(i))) } allLogs := []*types.Log{ - {Topics: []common.Hash{topic, common.HexToHash(addr.Hex()), i2h(1)}, Data: i2h(11).Bytes(), BlockNumber: 1}, - {Topics: []common.Hash{topic, common.HexToHash(addr.Hex()), i2h(2)}, Data: i2h(12).Bytes(), BlockNumber: 2}, - {Topics: []common.Hash{topic, common.HexToHash(addr.Hex()), i2h(3)}, Data: i2h(13).Bytes(), BlockNumber: 3}, - {Topics: []common.Hash{topic, common.HexToHash(addr.Hex()), i2h(4)}, Data: i2h(14).Bytes(), BlockNumber: 4}, + {Address: contract, Topics: []common.Hash{topic, common.HexToHash(addr.Hex()), i2h(1)}, Data: i2h(11).Bytes(), BlockNumber: 1, BlockHash: blocks[0].Hash(), TxHash: blocks[0].Transactions()[0].Hash()}, + {Address: contract, Topics: []common.Hash{topic, common.HexToHash(addr.Hex()), i2h(2)}, Data: i2h(12).Bytes(), BlockNumber: 2, BlockHash: blocks[1].Hash(), TxHash: blocks[1].Transactions()[0].Hash()}, + {Address: contract, Topics: []common.Hash{topic, common.HexToHash(addr.Hex()), i2h(3)}, Data: i2h(13).Bytes(), BlockNumber: 3, BlockHash: blocks[2].Hash(), TxHash: blocks[2].Transactions()[0].Hash()}, + {Address: contract, Topics: []common.Hash{topic, common.HexToHash(addr.Hex()), i2h(4)}, Data: i2h(14).Bytes(), BlockNumber: 4, BlockHash: blocks[3].Hash(), TxHash: blocks[3].Transactions()[0].Hash()}, } - testCases := []struct { crit FilterCriteria expected []*types.Log @@ -792,7 +799,7 @@ func TestLogsSubscription(t *testing.T) { // from 0 to latest { FilterCriteria{FromBlock: big.NewInt(0)}, - allLogs, newMockNotifier(), &rpc.Subscription{ID: rpc.NewID()}, nil, nil, + append(allLogs, liveLogs...), newMockNotifier(), &rpc.Subscription{ID: rpc.NewID()}, nil, nil, }, // from 1 to latest { @@ -802,12 +809,12 @@ func TestLogsSubscription(t *testing.T) { // from 2 to latest { FilterCriteria{FromBlock: big.NewInt(2)}, - allLogs[1:], newMockNotifier(), &rpc.Subscription{ID: rpc.NewID()}, nil, nil, + append(allLogs[1:], liveLogs...), newMockNotifier(), &rpc.Subscription{ID: rpc.NewID()}, nil, nil, }, // from latest to latest { FilterCriteria{}, - nil, newMockNotifier(), &rpc.Subscription{ID: rpc.NewID()}, nil, nil, + liveLogs, newMockNotifier(), &rpc.Subscription{ID: rpc.NewID()}, nil, nil, }, // from 1 to 3 { @@ -865,7 +872,7 @@ func TestLogsSubscription(t *testing.T) { for l := range fetched { have, want := fetched[l], tt.expected[l] - if have.Address != contract || len(have.Topics) != len(want.Topics) || !bytes.Equal(have.Topics[2].Bytes(), want.Topics[2].Bytes()) || !bytes.Equal(have.Data, want.Data) || have.BlockNumber != want.BlockNumber { + if !reflect.DeepEqual(have, want) { tt.err <- fmt.Errorf("invalid log on index %d for case %d have: %+v want: %+v\n", l, i, have, want) return } @@ -874,6 +881,11 @@ func TestLogsSubscription(t *testing.T) { }() } + // Wait for historical logs to be processed + time.Sleep(1 * time.Second) + // Send live logs + backend.logsFeed.Send(liveLogs) + for i := range testCases { err := <-testCases[i].err if err != nil { From 1589685a59fd5017179478419db1bc98f5978a88 Mon Sep 17 00:00:00 2001 From: jsvisa Date: Tue, 4 Jul 2023 22:46:33 +0800 Subject: [PATCH 29/70] eth/filters: shadow copy && reset reorglog Signed-off-by: jsvisa --- eth/filters/api.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eth/filters/api.go b/eth/filters/api.go index 3086c68e9d..15ad88533d 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -399,6 +399,7 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from } case logs := <-reorgLogsCh: for _, log := range logs { + log := log reorgLogs = append(reorgLogs, log) // Update the reorgBlock to the last removed log's block number or the first added log's if reorgBlock == 0 || log.Removed { @@ -408,7 +409,7 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from reorged = reorgBlock < delivered // Reorg happens in the future if !reorged { - reorgLogs = reorgLogs[:] + reorgLogs = nil } case err := <-errChan: // Range filter is done or error, let's also stop the reorgLogs subscribe From 8b65f9143c1e4f61adf6d3ae99a47e008c9139a7 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Tue, 4 Jul 2023 20:26:57 +0200 Subject: [PATCH 30/70] add comment Signed-off-by: jsvisa --- eth/filters/filter_system_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index e3bc7fca97..4c759570fc 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -881,7 +881,9 @@ func TestLogsSubscription(t *testing.T) { }() } - // Wait for historical logs to be processed + // Wait for historical logs to be processed. + // The reason we need to wait is this test is artificial + // and no new block containing the logs is mined. time.Sleep(1 * time.Second) // Send live logs backend.logsFeed.Send(liveLogs) From e27857f931d7b91f9191f56baf58d783c16ca3db Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Thu, 6 Jul 2023 18:25:56 +0200 Subject: [PATCH 31/70] add test case with reorg Signed-off-by: jsvisa --- eth/filters/filter_system_test.go | 237 +++++++++++++++++++++++++----- 1 file changed, 199 insertions(+), 38 deletions(-) diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index 4c759570fc..aa2eb0e96e 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -32,12 +32,14 @@ import ( "github.com/ethereum/go-ethereum/core/filtermaps" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/trie" ) type testBackend struct { @@ -169,34 +171,20 @@ func (b *testBackend) NewMatcherBackend() filtermaps.MatcherBackend { return b.fm.NewMatcherBackend() } -func (b *testBackend) startFilterMaps(history uint64, disabled bool, params filtermaps.Params) { - head := b.CurrentBlock() - chainView := filtermaps.NewChainView(b, head.Number.Uint64(), head.Hash()) - config := filtermaps.Config{ - History: history, - Disabled: disabled, - ExportFileName: "", - } - b.fm, _ = filtermaps.NewFilterMaps(b.db, chainView, 0, 0, params, config) - b.fm.Start() - b.fm.WaitIdle() +func (b *testBackend) forwardLogEvents(logCh chan []*types.Log, removedLogCh chan core.RemovedLogsEvent) { + go func() { + for { + select { + case logs := <-logCh: + b.logsFeed.Send(logs) + case logs := <-removedLogCh: + b.rmLogsFeed.Send(logs) + } + } + }() } -func (b *testBackend) stopFilterMaps() { - b.fm.Stop() - b.fm = nil -} - -func (b *testBackend) setPending(block *types.Block, receipts types.Receipts) { - b.pendingBlock = block - b.pendingReceipts = receipts -} - -func (b *testBackend) HistoryPruningCutoff() uint64 { - return 0 -} - -func newTestFilterSystem(db ethdb.Database, cfg Config) (*testBackend, *FilterSystem) { +func newTestFilterSystem(t testing.TB, db ethdb.Database, cfg Config) (*testBackend, *FilterSystem) { backend := &testBackend{db: db} sys := NewFilterSystem(backend, cfg) return backend, sys @@ -727,6 +715,7 @@ func TestLogsSubscription(t *testing.T) { t.Parallel() var ( + db = rawdb.NewMemoryDatabase() signer = types.HomesteadSigner{} key, _ = crypto.GenerateKey() addr = crypto.PubkeyToAddress(key.PublicKey) @@ -748,20 +737,27 @@ func TestLogsSubscription(t *testing.T) { addr: {Balance: big.NewInt(params.Ether)}, }, } - - db, blocks, receipts = core.GenerateChainWithGenesis(genesis, ethash.NewFaker(), 4, func(i int, b *core.BlockGen) { - // transfer(address to, uint256 value) - data := fmt.Sprintf("0xa9059cbb%s%s", common.HexToHash(common.BigToAddress(big.NewInt(int64(i + 1))).Hex()).String()[2:], common.BytesToHash([]byte{byte(i + 11)}).String()[2:]) - tx, _ := types.SignTx(types.NewTx(&types.LegacyTx{Nonce: uint64(i), To: &contract, Value: big.NewInt(0), Gas: 46000, GasPrice: b.BaseFee(), Data: common.FromHex(data)}), signer, key) - b.AddTx(tx) - }) ) - for i, block := range blocks { - rawdb.WriteBlock(db, block) - rawdb.WriteCanonicalHash(db, block.Hash(), block.NumberU64()) - rawdb.WriteHeadBlockHash(db, block.Hash()) - rawdb.WriteReceipts(db, block.Hash(), block.NumberU64(), receipts[i]) + // Hack: GenerateChainWithGenesis creates a new db. + // Commit the genesis manually and use GenerateChain. + _, err := genesis.Commit(db, trie.NewDatabase(db)) + if err != nil { + t.Fatal(err) + } + blocks, _ := core.GenerateChain(genesis.Config, genesis.ToBlock(), ethash.NewFaker(), db, 4, func(i int, b *core.BlockGen) { + // transfer(address to, uint256 value) + data := fmt.Sprintf("0xa9059cbb%s%s", common.HexToHash(common.BigToAddress(big.NewInt(int64(i + 1))).Hex()).String()[2:], common.BytesToHash([]byte{byte(i + 11)}).String()[2:]) + tx, _ := types.SignTx(types.NewTx(&types.LegacyTx{Nonce: uint64(i), To: &contract, Value: big.NewInt(0), Gas: 46000, GasPrice: b.BaseFee(), Data: common.FromHex(data)}), signer, key) + b.AddTx(tx) + }) + bc, err := core.NewBlockChain(db, nil, genesis, nil, ethash.NewFaker(), vm.Config{}, nil, new(uint64)) + if err != nil { + t.Fatal(err) + } + _, err = bc.InsertChain(blocks) + if err != nil { + t.Fatal(err) } // Generate pending block, logs for which @@ -895,3 +891,168 @@ func TestLogsSubscription(t *testing.T) { } } } + +// TestLogsSubscription tests if a rpc subscription receives the correct logs +func TestLogsSubscriptionReorg(t *testing.T) { + t.Parallel() + + var ( + db = rawdb.NewMemoryDatabase() + backend, sys = newTestFilterSystem(t, db, Config{}) + api = NewFilterAPI(sys, false) + signer = types.HomesteadSigner{} + key, _ = crypto.GenerateKey() + addr = crypto.PubkeyToAddress(key.PublicKey) + contract = common.HexToAddress("0000000000000000000000000000000000031ec7") + // Transfer(address indexed from, address indexed to, uint256 value); + topic = common.HexToHash("0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef") + genesis = &core.Genesis{ + Config: params.TestChainConfig, + Alloc: core.GenesisAlloc{ + // // SPDX-License-Identifier: GPL-3.0 + // pragma solidity >=0.7.0 <0.9.0; + // + // contract Token { + // event Transfer(address indexed from, address indexed to, uint256 value); + // function transfer(address to, uint256 value) public returns (bool) { + // emit Transfer(msg.sender, to, value); + // return true; + // } + // } + contract: {Balance: big.NewInt(params.Ether), Code: common.FromHex("0x608060405234801561001057600080fd5b506004361061002b5760003560e01c8063a9059cbb14610030575b600080fd5b61004a6004803603810190610045919061016a565b610060565b60405161005791906101c5565b60405180910390f35b60008273ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff167fddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef846040516100bf91906101ef565b60405180910390a36001905092915050565b600080fd5b600073ffffffffffffffffffffffffffffffffffffffff82169050919050565b6000610101826100d6565b9050919050565b610111816100f6565b811461011c57600080fd5b50565b60008135905061012e81610108565b92915050565b6000819050919050565b61014781610134565b811461015257600080fd5b50565b6000813590506101648161013e565b92915050565b60008060408385031215610181576101806100d1565b5b600061018f8582860161011f565b92505060206101a085828601610155565b9150509250929050565b60008115159050919050565b6101bf816101aa565b82525050565b60006020820190506101da60008301846101b6565b92915050565b6101e981610134565b82525050565b600060208201905061020460008301846101e0565b9291505056fea2646970667358221220b469033f4b77b9565ee84e0a2f04d496b18160d26034d54f9487e57788fd36d564736f6c63430008120033")}, + addr: {Balance: big.NewInt(params.Ether)}, + }, + } + ) + + // Hack: GenerateChainWithGenesis creates a new db. + // Commit the genesis manually and use GenerateChain. + _, err := genesis.Commit(db, trie.NewDatabase(db)) + if err != nil { + t.Fatal(err) + } + blocks, _ := core.GenerateChain(genesis.Config, genesis.ToBlock(), ethash.NewFaker(), db, 5, func(i int, b *core.BlockGen) { + // transfer(address to, uint256 value) + data := fmt.Sprintf("0xa9059cbb%s%s", common.HexToHash(common.BigToAddress(big.NewInt(int64(i + 1))).Hex()).String()[2:], common.BytesToHash([]byte{byte(i + 11)}).String()[2:]) + tx, _ := types.SignTx(types.NewTx(&types.LegacyTx{Nonce: uint64(i), To: &contract, Value: big.NewInt(0), Gas: 46000, GasPrice: b.BaseFee(), Data: common.FromHex(data)}), signer, key) + b.AddTx(tx) + }) + bc, err := core.NewBlockChain(db, nil, genesis, nil, ethash.NewFaker(), vm.Config{}, nil, new(uint64)) + if err != nil { + t.Fatal(err) + } + // Hack: FilterSystem is using mock backend, i.e. blockchain events will be not received + // by it. Forward them manually. + var ( + bcLogsFeed = make(chan []*types.Log) + bcRmLogsFeed = make(chan core.RemovedLogsEvent) + ) + backend.forwardLogEvents(bcLogsFeed, bcRmLogsFeed) + bc.SubscribeLogsEvent(bcLogsFeed) + bc.SubscribeRemovedLogsEvent(bcRmLogsFeed) + + _, err = bc.InsertChain(blocks) + if err != nil { + t.Fatal(err) + } + reorgChain, _ := core.GenerateChain(genesis.Config, blocks[1], ethash.NewFaker(), db, 4, func(i int, b *core.BlockGen) { + // transfer(address to, uint256 value) + data := fmt.Sprintf("0xa9059cbb%s%s", common.HexToHash(common.BigToAddress(big.NewInt(int64(i + 1))).Hex()).String()[2:], common.BytesToHash([]byte{byte(i + 1000)}).String()[2:]) + tx, _ := types.SignTx(types.NewTx(&types.LegacyTx{Nonce: uint64(2 + i), To: &contract, Value: big.NewInt(0), Gas: 46000, GasPrice: b.BaseFee(), Data: common.FromHex(data)}), signer, key) + b.AddTx(tx) + }) + + // Generate pending block, logs for which + // will be sent to subscription feed. + _, preceipts := core.GenerateChain(genesis.Config, blocks[len(blocks)-1], ethash.NewFaker(), db, 1, func(i int, gen *core.BlockGen) { + // transfer(address to, uint256 value) + data := fmt.Sprintf("0xa9059cbb%s%s", common.HexToHash(common.BigToAddress(big.NewInt(int64(i + 1))).Hex()).String()[2:], common.BytesToHash([]byte{byte(i + 11)}).String()[2:]) + tx, _ := types.SignTx(types.NewTx(&types.LegacyTx{Nonce: uint64(len(blocks) + i), To: &contract, Value: big.NewInt(0), Gas: 46000, GasPrice: gen.BaseFee(), Data: common.FromHex(data)}), signer, key) + gen.AddTx(tx) + }) + liveLogs := preceipts[0][0].Logs + + i2h := func(i int) common.Hash { return common.BigToHash(big.NewInt(int64(i))) } + expected := []*types.Log{ + // Original chain until block 3 + {Address: contract, Topics: []common.Hash{topic, common.HexToHash(addr.Hex()), i2h(1)}, Data: i2h(11).Bytes(), BlockNumber: 1, BlockHash: blocks[0].Hash(), TxHash: blocks[0].Transactions()[0].Hash()}, + {Address: contract, Topics: []common.Hash{topic, common.HexToHash(addr.Hex()), i2h(2)}, Data: i2h(12).Bytes(), BlockNumber: 2, BlockHash: blocks[1].Hash(), TxHash: blocks[1].Transactions()[0].Hash()}, + {Address: contract, Topics: []common.Hash{topic, common.HexToHash(addr.Hex()), i2h(3)}, Data: i2h(13).Bytes(), BlockNumber: 3, BlockHash: blocks[2].Hash(), TxHash: blocks[2].Transactions()[0].Hash()}, + // Removed log for block 3 + {Address: contract, Topics: []common.Hash{topic, common.HexToHash(addr.Hex()), i2h(3)}, Data: i2h(13).Bytes(), BlockNumber: 3, BlockHash: blocks[2].Hash(), TxHash: blocks[2].Transactions()[0].Hash(), Removed: true}, + // New logs for 3 onwards + {Address: contract, Topics: []common.Hash{topic, common.HexToHash(addr.Hex()), i2h(3)}, Data: i2h(1003).Bytes(), BlockNumber: 3, BlockHash: reorgChain[0].Hash(), TxHash: reorgChain[0].Transactions()[0].Hash()}, + {Address: contract, Topics: []common.Hash{topic, common.HexToHash(addr.Hex()), i2h(4)}, Data: i2h(1004).Bytes(), BlockNumber: 4, BlockHash: reorgChain[1].Hash(), TxHash: reorgChain[1].Transactions()[0].Hash()}, + {Address: contract, Topics: []common.Hash{topic, common.HexToHash(addr.Hex()), i2h(5)}, Data: i2h(1005).Bytes(), BlockNumber: 5, BlockHash: reorgChain[2].Hash(), TxHash: reorgChain[2].Transactions()[0].Hash()}, + } + expected = append(expected, liveLogs...) + + // Subscribe to logs + var ( + errc = make(chan error) + notifier = newMockNotifier() + sub = &rpc.Subscription{ID: rpc.NewID()} + crit = FilterCriteria{FromBlock: big.NewInt(1)} + reorgSignal = make(chan struct{}) + ) + err = api.logs(context.Background(), notifier, sub, crit) + if err != nil { + t.Fatal(err) + } + + go func() { + var fetched []*types.Log + + timeout := time.After(3 * time.Second) + fetchLoop: + for { + select { + case log := <-notifier.c: + l := *log.(**types.Log) + fetched = append(fetched, l) + // We halt the sender by blocking Notify(). However sender will already prepare + // logs for next block and send as soon as Notify is released. So we do reorg + // one block earlier than we intend. + if l.BlockNumber == 2 { + // signal reorg + reorgSignal <- struct{}{} + // wait for reorg to happen + <-reorgSignal + } + case <-timeout: + break fetchLoop + } + } + + if len(fetched) != len(expected) { + errc <- fmt.Errorf("invalid number of logs, want %d log(s), got %d", len(expected), len(fetched)) + return + } + + for l := range fetched { + have, want := fetched[l], expected[l] + if !reflect.DeepEqual(have, want) { + errc <- fmt.Errorf("invalid log on index %d have: %+v want: %+v\n", l, have, want) + return + } + } + errc <- nil + }() + <-reorgSignal + if n, err := bc.InsertChain(reorgChain); err != nil { + t.Fatalf("failed to insert forked chain at %d: %v", n, err) + } + reorgSignal <- struct{}{} + + // Wait for historical logs to be processed. + // The reason we need to wait is this test is artificial + // and no new block containing the logs is mined. + time.Sleep(1 * time.Second) + // Send live logs + backend.logsFeed.Send(liveLogs) + + err = <-errc + if err != nil { + t.Fatalf("test failed: %v", err) + } +} From 87d8a67b4df273f848a75b40bb5d610ab5b5268f Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Thu, 6 Jul 2023 19:23:19 +0200 Subject: [PATCH 32/70] fix and simplify reorged logs handling Signed-off-by: jsvisa --- eth/filters/api.go | 67 ++++++++++++++++++++++------------------------ 1 file changed, 32 insertions(+), 35 deletions(-) diff --git a/eth/filters/api.go b/eth/filters/api.go index 15ad88533d..04debf449c 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -31,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/core/history" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/internal/ethapi" + logger "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rpc" ) @@ -358,11 +359,11 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from logChan, errChan := f.rangeLogsAsync(ctx) // Subscribe the ChainReorg logs - // if an ChainReorg occured, + // if an ChainReorg occurred, // we will first recv the old chain's deleted logs in descending order, - // and then the new chain's added logs in desecnding order + // and then the new chain's added logs in descending order // see core/blockchain.go#reorg(oldHead *types.Header, newHead *types.Block) for more details - // if an reorg happened between `from` and `to`, we will need to think about some senarios: + // if an reorg happened between `from` and `to`, we will need to think about some scenarios: // 1. if a reorg occurs after the currently delivered block, then because this is happening in the future, has nothing to do with the current historical sync, we can just ignore it. // 2. if a reorg occurs before the currently delivered block, then we need to stop the historical delivery, and send all replaced logs instead query := ethereum.FilterQuery{FromBlock: big.NewInt(from), ToBlock: big.NewInt(head), Addresses: crit.Addresses, Topics: crit.Topics} @@ -375,7 +376,6 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from delivered uint64 reorged bool reorgBlock uint64 - reorgLogs []*types.Log ) FORLOOP: for { @@ -387,29 +387,39 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from reorgLogsSub.Unsubscribe() return 0, err case log := <-logChan: - // We transmit all data that meets the following conditions: - // 1. reorg not happened - // 2. reorg happened but the the log is in the remainder of the currently delivered block - if !reorged || log.BlockNumber < reorgBlock || log.BlockNumber <= delivered { - notifier.Notify(rpcSub.ID, &log) - delivered = log.BlockNumber - } else { - // No more rangelogs are needed, let's stop the rangeLogsAsync + // TODO: deliver all logs of the current block when we detect a reorg + if reorged { + logger.Debug("reorged, dropping log from old chain", "log", log) cancel() + continue } + delivered = log.BlockNumber + notifier.Notify(rpcSub.ID, &log) case logs := <-reorgLogsCh: - for _, log := range logs { - log := log - reorgLogs = append(reorgLogs, log) - // Update the reorgBlock to the last removed log's block number or the first added log's - if reorgBlock == 0 || log.Removed { - reorgBlock = log.BlockNumber - } + if len(logs) == 0 { + continue + } + if reorgBlock == 0 { + reorgBlock = logs[0].BlockNumber + reorged = reorgBlock <= delivered } - reorged = reorgBlock < delivered - // Reorg happens in the future if !reorged { - reorgLogs = nil + continue + } + if logs[0].Removed { + // Send removed logs notification up until the point + // we have delivered logs from old chain. + for _, log := range logs { + if log.BlockNumber <= delivered { + notifier.Notify(rpcSub.ID, &log) + } + } + } else { + // New logs are emitted for the whole new chain since the reorg block. + // Send them all. + for _, log := range logs { + notifier.Notify(rpcSub.ID, &log) + } } case err := <-errChan: // Range filter is done or error, let's also stop the reorgLogs subscribe @@ -421,22 +431,9 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from } } - // Send the reorged logs - for _, log := range reorgLogs { - select { - case <-notifier.Closed(): - return head, errConnectDropped - case err := <-rpcSub.Err(): // client send an unsubscribe request - return head, err - default: - notifier.Notify(rpcSub.ID, &log) - } - } - // Move forward to the next batch from = head + 1 } - return 0, nil } // FilterCriteria represents a request to create a new filter. From c1f986b8d330455a9e62549884f11e556efd9630 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Fri, 7 Jul 2023 18:37:53 +0200 Subject: [PATCH 33/70] minor fix Signed-off-by: jsvisa --- eth/filters/api.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eth/filters/api.go b/eth/filters/api.go index 04debf449c..e93ae811b1 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -411,6 +411,7 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from // we have delivered logs from old chain. for _, log := range logs { if log.BlockNumber <= delivered { + log := log notifier.Notify(rpcSub.ID, &log) } } @@ -418,6 +419,7 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from // New logs are emitted for the whole new chain since the reorg block. // Send them all. for _, log := range logs { + log := log notifier.Notify(rpcSub.ID, &log) } } From d7bb05a4994652e7acbe9c2b3c55c397b678e601 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Fri, 7 Jul 2023 18:50:49 +0200 Subject: [PATCH 34/70] fix live log in test Signed-off-by: jsvisa --- eth/filters/filter_system_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index aa2eb0e96e..88e275a875 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -964,10 +964,10 @@ func TestLogsSubscriptionReorg(t *testing.T) { // Generate pending block, logs for which // will be sent to subscription feed. - _, preceipts := core.GenerateChain(genesis.Config, blocks[len(blocks)-1], ethash.NewFaker(), db, 1, func(i int, gen *core.BlockGen) { + _, preceipts := core.GenerateChain(genesis.Config, reorgChain[len(reorgChain)-1], ethash.NewFaker(), db, 1, func(i int, gen *core.BlockGen) { // transfer(address to, uint256 value) data := fmt.Sprintf("0xa9059cbb%s%s", common.HexToHash(common.BigToAddress(big.NewInt(int64(i + 1))).Hex()).String()[2:], common.BytesToHash([]byte{byte(i + 11)}).String()[2:]) - tx, _ := types.SignTx(types.NewTx(&types.LegacyTx{Nonce: uint64(len(blocks) + i), To: &contract, Value: big.NewInt(0), Gas: 46000, GasPrice: gen.BaseFee(), Data: common.FromHex(data)}), signer, key) + tx, _ := types.SignTx(types.NewTx(&types.LegacyTx{Nonce: uint64(6 + i), To: &contract, Value: big.NewInt(0), Gas: 46000, GasPrice: gen.BaseFee(), Data: common.FromHex(data)}), signer, key) gen.AddTx(tx) }) liveLogs := preceipts[0][0].Logs @@ -1047,7 +1047,7 @@ func TestLogsSubscriptionReorg(t *testing.T) { // Wait for historical logs to be processed. // The reason we need to wait is this test is artificial // and no new block containing the logs is mined. - time.Sleep(1 * time.Second) + time.Sleep(2 * time.Second) // Send live logs backend.logsFeed.Send(liveLogs) From c68d512e972715d836ac4a9f5422f91268a79898 Mon Sep 17 00:00:00 2001 From: jsvisa Date: Mon, 24 Jul 2023 16:58:02 +0800 Subject: [PATCH 35/70] eth/filters: live && hist coworking Signed-off-by: jsvisa --- eth/filters/api.go | 164 +++++++++++++++++++++++++++------------------ 1 file changed, 99 insertions(+), 65 deletions(-) diff --git a/eth/filters/api.go b/eth/filters/api.go index e93ae811b1..fd608719b3 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -23,6 +23,7 @@ import ( "fmt" "math/big" "sync" + "sync/atomic" "time" "github.com/ethereum/go-ethereum" @@ -293,16 +294,7 @@ func (api *FilterAPI) logs(ctx context.Context, notifier notifier, rpcSub *rpc.S if from < 0 { return errInvalidFromBlock } - go func() { - // do historical sync first - _, err := api.histLogs(notifier, rpcSub, int64(from), crit) - if err != nil { - return - } - // then subscribe from the header - api.liveLogs(notifier, rpcSub, crit) - }() - return nil + return api.histLogs(notifier, rpcSub, int64(from), crit) } // liveLogs only retrieves live logs @@ -334,76 +326,63 @@ func (api *FilterAPI) liveLogs(notify notifier, rpcSub *rpc.Subscription, crit F } // histLogs retrieves the logs older than current header. -func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from int64, crit FilterCriteria) (int64, error) { - // The original request ctx will be canceled as soon as the parent goroutine - // returns a subscription. Use a new context instead. +func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from int64, crit FilterCriteria) error { + // Subscribe the Live logs + // if an ChainReorg occurred, + // we will first recv the old chain's deleted logs in descending order, + // and then the new chain's added logs in descending order + // see core/blockchain.go#reorg(oldHead *types.Header, newHead *types.Block) for more details + // if an reorg happened between `from` and `to`, we will need to think about some scenarios: + // 1. if a reorg occurs after the currently delivered block, then because this is happening in the future, has nothing to do with the current historical sync, we can just ignore it. + // 2. if a reorg occurs before the currently delivered block, then we need to stop the historical delivery, and send all replaced logs instead var ( - ctx, cancel = context.WithCancel(context.Background()) - reorgLogsCh = make(chan []*types.Log) + liveLogs = make(chan []*types.Log) + histLogs = make(chan []*types.Log) + histErr = make(chan error) + reorged atomic.Bool ) - defer cancel() + liveLogsSub, err := api.events.SubscribeLogs(ethereum.FilterQuery(crit), liveLogs) + if err != nil { + return err + } - for { - // Get the latest block header. - header := api.sys.backend.CurrentHeader() - if header == nil { - return 0, errors.New("unexpected error: no header block found") - } - head := header.Number.Int64() - if from >= head { - return head, nil - } - - // Do historical sync beginning at `from` to head. - f := api.sys.NewRangeFilter(from, head, crit.Addresses, crit.Topics) - logChan, errChan := f.rangeLogsAsync(ctx) - - // Subscribe the ChainReorg logs - // if an ChainReorg occurred, - // we will first recv the old chain's deleted logs in descending order, - // and then the new chain's added logs in descending order - // see core/blockchain.go#reorg(oldHead *types.Header, newHead *types.Block) for more details - // if an reorg happened between `from` and `to`, we will need to think about some scenarios: - // 1. if a reorg occurs after the currently delivered block, then because this is happening in the future, has nothing to do with the current historical sync, we can just ignore it. - // 2. if a reorg occurs before the currently delivered block, then we need to stop the historical delivery, and send all replaced logs instead - query := ethereum.FilterQuery{FromBlock: big.NewInt(from), ToBlock: big.NewInt(head), Addresses: crit.Addresses, Topics: crit.Topics} - reorgLogsSub, err := api.events.SubscribeLogs(query, reorgLogsCh) - if err != nil { - return 0, err - } + go func() { + histErr <- api.doHistLogs(from, crit, &reorged, histLogs) + }() + // Compose and notify the logs from liveLogs and histLogs + go func() { var ( delivered uint64 - reorged bool reorgBlock uint64 + liveOnly bool ) - FORLOOP: for { select { - case <-notifier.Closed(): - reorgLogsSub.Unsubscribe() - return 0, errConnectDropped - case err := <-rpcSub.Err(): // client send an unsubscribe request - reorgLogsSub.Unsubscribe() - return 0, err - case log := <-logChan: - // TODO: deliver all logs of the current block when we detect a reorg - if reorged { - logger.Debug("reorged, dropping log from old chain", "log", log) - cancel() + case err := <-histErr: + if err != nil { + liveLogsSub.Unsubscribe() + return + } + // else historical logs are all delivered, let's switch to live mode + liveOnly = true + histLogs = nil + case logs := <-liveLogs: + if len(logs) == 0 { continue } - delivered = log.BlockNumber - notifier.Notify(rpcSub.ID, &log) - case logs := <-reorgLogsCh: - if len(logs) == 0 { + if liveOnly { + for _, log := range logs { + log := log + notifier.Notify(rpcSub.ID, &log) + } continue } if reorgBlock == 0 { reorgBlock = logs[0].BlockNumber - reorged = reorgBlock <= delivered + reorged.Store(reorgBlock <= delivered) } - if !reorged { + if !reorged.Load() { continue } if logs[0].Removed { @@ -423,11 +402,66 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from notifier.Notify(rpcSub.ID, &log) } } + case logs := <-histLogs: + for _, log := range logs { + log := log + delivered = log.BlockNumber + notifier.Notify(rpcSub.ID, &log) + } + case <-rpcSub.Err(): // client send an unsubscribe request + liveLogsSub.Unsubscribe() + return + case <-notifier.Closed(): // connection dropped + liveLogsSub.Unsubscribe() + return + } + } + }() + + return nil +} + +// doHistLogs retrieves the logs older than current header. +func (api *FilterAPI) doHistLogs(from int64, crit FilterCriteria, reorged *atomic.Bool, histLogs chan<- []*types.Log) error { + // The original request ctx will be canceled as soon as the parent goroutine + // returns a subscription. Use a new context instead. + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + for { + // Get the latest block header. + header := api.sys.backend.CurrentHeader() + if header == nil { + return errors.New("unexpected error: no header block found") + } + head := header.Number.Int64() + if from >= head { + return nil + } + + // Do historical sync beginning at `from` to head. + f := api.sys.NewRangeFilter(from, head, crit.Addresses, crit.Topics) + logsCh, errChan := f.rangeLogsAsync(ctx) + + FORLOOP: + for { + select { + case log := <-logsCh: + // TODO: deliver all logs of the current block when we detect a reorg + if reorged.Load() { + logger.Debug("reorged, dropping log from old chain", "log", log) + cancel() + continue + } + histLogs <- []*types.Log{log} case err := <-errChan: // Range filter is done or error, let's also stop the reorgLogs subscribe - reorgLogsSub.Unsubscribe() if err != nil { - return 0, err + if err != context.Canceled { + logger.Error("error while fetching historical logs", "err", err) + return err + } + return nil } break FORLOOP } From 98c1342898db36cff07c6172529bdff55773e371 Mon Sep 17 00:00:00 2001 From: jsvisa Date: Tue, 25 Jul 2023 10:37:09 +0800 Subject: [PATCH 36/70] eth/filters: pass reorg filter Signed-off-by: jsvisa --- eth/filters/filter_system_test.go | 98 +++++++++++++++++++++++-------- 1 file changed, 75 insertions(+), 23 deletions(-) diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index 88e275a875..ec04495023 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -37,6 +37,7 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/internal/ethapi" + logger "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/trie" @@ -892,7 +893,7 @@ func TestLogsSubscription(t *testing.T) { } } -// TestLogsSubscription tests if a rpc subscription receives the correct logs +// TestLogsSubscriptionReorg tests the behavior of the filter system when a reorganization occurs. func TestLogsSubscriptionReorg(t *testing.T) { t.Parallel() @@ -931,7 +932,7 @@ func TestLogsSubscriptionReorg(t *testing.T) { if err != nil { t.Fatal(err) } - blocks, _ := core.GenerateChain(genesis.Config, genesis.ToBlock(), ethash.NewFaker(), db, 5, func(i int, b *core.BlockGen) { + oldChain, _ := core.GenerateChain(genesis.Config, genesis.ToBlock(), ethash.NewFaker(), db, 5, func(i int, b *core.BlockGen) { // transfer(address to, uint256 value) data := fmt.Sprintf("0xa9059cbb%s%s", common.HexToHash(common.BigToAddress(big.NewInt(int64(i + 1))).Hex()).String()[2:], common.BytesToHash([]byte{byte(i + 11)}).String()[2:]) tx, _ := types.SignTx(types.NewTx(&types.LegacyTx{Nonce: uint64(i), To: &contract, Value: big.NewInt(0), Gas: 46000, GasPrice: b.BaseFee(), Data: common.FromHex(data)}), signer, key) @@ -951,42 +952,79 @@ func TestLogsSubscriptionReorg(t *testing.T) { bc.SubscribeLogsEvent(bcLogsFeed) bc.SubscribeRemovedLogsEvent(bcRmLogsFeed) - _, err = bc.InsertChain(blocks) + _, err = bc.InsertChain(oldChain) if err != nil { t.Fatal(err) } - reorgChain, _ := core.GenerateChain(genesis.Config, blocks[1], ethash.NewFaker(), db, 4, func(i int, b *core.BlockGen) { + newChain, _ := core.GenerateChain(genesis.Config, oldChain[1], ethash.NewFaker(), db, 4, func(i int, b *core.BlockGen) { // transfer(address to, uint256 value) - data := fmt.Sprintf("0xa9059cbb%s%s", common.HexToHash(common.BigToAddress(big.NewInt(int64(i + 1))).Hex()).String()[2:], common.BytesToHash([]byte{byte(i + 1000)}).String()[2:]) + data := fmt.Sprintf("0xa9059cbb%s%s", common.HexToHash(common.BigToAddress(big.NewInt(int64(i + 1))).Hex()).String()[2:], common.BytesToHash([]byte{byte(i + 103)}).String()[2:]) tx, _ := types.SignTx(types.NewTx(&types.LegacyTx{Nonce: uint64(2 + i), To: &contract, Value: big.NewInt(0), Gas: 46000, GasPrice: b.BaseFee(), Data: common.FromHex(data)}), signer, key) b.AddTx(tx) }) // Generate pending block, logs for which // will be sent to subscription feed. - _, preceipts := core.GenerateChain(genesis.Config, reorgChain[len(reorgChain)-1], ethash.NewFaker(), db, 1, func(i int, gen *core.BlockGen) { + _, preceipts := core.GenerateChain(genesis.Config, newChain[len(newChain)-1], ethash.NewFaker(), db, 1, func(i int, gen *core.BlockGen) { // transfer(address to, uint256 value) - data := fmt.Sprintf("0xa9059cbb%s%s", common.HexToHash(common.BigToAddress(big.NewInt(int64(i + 1))).Hex()).String()[2:], common.BytesToHash([]byte{byte(i + 11)}).String()[2:]) + data := fmt.Sprintf("0xa9059cbb%s%s", common.HexToHash(common.BigToAddress(big.NewInt(int64(i + 1))).Hex()).String()[2:], common.BytesToHash([]byte{byte(i + 21)}).String()[2:]) tx, _ := types.SignTx(types.NewTx(&types.LegacyTx{Nonce: uint64(6 + i), To: &contract, Value: big.NewInt(0), Gas: 46000, GasPrice: gen.BaseFee(), Data: common.FromHex(data)}), signer, key) gen.AddTx(tx) }) liveLogs := preceipts[0][0].Logs i2h := func(i int) common.Hash { return common.BigToHash(big.NewInt(int64(i))) } + a2h := func(a common.Address) common.Hash { return common.HexToHash(a.Hex()) } + c2h := func(bs []*types.Block, i int) common.Hash { return bs[i].Transactions()[0].Hash() } expected := []*types.Log{ // Original chain until block 3 - {Address: contract, Topics: []common.Hash{topic, common.HexToHash(addr.Hex()), i2h(1)}, Data: i2h(11).Bytes(), BlockNumber: 1, BlockHash: blocks[0].Hash(), TxHash: blocks[0].Transactions()[0].Hash()}, - {Address: contract, Topics: []common.Hash{topic, common.HexToHash(addr.Hex()), i2h(2)}, Data: i2h(12).Bytes(), BlockNumber: 2, BlockHash: blocks[1].Hash(), TxHash: blocks[1].Transactions()[0].Hash()}, - {Address: contract, Topics: []common.Hash{topic, common.HexToHash(addr.Hex()), i2h(3)}, Data: i2h(13).Bytes(), BlockNumber: 3, BlockHash: blocks[2].Hash(), TxHash: blocks[2].Transactions()[0].Hash()}, + {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(1)}, Data: i2h(11).Bytes(), BlockNumber: 1, BlockHash: oldChain[0].Hash(), TxHash: c2h(oldChain, 0)}, + {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(2)}, Data: i2h(12).Bytes(), BlockNumber: 2, BlockHash: oldChain[1].Hash(), TxHash: c2h(oldChain, 1)}, + {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(3)}, Data: i2h(13).Bytes(), BlockNumber: 3, BlockHash: oldChain[2].Hash(), TxHash: c2h(oldChain, 2)}, // Removed log for block 3 - {Address: contract, Topics: []common.Hash{topic, common.HexToHash(addr.Hex()), i2h(3)}, Data: i2h(13).Bytes(), BlockNumber: 3, BlockHash: blocks[2].Hash(), TxHash: blocks[2].Transactions()[0].Hash(), Removed: true}, - // New logs for 3 onwards - {Address: contract, Topics: []common.Hash{topic, common.HexToHash(addr.Hex()), i2h(3)}, Data: i2h(1003).Bytes(), BlockNumber: 3, BlockHash: reorgChain[0].Hash(), TxHash: reorgChain[0].Transactions()[0].Hash()}, - {Address: contract, Topics: []common.Hash{topic, common.HexToHash(addr.Hex()), i2h(4)}, Data: i2h(1004).Bytes(), BlockNumber: 4, BlockHash: reorgChain[1].Hash(), TxHash: reorgChain[1].Transactions()[0].Hash()}, - {Address: contract, Topics: []common.Hash{topic, common.HexToHash(addr.Hex()), i2h(5)}, Data: i2h(1005).Bytes(), BlockNumber: 5, BlockHash: reorgChain[2].Hash(), TxHash: reorgChain[2].Transactions()[0].Hash()}, + {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(3)}, Data: i2h(13).Bytes(), BlockNumber: 3, BlockHash: oldChain[2].Hash(), TxHash: c2h(oldChain, 2), Removed: true}, + // New logs for 4 onwards + {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(1)}, Data: i2h(103).Bytes(), BlockNumber: 3, BlockHash: newChain[0].Hash(), TxHash: c2h(newChain, 0)}, + {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(2)}, Data: i2h(104).Bytes(), BlockNumber: 4, BlockHash: newChain[1].Hash(), TxHash: c2h(newChain, 1)}, + {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(3)}, Data: i2h(105).Bytes(), BlockNumber: 5, BlockHash: newChain[2].Hash(), TxHash: c2h(newChain, 2)}, + {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(4)}, Data: i2h(106).Bytes(), BlockNumber: 6, BlockHash: newChain[3].Hash(), TxHash: c2h(newChain, 3)}, } expected = append(expected, liveLogs...) + // Calculate address balances + balanceDiffer := func(logs []*types.Log) map[common.Address]uint64 { + balances := make(map[common.Address]uint64) + for _, log := range logs { + log := log + from := common.BytesToAddress(log.Topics[1].Bytes()) + to := common.BytesToAddress(log.Topics[2].Bytes()) + amount := common.BytesToHash(log.Data).Big().Uint64() + + if _, ok := balances[from]; !ok { + balances[from] = 0 + } + if _, ok := balances[to]; !ok { + balances[to] = 0 + } + + if log.Removed { // revert + balances[from] += amount + balances[to] -= amount + } else { + balances[from] -= amount + balances[to] += amount + } + } + // Remove zero balances + for addr, balance := range balances { + if balance == 0 { + delete(balances, addr) + } + } + return balances + } + expectedBalance := balanceDiffer(expected) + // Subscribe to logs var ( errc = make(chan error) @@ -1024,22 +1062,36 @@ func TestLogsSubscriptionReorg(t *testing.T) { } } - if len(fetched) != len(expected) { - errc <- fmt.Errorf("invalid number of logs, want %d log(s), got %d", len(expected), len(fetched)) + fetchedBalance := balanceDiffer(fetched) + + for i, log := range fetched { + logger.Info("Flog", "i", i, "blknum", log.BlockNumber, "index", log.Index, "removed", log.Removed, "to", common.BytesToAddress(log.Topics[2].Bytes()), "amount", common.BytesToHash(log.Data).Big().Uint64()) + } + + for i, log := range expected { + logger.Info("Elog", "i", i, "blknum", log.BlockNumber, "index", log.Index, "removed", log.Removed, "to", common.BytesToAddress(log.Topics[2].Bytes()), "amount", common.BytesToHash(log.Data).Big().Uint64()) + } + if len(fetchedBalance) != len(expectedBalance) { + errc <- fmt.Errorf("invalid number of balances, have %d, want %d", len(fetchedBalance), len(expectedBalance)) + logger.Info("balance diff", "fetched", fetchedBalance, "expected", expectedBalance) return } - for l := range fetched { - have, want := fetched[l], expected[l] - if !reflect.DeepEqual(have, want) { - errc <- fmt.Errorf("invalid log on index %d have: %+v want: %+v\n", l, have, want) - return + diffBalance := make(map[common.Address]struct{ have, want uint64 }) + for addr, balance := range expectedBalance { + if fetchedBalance[addr] != balance { + diffBalance[addr] = struct{ have, want uint64 }{fetchedBalance[addr], balance} } } + if len(diffBalance) > 0 { + errc <- fmt.Errorf("invalid balance detected: %+v\n", diffBalance) + return + } + errc <- nil }() <-reorgSignal - if n, err := bc.InsertChain(reorgChain); err != nil { + if n, err := bc.InsertChain(newChain); err != nil { t.Fatalf("failed to insert forked chain at %d: %v", n, err) } reorgSignal <- struct{}{} From 5033d3931571556320602f032e5fb6b092fa2575 Mon Sep 17 00:00:00 2001 From: jsvisa Date: Tue, 25 Jul 2023 10:44:18 +0800 Subject: [PATCH 37/70] eth/filters: don't print logs Signed-off-by: jsvisa --- eth/filters/filter_system_test.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index ec04495023..893d321e06 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -1062,15 +1062,15 @@ func TestLogsSubscriptionReorg(t *testing.T) { } } + // for i, log := range fetched { + // logger.Debug("Flog", "i", i, "blknum", log.BlockNumber, "index", log.Index, "removed", log.Removed, "to", common.BytesToAddress(log.Topics[2].Bytes()), "amount", common.BytesToHash(log.Data).Big().Uint64()) + // } + // + // for i, log := range expected { + // logger.Debug("Elog", "i", i, "blknum", log.BlockNumber, "index", log.Index, "removed", log.Removed, "to", common.BytesToAddress(log.Topics[2].Bytes()), "amount", common.BytesToHash(log.Data).Big().Uint64()) + // } + fetchedBalance := balanceDiffer(fetched) - - for i, log := range fetched { - logger.Info("Flog", "i", i, "blknum", log.BlockNumber, "index", log.Index, "removed", log.Removed, "to", common.BytesToAddress(log.Topics[2].Bytes()), "amount", common.BytesToHash(log.Data).Big().Uint64()) - } - - for i, log := range expected { - logger.Info("Elog", "i", i, "blknum", log.BlockNumber, "index", log.Index, "removed", log.Removed, "to", common.BytesToAddress(log.Topics[2].Bytes()), "amount", common.BytesToHash(log.Data).Big().Uint64()) - } if len(fetchedBalance) != len(expectedBalance) { errc <- fmt.Errorf("invalid number of balances, have %d, want %d", len(fetchedBalance), len(expectedBalance)) logger.Info("balance diff", "fetched", fetchedBalance, "expected", expectedBalance) From c95bf8daf7fd326b4daf636e4595b854d6ccbe0c Mon Sep 17 00:00:00 2001 From: jsvisa Date: Tue, 25 Jul 2023 10:50:10 +0800 Subject: [PATCH 38/70] eth/filters: add reason for check with balance instead of logs Signed-off-by: jsvisa --- eth/filters/filter_system_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index 893d321e06..1d610dd539 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -992,6 +992,9 @@ func TestLogsSubscriptionReorg(t *testing.T) { expected = append(expected, liveLogs...) // Calculate address balances + // We check with the balance instead of logs because there is a gap between the ChainReorg occurred and detected, + // so we can't fully control how many logs we'll receive. + // By checking the balance, we can test with the token transfer amount. balanceDiffer := func(logs []*types.Log) map[common.Address]uint64 { balances := make(map[common.Address]uint64) for _, log := range logs { From 3b3eb5aab546bb1300468f26a7261d5f7c459987 Mon Sep 17 00:00:00 2001 From: jsvisa Date: Tue, 25 Jul 2023 11:09:43 +0800 Subject: [PATCH 39/70] eth/filters: almost working Signed-off-by: jsvisa --- eth/filters/api.go | 60 +++++++++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/eth/filters/api.go b/eth/filters/api.go index fd608719b3..fc6341804e 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -23,7 +23,6 @@ import ( "fmt" "math/big" "sync" - "sync/atomic" "time" "github.com/ethereum/go-ethereum" @@ -337,9 +336,9 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from // 2. if a reorg occurs before the currently delivered block, then we need to stop the historical delivery, and send all replaced logs instead var ( liveLogs = make(chan []*types.Log) - histLogs = make(chan []*types.Log) - histErr = make(chan error) - reorged atomic.Bool + histLogs = make(chan *types.Log) + histDone = make(chan error) + closeC = make(chan struct{}) ) liveLogsSub, err := api.events.SubscribeLogs(ethereum.FilterQuery(crit), liveLogs) if err != nil { @@ -347,7 +346,7 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from } go func() { - histErr <- api.doHistLogs(from, crit, &reorged, histLogs) + histDone <- api.doHistLogs(from, crit, histLogs, closeC) }() // Compose and notify the logs from liveLogs and histLogs @@ -356,15 +355,17 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from delivered uint64 reorgBlock uint64 liveOnly bool + reorged bool ) for { select { - case err := <-histErr: + case err := <-histDone: if err != nil { + logger.Warn("History logs delivery failed", "err", err) liveLogsSub.Unsubscribe() return } - // else historical logs are all delivered, let's switch to live mode + // Else historical logs are all delivered, let's switch to live mode liveOnly = true histLogs = nil case logs := <-liveLogs: @@ -378,11 +379,15 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from } continue } + // TODO: if reorg occured in more than once? if reorgBlock == 0 { reorgBlock = logs[0].BlockNumber - reorged.Store(reorgBlock <= delivered) + if reorgBlock <= delivered { + logger.Info("Reorg detected", "reorgBlock", reorgBlock, "delivered", delivered) + reorged = true + } } - if !reorged.Load() { + if !reorged { continue } if logs[0].Removed { @@ -402,17 +407,20 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from notifier.Notify(rpcSub.ID, &log) } } - case logs := <-histLogs: - for _, log := range logs { - log := log - delivered = log.BlockNumber - notifier.Notify(rpcSub.ID, &log) + case log := <-histLogs: + if reorged { + continue } + // TODO: deliver the same block logs + delivered = log.BlockNumber + notifier.Notify(rpcSub.ID, &log) case <-rpcSub.Err(): // client send an unsubscribe request liveLogsSub.Unsubscribe() + closeC <- struct{}{} return case <-notifier.Closed(): // connection dropped liveLogsSub.Unsubscribe() + closeC <- struct{}{} return } } @@ -421,12 +429,11 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from return nil } -// doHistLogs retrieves the logs older than current header. -func (api *FilterAPI) doHistLogs(from int64, crit FilterCriteria, reorged *atomic.Bool, histLogs chan<- []*types.Log) error { +// doHistLogs retrieves the logs older than current header, and forward them to the histLogs channel. +func (api *FilterAPI) doHistLogs(from int64, crit FilterCriteria, histLogs chan<- *types.Log, closeC chan struct{}) error { // The original request ctx will be canceled as soon as the parent goroutine // returns a subscription. Use a new context instead. - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := context.Background() for { // Get the latest block header. @@ -446,22 +453,15 @@ func (api *FilterAPI) doHistLogs(from int64, crit FilterCriteria, reorged *atomi FORLOOP: for { select { + case <-closeC: + return nil case log := <-logsCh: - // TODO: deliver all logs of the current block when we detect a reorg - if reorged.Load() { - logger.Debug("reorged, dropping log from old chain", "log", log) - cancel() - continue - } - histLogs <- []*types.Log{log} + histLogs <- log case err := <-errChan: // Range filter is done or error, let's also stop the reorgLogs subscribe if err != nil { - if err != context.Canceled { - logger.Error("error while fetching historical logs", "err", err) - return err - } - return nil + logger.Error("error while fetching historical logs", "err", err) + return err } break FORLOOP } From 43afa1e462a082136357d2d872429b20e84ba7cd Mon Sep 17 00:00:00 2001 From: jsvisa Date: Tue, 25 Jul 2023 11:17:33 +0800 Subject: [PATCH 40/70] eth/filters: deliver the same block logs if reorg occured Signed-off-by: jsvisa --- eth/filters/api.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/eth/filters/api.go b/eth/filters/api.go index fc6341804e..3365072725 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -408,12 +408,11 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from } } case log := <-histLogs: - if reorged { - continue + // If ChainReorg is detected, we need to deliver the last block's full logs + if !reorged || delivered == log.BlockNumber { + delivered = log.BlockNumber + notifier.Notify(rpcSub.ID, &log) } - // TODO: deliver the same block logs - delivered = log.BlockNumber - notifier.Notify(rpcSub.ID, &log) case <-rpcSub.Err(): // client send an unsubscribe request liveLogsSub.Unsubscribe() closeC <- struct{}{} From 3c3a183d9db13dc2178cee8c08a937d39bd26327 Mon Sep 17 00:00:00 2001 From: jsvisa Date: Tue, 25 Jul 2023 11:28:44 +0800 Subject: [PATCH 41/70] eth/filters: dry Signed-off-by: jsvisa --- eth/filters/filter_system_test.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index 1d610dd539..08174adbcf 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -932,10 +932,14 @@ func TestLogsSubscriptionReorg(t *testing.T) { if err != nil { t.Fatal(err) } - oldChain, _ := core.GenerateChain(genesis.Config, genesis.ToBlock(), ethash.NewFaker(), db, 5, func(i int, b *core.BlockGen) { + makeTx := func(to int, value int, nonce int, b *core.BlockGen) *types.Transaction { // transfer(address to, uint256 value) - data := fmt.Sprintf("0xa9059cbb%s%s", common.HexToHash(common.BigToAddress(big.NewInt(int64(i + 1))).Hex()).String()[2:], common.BytesToHash([]byte{byte(i + 11)}).String()[2:]) - tx, _ := types.SignTx(types.NewTx(&types.LegacyTx{Nonce: uint64(i), To: &contract, Value: big.NewInt(0), Gas: 46000, GasPrice: b.BaseFee(), Data: common.FromHex(data)}), signer, key) + data := fmt.Sprintf("0xa9059cbb%s%s", common.HexToHash(common.BigToAddress(big.NewInt(int64(to))).Hex()).String()[2:], common.BytesToHash([]byte{byte(value)}).String()[2:]) + tx, _ := types.SignTx(types.NewTx(&types.LegacyTx{Nonce: uint64(nonce), To: &contract, Gas: 46000, GasPrice: b.BaseFee(), Data: common.FromHex(data)}), signer, key) + return tx + } + oldChain, _ := core.GenerateChain(genesis.Config, genesis.ToBlock(), ethash.NewFaker(), db, 5, func(i int, b *core.BlockGen) { + tx := makeTx(i+1, i+11, i, b) b.AddTx(tx) }) bc, err := core.NewBlockChain(db, nil, genesis, nil, ethash.NewFaker(), vm.Config{}, nil, new(uint64)) @@ -957,19 +961,15 @@ func TestLogsSubscriptionReorg(t *testing.T) { t.Fatal(err) } newChain, _ := core.GenerateChain(genesis.Config, oldChain[1], ethash.NewFaker(), db, 4, func(i int, b *core.BlockGen) { - // transfer(address to, uint256 value) - data := fmt.Sprintf("0xa9059cbb%s%s", common.HexToHash(common.BigToAddress(big.NewInt(int64(i + 1))).Hex()).String()[2:], common.BytesToHash([]byte{byte(i + 103)}).String()[2:]) - tx, _ := types.SignTx(types.NewTx(&types.LegacyTx{Nonce: uint64(2 + i), To: &contract, Value: big.NewInt(0), Gas: 46000, GasPrice: b.BaseFee(), Data: common.FromHex(data)}), signer, key) + tx := makeTx(i+1, i+103, 2+i, b) b.AddTx(tx) }) // Generate pending block, logs for which // will be sent to subscription feed. - _, preceipts := core.GenerateChain(genesis.Config, newChain[len(newChain)-1], ethash.NewFaker(), db, 1, func(i int, gen *core.BlockGen) { - // transfer(address to, uint256 value) - data := fmt.Sprintf("0xa9059cbb%s%s", common.HexToHash(common.BigToAddress(big.NewInt(int64(i + 1))).Hex()).String()[2:], common.BytesToHash([]byte{byte(i + 21)}).String()[2:]) - tx, _ := types.SignTx(types.NewTx(&types.LegacyTx{Nonce: uint64(6 + i), To: &contract, Value: big.NewInt(0), Gas: 46000, GasPrice: gen.BaseFee(), Data: common.FromHex(data)}), signer, key) - gen.AddTx(tx) + _, preceipts := core.GenerateChain(genesis.Config, newChain[len(newChain)-1], ethash.NewFaker(), db, 1, func(i int, b *core.BlockGen) { + tx := makeTx(i+1, i+21, i+6, b) + b.AddTx(tx) }) liveLogs := preceipts[0][0].Logs From b9f695cef8cb1e22f527cd50d0b6eb50035d1c45 Mon Sep 17 00:00:00 2001 From: jsvisa Date: Tue, 25 Jul 2023 14:43:18 +0800 Subject: [PATCH 42/70] eth/filters: dry 2 Signed-off-by: jsvisa --- eth/filters/filter_system_test.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index 08174adbcf..f612244b76 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -932,14 +932,14 @@ func TestLogsSubscriptionReorg(t *testing.T) { if err != nil { t.Fatal(err) } - makeTx := func(to int, value int, nonce int, b *core.BlockGen) *types.Transaction { + makeTx := func(to int, value int, b *core.BlockGen) *types.Transaction { // transfer(address to, uint256 value) data := fmt.Sprintf("0xa9059cbb%s%s", common.HexToHash(common.BigToAddress(big.NewInt(int64(to))).Hex()).String()[2:], common.BytesToHash([]byte{byte(value)}).String()[2:]) - tx, _ := types.SignTx(types.NewTx(&types.LegacyTx{Nonce: uint64(nonce), To: &contract, Gas: 46000, GasPrice: b.BaseFee(), Data: common.FromHex(data)}), signer, key) + tx, _ := types.SignTx(types.NewTx(&types.LegacyTx{Nonce: b.TxNonce(addr), To: &contract, Gas: 46000, GasPrice: b.BaseFee(), Data: common.FromHex(data)}), signer, key) return tx } oldChain, _ := core.GenerateChain(genesis.Config, genesis.ToBlock(), ethash.NewFaker(), db, 5, func(i int, b *core.BlockGen) { - tx := makeTx(i+1, i+11, i, b) + tx := makeTx(i+1, i+11, b) b.AddTx(tx) }) bc, err := core.NewBlockChain(db, nil, genesis, nil, ethash.NewFaker(), vm.Config{}, nil, new(uint64)) @@ -961,14 +961,15 @@ func TestLogsSubscriptionReorg(t *testing.T) { t.Fatal(err) } newChain, _ := core.GenerateChain(genesis.Config, oldChain[1], ethash.NewFaker(), db, 4, func(i int, b *core.BlockGen) { - tx := makeTx(i+1, i+103, 2+i, b) + tx := makeTx(i+1, i+103, b) b.AddTx(tx) }) + logger.Info("oldChain/newChain", "oldChain", fmt.Sprintf("%d..%d", oldChain[0].Number(), oldChain[len(oldChain)-1].Number()), "newChain", fmt.Sprintf("%d..%d", newChain[0].Number(), newChain[len(newChain)-1].Number())) // Generate pending block, logs for which // will be sent to subscription feed. _, preceipts := core.GenerateChain(genesis.Config, newChain[len(newChain)-1], ethash.NewFaker(), db, 1, func(i int, b *core.BlockGen) { - tx := makeTx(i+1, i+21, i+6, b) + tx := makeTx(i+1, i+21, b) b.AddTx(tx) }) liveLogs := preceipts[0][0].Logs @@ -1050,11 +1051,12 @@ func TestLogsSubscriptionReorg(t *testing.T) { select { case log := <-notifier.c: l := *log.(**types.Log) + fetched = append(fetched, l) // We halt the sender by blocking Notify(). However sender will already prepare // logs for next block and send as soon as Notify is released. So we do reorg // one block earlier than we intend. - if l.BlockNumber == 2 { + if l.BlockNumber == 2 && l.Index == 0 { // signal reorg reorgSignal <- struct{}{} // wait for reorg to happen @@ -1066,11 +1068,11 @@ func TestLogsSubscriptionReorg(t *testing.T) { } // for i, log := range fetched { - // logger.Debug("Flog", "i", i, "blknum", log.BlockNumber, "index", log.Index, "removed", log.Removed, "to", common.BytesToAddress(log.Topics[2].Bytes()), "amount", common.BytesToHash(log.Data).Big().Uint64()) + // logger.Debug("Flog", "i", fmt.Sprintf("%02d", i), "blknum", log.BlockNumber, "index", log.Index, "removed", log.Removed, "to", common.BytesToAddress(log.Topics[2].Bytes()), "amount", common.BytesToHash(log.Data).Big().Uint64()) // } // // for i, log := range expected { - // logger.Debug("Elog", "i", i, "blknum", log.BlockNumber, "index", log.Index, "removed", log.Removed, "to", common.BytesToAddress(log.Topics[2].Bytes()), "amount", common.BytesToHash(log.Data).Big().Uint64()) + // logger.Debug("Elog", "i", fmt.Sprintf("%02d", i), "blknum", log.BlockNumber, "index", log.Index, "removed", log.Removed, "to", common.BytesToAddress(log.Topics[2].Bytes()), "amount", common.BytesToHash(log.Data).Big().Uint64()) // } fetchedBalance := balanceDiffer(fetched) From 77aa3cc6632a2b0404ad0d35e49d968a17d378cd Mon Sep 17 00:00:00 2001 From: jsvisa Date: Tue, 25 Jul 2023 15:50:43 +0800 Subject: [PATCH 43/70] eth/filters: history logs should be requested > head Signed-off-by: jsvisa --- eth/filters/api.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/eth/filters/api.go b/eth/filters/api.go index 3365072725..3d93cf3cc5 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -366,6 +366,7 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from return } // Else historical logs are all delivered, let's switch to live mode + logger.Info("History logs delivery finished, and now enter into live mode", "delivered", delivered) liveOnly = true histLogs = nil case logs := <-liveLogs: @@ -441,7 +442,8 @@ func (api *FilterAPI) doHistLogs(from int64, crit FilterCriteria, histLogs chan< return errors.New("unexpected error: no header block found") } head := header.Number.Int64() - if from >= head { + if from > head { + logger.Info("Finish historical sync", "from", from, "head", head) return nil } @@ -459,7 +461,7 @@ func (api *FilterAPI) doHistLogs(from int64, crit FilterCriteria, histLogs chan< case err := <-errChan: // Range filter is done or error, let's also stop the reorgLogs subscribe if err != nil { - logger.Error("error while fetching historical logs", "err", err) + logger.Error("Error while fetching historical logs", "err", err) return err } break FORLOOP From 7763574908aeca36ef1fd858dceb69804a138cd5 Mon Sep 17 00:00:00 2001 From: jsvisa Date: Tue, 25 Jul 2023 15:52:26 +0800 Subject: [PATCH 44/70] eth/filters: test with N logs in one block Signed-off-by: jsvisa --- eth/filters/filter_system_test.go | 33 +++++++++++++++++++------------ 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index f612244b76..9cca3f9095 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -939,8 +939,10 @@ func TestLogsSubscriptionReorg(t *testing.T) { return tx } oldChain, _ := core.GenerateChain(genesis.Config, genesis.ToBlock(), ethash.NewFaker(), db, 5, func(i int, b *core.BlockGen) { - tx := makeTx(i+1, i+11, b) - b.AddTx(tx) + for j := 0; j < 5; j++ { + tx := makeTx(i+j+1, i+11, b) + b.AddTx(tx) + } }) bc, err := core.NewBlockChain(db, nil, genesis, nil, ethash.NewFaker(), vm.Config{}, nil, new(uint64)) if err != nil { @@ -976,19 +978,24 @@ func TestLogsSubscriptionReorg(t *testing.T) { i2h := func(i int) common.Hash { return common.BigToHash(big.NewInt(int64(i))) } a2h := func(a common.Address) common.Hash { return common.HexToHash(a.Hex()) } - c2h := func(bs []*types.Block, i int) common.Hash { return bs[i].Transactions()[0].Hash() } expected := []*types.Log{ // Original chain until block 3 - {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(1)}, Data: i2h(11).Bytes(), BlockNumber: 1, BlockHash: oldChain[0].Hash(), TxHash: c2h(oldChain, 0)}, - {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(2)}, Data: i2h(12).Bytes(), BlockNumber: 2, BlockHash: oldChain[1].Hash(), TxHash: c2h(oldChain, 1)}, - {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(3)}, Data: i2h(13).Bytes(), BlockNumber: 3, BlockHash: oldChain[2].Hash(), TxHash: c2h(oldChain, 2)}, - // Removed log for block 3 - {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(3)}, Data: i2h(13).Bytes(), BlockNumber: 3, BlockHash: oldChain[2].Hash(), TxHash: c2h(oldChain, 2), Removed: true}, + {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(1)}, Data: i2h(11).Bytes(), BlockNumber: 1, Index: 0}, + {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(2)}, Data: i2h(11).Bytes(), BlockNumber: 1, Index: 1}, + {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(3)}, Data: i2h(11).Bytes(), BlockNumber: 1, Index: 2}, + {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(4)}, Data: i2h(11).Bytes(), BlockNumber: 1, Index: 3}, + {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(5)}, Data: i2h(11).Bytes(), BlockNumber: 1, Index: 4}, + {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(2)}, Data: i2h(12).Bytes(), BlockNumber: 2, Index: 0}, + {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(3)}, Data: i2h(12).Bytes(), BlockNumber: 2, Index: 1}, + {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(4)}, Data: i2h(12).Bytes(), BlockNumber: 2, Index: 2}, + {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(5)}, Data: i2h(12).Bytes(), BlockNumber: 2, Index: 3}, + {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(6)}, Data: i2h(12).Bytes(), BlockNumber: 2, Index: 4}, + // New logs for 4 onwards - {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(1)}, Data: i2h(103).Bytes(), BlockNumber: 3, BlockHash: newChain[0].Hash(), TxHash: c2h(newChain, 0)}, - {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(2)}, Data: i2h(104).Bytes(), BlockNumber: 4, BlockHash: newChain[1].Hash(), TxHash: c2h(newChain, 1)}, - {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(3)}, Data: i2h(105).Bytes(), BlockNumber: 5, BlockHash: newChain[2].Hash(), TxHash: c2h(newChain, 2)}, - {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(4)}, Data: i2h(106).Bytes(), BlockNumber: 6, BlockHash: newChain[3].Hash(), TxHash: c2h(newChain, 3)}, + {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(1)}, Data: i2h(103).Bytes(), BlockNumber: 3, Index: 0}, + {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(2)}, Data: i2h(104).Bytes(), BlockNumber: 4, Index: 0}, + {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(3)}, Data: i2h(105).Bytes(), BlockNumber: 5, Index: 0}, + {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(4)}, Data: i2h(106).Bytes(), BlockNumber: 6, Index: 0}, } expected = append(expected, liveLogs...) @@ -1068,7 +1075,7 @@ func TestLogsSubscriptionReorg(t *testing.T) { } // for i, log := range fetched { - // logger.Debug("Flog", "i", fmt.Sprintf("%02d", i), "blknum", log.BlockNumber, "index", log.Index, "removed", log.Removed, "to", common.BytesToAddress(log.Topics[2].Bytes()), "amount", common.BytesToHash(log.Data).Big().Uint64()) + // logger.Info("Flog", "i", fmt.Sprintf("%02d", i), "blknum", log.BlockNumber, "index", log.Index, "removed", log.Removed, "to", common.BytesToAddress(log.Topics[2].Bytes()), "amount", common.BytesToHash(log.Data).Big().Uint64()) // } // // for i, log := range expected { From 6d0d8ac38a0282c901b873e5f436594a6b432a1a Mon Sep 17 00:00:00 2001 From: jsvisa Date: Tue, 25 Jul 2023 17:18:05 +0800 Subject: [PATCH 45/70] eth/filters: check reorgBlock in idempotent Signed-off-by: jsvisa --- eth/filters/api.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/eth/filters/api.go b/eth/filters/api.go index 3d93cf3cc5..6184927a99 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -380,13 +380,10 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from } continue } - // TODO: if reorg occured in more than once? - if reorgBlock == 0 { - reorgBlock = logs[0].BlockNumber - if reorgBlock <= delivered { - logger.Info("Reorg detected", "reorgBlock", reorgBlock, "delivered", delivered) - reorged = true - } + reorgBlock = logs[0].BlockNumber + if reorgBlock <= delivered { + logger.Info("Reorg detected", "reorgBlock", reorgBlock, "delivered", delivered) + reorged = true } if !reorged { continue From f404fea40c03c3eb14e989534c6b8f61553f45d8 Mon Sep 17 00:00:00 2001 From: jsvisa Date: Tue, 25 Jul 2023 18:08:17 +0800 Subject: [PATCH 46/70] eth/filters: dry testcases Signed-off-by: jsvisa --- eth/filters/filter_system_test.go | 181 +++++++++++++++++++++--------- 1 file changed, 129 insertions(+), 52 deletions(-) diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index 9cca3f9095..4e4eb0b851 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -893,22 +893,33 @@ func TestLogsSubscription(t *testing.T) { } } -// TestLogsSubscriptionReorg tests the behavior of the filter system when a reorganization occurs. -func TestLogsSubscriptionReorg(t *testing.T) { - t.Parallel() +var ( + signer = types.HomesteadSigner{} + key, _ = crypto.GenerateKey() + addr = crypto.PubkeyToAddress(key.PublicKey) + contract = common.HexToAddress("0000000000000000000000000000000000031ec7") + // Transfer(address indexed from, address indexed to, uint256 value); + topic = common.HexToHash("0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef") +) +func makeTx(to int, value int, b *core.BlockGen) *types.Transaction { + // transfer(address to, uint256 value) + data := fmt.Sprintf("0xa9059cbb%s%s", common.HexToHash(common.BigToAddress(big.NewInt(int64(to))).Hex()).String()[2:], common.BytesToHash([]byte{byte(value)}).String()[2:]) + tx, _ := types.SignTx(types.NewTx(&types.LegacyTx{Nonce: b.TxNonce(addr), To: &contract, Gas: 46000, GasPrice: b.BaseFee(), Data: common.FromHex(data)}), signer, key) + return tx +} + +func i2h(i int) common.Hash { return common.BigToHash(big.NewInt(int64(i))) } +func a2h(a common.Address) common.Hash { return common.HexToHash(a.Hex()) } + +func testLogsSubscriptionReorg(t *testing.T, oldChainMaker, newChainMaker, pendingMaker func(i int, b *core.BlockGen), expected []*types.Log) { var ( db = rawdb.NewMemoryDatabase() backend, sys = newTestFilterSystem(t, db, Config{}) api = NewFilterAPI(sys, false) - signer = types.HomesteadSigner{} - key, _ = crypto.GenerateKey() - addr = crypto.PubkeyToAddress(key.PublicKey) - contract = common.HexToAddress("0000000000000000000000000000000000031ec7") - // Transfer(address indexed from, address indexed to, uint256 value); - topic = common.HexToHash("0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef") - genesis = &core.Genesis{ - Config: params.TestChainConfig, + genesis = &core.Genesis{ + Config: params.TestChainConfig, + GasLimit: 10000000000000, Alloc: core.GenesisAlloc{ // // SPDX-License-Identifier: GPL-3.0 // pragma solidity >=0.7.0 <0.9.0; @@ -932,18 +943,7 @@ func TestLogsSubscriptionReorg(t *testing.T) { if err != nil { t.Fatal(err) } - makeTx := func(to int, value int, b *core.BlockGen) *types.Transaction { - // transfer(address to, uint256 value) - data := fmt.Sprintf("0xa9059cbb%s%s", common.HexToHash(common.BigToAddress(big.NewInt(int64(to))).Hex()).String()[2:], common.BytesToHash([]byte{byte(value)}).String()[2:]) - tx, _ := types.SignTx(types.NewTx(&types.LegacyTx{Nonce: b.TxNonce(addr), To: &contract, Gas: 46000, GasPrice: b.BaseFee(), Data: common.FromHex(data)}), signer, key) - return tx - } - oldChain, _ := core.GenerateChain(genesis.Config, genesis.ToBlock(), ethash.NewFaker(), db, 5, func(i int, b *core.BlockGen) { - for j := 0; j < 5; j++ { - tx := makeTx(i+j+1, i+11, b) - b.AddTx(tx) - } - }) + oldChain, _ := core.GenerateChain(genesis.Config, genesis.ToBlock(), ethash.NewFaker(), db, 5, oldChainMaker) bc, err := core.NewBlockChain(db, nil, genesis, nil, ethash.NewFaker(), vm.Config{}, nil, new(uint64)) if err != nil { t.Fatal(err) @@ -962,41 +962,14 @@ func TestLogsSubscriptionReorg(t *testing.T) { if err != nil { t.Fatal(err) } - newChain, _ := core.GenerateChain(genesis.Config, oldChain[1], ethash.NewFaker(), db, 4, func(i int, b *core.BlockGen) { - tx := makeTx(i+1, i+103, b) - b.AddTx(tx) - }) + newChain, _ := core.GenerateChain(genesis.Config, oldChain[1], ethash.NewFaker(), db, 4, newChainMaker) logger.Info("oldChain/newChain", "oldChain", fmt.Sprintf("%d..%d", oldChain[0].Number(), oldChain[len(oldChain)-1].Number()), "newChain", fmt.Sprintf("%d..%d", newChain[0].Number(), newChain[len(newChain)-1].Number())) // Generate pending block, logs for which // will be sent to subscription feed. - _, preceipts := core.GenerateChain(genesis.Config, newChain[len(newChain)-1], ethash.NewFaker(), db, 1, func(i int, b *core.BlockGen) { - tx := makeTx(i+1, i+21, b) - b.AddTx(tx) - }) + _, preceipts := core.GenerateChain(genesis.Config, newChain[len(newChain)-1], ethash.NewFaker(), db, 1, pendingMaker) liveLogs := preceipts[0][0].Logs - i2h := func(i int) common.Hash { return common.BigToHash(big.NewInt(int64(i))) } - a2h := func(a common.Address) common.Hash { return common.HexToHash(a.Hex()) } - expected := []*types.Log{ - // Original chain until block 3 - {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(1)}, Data: i2h(11).Bytes(), BlockNumber: 1, Index: 0}, - {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(2)}, Data: i2h(11).Bytes(), BlockNumber: 1, Index: 1}, - {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(3)}, Data: i2h(11).Bytes(), BlockNumber: 1, Index: 2}, - {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(4)}, Data: i2h(11).Bytes(), BlockNumber: 1, Index: 3}, - {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(5)}, Data: i2h(11).Bytes(), BlockNumber: 1, Index: 4}, - {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(2)}, Data: i2h(12).Bytes(), BlockNumber: 2, Index: 0}, - {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(3)}, Data: i2h(12).Bytes(), BlockNumber: 2, Index: 1}, - {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(4)}, Data: i2h(12).Bytes(), BlockNumber: 2, Index: 2}, - {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(5)}, Data: i2h(12).Bytes(), BlockNumber: 2, Index: 3}, - {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(6)}, Data: i2h(12).Bytes(), BlockNumber: 2, Index: 4}, - - // New logs for 4 onwards - {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(1)}, Data: i2h(103).Bytes(), BlockNumber: 3, Index: 0}, - {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(2)}, Data: i2h(104).Bytes(), BlockNumber: 4, Index: 0}, - {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(3)}, Data: i2h(105).Bytes(), BlockNumber: 5, Index: 0}, - {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(4)}, Data: i2h(106).Bytes(), BlockNumber: 6, Index: 0}, - } expected = append(expected, liveLogs...) // Calculate address balances @@ -1120,3 +1093,107 @@ func TestLogsSubscriptionReorg(t *testing.T) { t.Fatalf("test failed: %v", err) } } + +// TestLogsSubscriptionReorg tests the behavior of the filter system when a reorganization occurs. +func TestLogsSubscriptionReorg1(t *testing.T) { + t.Parallel() + + expected := []*types.Log{ + // Original chain until block 3 + {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(1)}, Data: i2h(11).Bytes(), BlockNumber: 1, Index: 0}, + {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(2)}, Data: i2h(11).Bytes(), BlockNumber: 1, Index: 1}, + {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(3)}, Data: i2h(11).Bytes(), BlockNumber: 1, Index: 2}, + {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(4)}, Data: i2h(11).Bytes(), BlockNumber: 1, Index: 3}, + {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(5)}, Data: i2h(11).Bytes(), BlockNumber: 1, Index: 4}, + {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(2)}, Data: i2h(12).Bytes(), BlockNumber: 2, Index: 0}, + {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(3)}, Data: i2h(12).Bytes(), BlockNumber: 2, Index: 1}, + {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(4)}, Data: i2h(12).Bytes(), BlockNumber: 2, Index: 2}, + {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(5)}, Data: i2h(12).Bytes(), BlockNumber: 2, Index: 3}, + {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(6)}, Data: i2h(12).Bytes(), BlockNumber: 2, Index: 4}, + + // New logs for 4 onwards + {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(1)}, Data: i2h(103).Bytes(), BlockNumber: 3, Index: 0}, + {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(2)}, Data: i2h(104).Bytes(), BlockNumber: 4, Index: 0}, + {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(3)}, Data: i2h(105).Bytes(), BlockNumber: 5, Index: 0}, + {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(4)}, Data: i2h(106).Bytes(), BlockNumber: 6, Index: 0}, + } + testLogsSubscriptionReorg(t, func(i int, b *core.BlockGen) { + for j := 0; j < 5; j++ { + tx := makeTx(i+j+1, i+11, b) + b.AddTx(tx) + } + }, func(i int, b *core.BlockGen) { + tx := makeTx(i+1, i+103, b) + b.AddTx(tx) + }, func(i int, b *core.BlockGen) { + tx := makeTx(i+1, i+21, b) + b.AddTx(tx) + }, expected) +} + +// TestLogsSubscriptionReorged tests the behavior of the filter system when a reorganization occurs. +func TestLogsSubscriptionReorged2(t *testing.T) { + t.Parallel() + + expected := []*types.Log{ + // Original chain until block 3 + {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(1)}, Data: i2h(11).Bytes(), BlockNumber: 1, Index: 0}, + {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(2)}, Data: i2h(12).Bytes(), BlockNumber: 2, Index: 0}, + + // New logs for 4 onwards + {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(1)}, Data: i2h(103).Bytes(), BlockNumber: 3, Index: 0}, + {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(2)}, Data: i2h(104).Bytes(), BlockNumber: 4, Index: 0}, + {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(3)}, Data: i2h(105).Bytes(), BlockNumber: 5, Index: 0}, + {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(4)}, Data: i2h(106).Bytes(), BlockNumber: 6, Index: 0}, + } + testLogsSubscriptionReorg(t, func(i int, b *core.BlockGen) { + tx := makeTx(i+1, i+11, b) + b.AddTx(tx) + }, func(i int, b *core.BlockGen) { + tx := makeTx(i+1, i+103, b) + b.AddTx(tx) + }, func(i int, b *core.BlockGen) { + tx := makeTx(i+1, i+21, b) + b.AddTx(tx) + }, expected) +} + +func TestLogsSubscriptionReorg3(t *testing.T) { + t.Parallel() + + txs := 520 + expected := []*types.Log{} + // oldChain + for blknum := 1; blknum <= 2; blknum++ { + for i := 0; i < txs; i++ { + expected = append(expected, &types.Log{ + Address: contract, + Topics: []common.Hash{topic, a2h(addr), i2h(blknum + i)}, Data: i2h(10 + blknum).Bytes(), + BlockNumber: uint64(blknum), + Index: uint(i), + }) + } + } + expected = append( + expected, + // newChain + []*types.Log{ + {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(1)}, Data: i2h(103).Bytes(), BlockNumber: 3, Index: 0}, + {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(2)}, Data: i2h(104).Bytes(), BlockNumber: 4, Index: 0}, + {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(3)}, Data: i2h(105).Bytes(), BlockNumber: 5, Index: 0}, + {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(4)}, Data: i2h(106).Bytes(), BlockNumber: 6, Index: 0}, + }..., + ) + testLogsSubscriptionReorg(t, func(i int, b *core.BlockGen) { + for j := 0; j < txs; j++ { + tx := makeTx(i+j+1, i+11, b) + b.AddTx(tx) + } + }, func(i int, b *core.BlockGen) { + tx := makeTx(i+1, i+103, b) + b.AddTx(tx) + }, func(i int, b *core.BlockGen) { + tx := makeTx(i+1, i+21, b) + b.AddTx(tx) + }, expected) +} From 67bb04d5d67ad9cdb69bc86fee7342694ad75193 Mon Sep 17 00:00:00 2001 From: jsvisa Date: Tue, 25 Jul 2023 20:15:33 +0800 Subject: [PATCH 47/70] eth/filters: more testcase Signed-off-by: jsvisa --- eth/filters/filter_system_test.go | 89 +++++++++++++++---------------- 1 file changed, 43 insertions(+), 46 deletions(-) diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index 4e4eb0b851..c7ba9736c1 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -912,7 +912,8 @@ func makeTx(to int, value int, b *core.BlockGen) *types.Transaction { func i2h(i int) common.Hash { return common.BigToHash(big.NewInt(int64(i))) } func a2h(a common.Address) common.Hash { return common.HexToHash(a.Hex()) } -func testLogsSubscriptionReorg(t *testing.T, oldChainMaker, newChainMaker, pendingMaker func(i int, b *core.BlockGen), expected []*types.Log) { +// TestLogsSubscriptionReorg tests that logs subscription works correctly in case of reorg. +func testLogsSubscriptionReorg(t *testing.T, oldChainMaker, newChainMaker, pendingMaker func(i int, b *core.BlockGen), oldChainLen, newChainLen, forkAt, reorgAt int, expected []*types.Log) { var ( db = rawdb.NewMemoryDatabase() backend, sys = newTestFilterSystem(t, db, Config{}) @@ -943,7 +944,7 @@ func testLogsSubscriptionReorg(t *testing.T, oldChainMaker, newChainMaker, pendi if err != nil { t.Fatal(err) } - oldChain, _ := core.GenerateChain(genesis.Config, genesis.ToBlock(), ethash.NewFaker(), db, 5, oldChainMaker) + oldChain, _ := core.GenerateChain(genesis.Config, genesis.ToBlock(), ethash.NewFaker(), db, oldChainLen, oldChainMaker) bc, err := core.NewBlockChain(db, nil, genesis, nil, ethash.NewFaker(), vm.Config{}, nil, new(uint64)) if err != nil { t.Fatal(err) @@ -962,7 +963,7 @@ func testLogsSubscriptionReorg(t *testing.T, oldChainMaker, newChainMaker, pendi if err != nil { t.Fatal(err) } - newChain, _ := core.GenerateChain(genesis.Config, oldChain[1], ethash.NewFaker(), db, 4, newChainMaker) + newChain, _ := core.GenerateChain(genesis.Config, oldChain[forkAt], ethash.NewFaker(), db, newChainLen, newChainMaker) logger.Info("oldChain/newChain", "oldChain", fmt.Sprintf("%d..%d", oldChain[0].Number(), oldChain[len(oldChain)-1].Number()), "newChain", fmt.Sprintf("%d..%d", newChain[0].Number(), newChain[len(newChain)-1].Number())) // Generate pending block, logs for which @@ -1036,7 +1037,7 @@ func testLogsSubscriptionReorg(t *testing.T, oldChainMaker, newChainMaker, pendi // We halt the sender by blocking Notify(). However sender will already prepare // logs for next block and send as soon as Notify is released. So we do reorg // one block earlier than we intend. - if l.BlockNumber == 2 && l.Index == 0 { + if l.BlockNumber == uint64(reorgAt) && l.Index == 0 { // signal reorg reorgSignal <- struct{}{} // wait for reorg to happen @@ -1094,45 +1095,7 @@ func testLogsSubscriptionReorg(t *testing.T, oldChainMaker, newChainMaker, pendi } } -// TestLogsSubscriptionReorg tests the behavior of the filter system when a reorganization occurs. -func TestLogsSubscriptionReorg1(t *testing.T) { - t.Parallel() - - expected := []*types.Log{ - // Original chain until block 3 - {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(1)}, Data: i2h(11).Bytes(), BlockNumber: 1, Index: 0}, - {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(2)}, Data: i2h(11).Bytes(), BlockNumber: 1, Index: 1}, - {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(3)}, Data: i2h(11).Bytes(), BlockNumber: 1, Index: 2}, - {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(4)}, Data: i2h(11).Bytes(), BlockNumber: 1, Index: 3}, - {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(5)}, Data: i2h(11).Bytes(), BlockNumber: 1, Index: 4}, - {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(2)}, Data: i2h(12).Bytes(), BlockNumber: 2, Index: 0}, - {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(3)}, Data: i2h(12).Bytes(), BlockNumber: 2, Index: 1}, - {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(4)}, Data: i2h(12).Bytes(), BlockNumber: 2, Index: 2}, - {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(5)}, Data: i2h(12).Bytes(), BlockNumber: 2, Index: 3}, - {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(6)}, Data: i2h(12).Bytes(), BlockNumber: 2, Index: 4}, - - // New logs for 4 onwards - {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(1)}, Data: i2h(103).Bytes(), BlockNumber: 3, Index: 0}, - {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(2)}, Data: i2h(104).Bytes(), BlockNumber: 4, Index: 0}, - {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(3)}, Data: i2h(105).Bytes(), BlockNumber: 5, Index: 0}, - {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(4)}, Data: i2h(106).Bytes(), BlockNumber: 6, Index: 0}, - } - testLogsSubscriptionReorg(t, func(i int, b *core.BlockGen) { - for j := 0; j < 5; j++ { - tx := makeTx(i+j+1, i+11, b) - b.AddTx(tx) - } - }, func(i int, b *core.BlockGen) { - tx := makeTx(i+1, i+103, b) - b.AddTx(tx) - }, func(i int, b *core.BlockGen) { - tx := makeTx(i+1, i+21, b) - b.AddTx(tx) - }, expected) -} - -// TestLogsSubscriptionReorged tests the behavior of the filter system when a reorganization occurs. -func TestLogsSubscriptionReorged2(t *testing.T) { +func TestLogsSubscriptionReorgedSimple(t *testing.T) { t.Parallel() expected := []*types.Log{ @@ -1155,10 +1118,10 @@ func TestLogsSubscriptionReorged2(t *testing.T) { }, func(i int, b *core.BlockGen) { tx := makeTx(i+1, i+21, b) b.AddTx(tx) - }, expected) + }, 5, 4, 1, 2, expected) } -func TestLogsSubscriptionReorg3(t *testing.T) { +func TestLogsSubscriptionReorgOldMore(t *testing.T) { t.Parallel() txs := 520 @@ -1195,5 +1158,39 @@ func TestLogsSubscriptionReorg3(t *testing.T) { }, func(i int, b *core.BlockGen) { tx := makeTx(i+1, i+21, b) b.AddTx(tx) - }, expected) + }, 5, 4, 1, 2, expected) +} + +func TestLogsSubscriptionReorgNewMore(t *testing.T) { + t.Parallel() + + txs := 520 + // oldChain + expected := []*types.Log{ + {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(1)}, Data: i2h(11).Bytes(), BlockNumber: 1, Index: 0}, + {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(2)}, Data: i2h(12).Bytes(), BlockNumber: 2, Index: 0}, + } + // newChain + for blknum := 3; blknum <= 6; blknum++ { + for i := 0; i < txs; i++ { + expected = append(expected, &types.Log{ + Address: contract, + Topics: []common.Hash{topic, a2h(addr), i2h(blknum + i - 2)}, Data: i2h(100 + blknum).Bytes(), + BlockNumber: uint64(blknum), + Index: uint(i), + }) + } + } + testLogsSubscriptionReorg(t, func(i int, b *core.BlockGen) { + tx := makeTx(i+1, i+11, b) + b.AddTx(tx) + }, func(i int, b *core.BlockGen) { + for j := 0; j < txs; j++ { + tx := makeTx(i+j+1, i+103, b) + b.AddTx(tx) + } + }, func(i int, b *core.BlockGen) { + tx := makeTx(i+1, i+21, b) + b.AddTx(tx) + }, 5, 4, 1, 2, expected) } From 3e9bc42f4c6b17553e66ce8f501f3993bc90eb55 Mon Sep 17 00:00:00 2001 From: jsvisa Date: Tue, 25 Jul 2023 20:48:34 +0800 Subject: [PATCH 48/70] eth/filters: balanceDiffer as private function Signed-off-by: jsvisa --- eth/filters/filter_system_test.go | 75 ++++++++++++++++--------------- 1 file changed, 38 insertions(+), 37 deletions(-) diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index c7ba9736c1..3d0ea785be 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -912,6 +912,42 @@ func makeTx(to int, value int, b *core.BlockGen) *types.Transaction { func i2h(i int) common.Hash { return common.BigToHash(big.NewInt(int64(i))) } func a2h(a common.Address) common.Hash { return common.HexToHash(a.Hex()) } +// calculateBalance calculates the address balances of all Transfer events +// We check with the balance instead of logs because there is a gap between the ChainReorg occurred and detected, +// so we can't fully control how many logs we'll receive. +// By checking the balance, we can test with the token transfer amount. +func calculateBalance(logs []*types.Log) map[common.Address]uint64 { + balances := make(map[common.Address]uint64) + for _, log := range logs { + log := log + from := common.BytesToAddress(log.Topics[1].Bytes()) + to := common.BytesToAddress(log.Topics[2].Bytes()) + amount := common.BytesToHash(log.Data).Big().Uint64() + + if _, ok := balances[from]; !ok { + balances[from] = 0 + } + if _, ok := balances[to]; !ok { + balances[to] = 0 + } + + if log.Removed { // revert + balances[from] += amount + balances[to] -= amount + } else { + balances[from] -= amount + balances[to] += amount + } + } + // Remove zero balances + for addr, balance := range balances { + if balance == 0 { + delete(balances, addr) + } + } + return balances +} + // TestLogsSubscriptionReorg tests that logs subscription works correctly in case of reorg. func testLogsSubscriptionReorg(t *testing.T, oldChainMaker, newChainMaker, pendingMaker func(i int, b *core.BlockGen), oldChainLen, newChainLen, forkAt, reorgAt int, expected []*types.Log) { var ( @@ -973,42 +1009,7 @@ func testLogsSubscriptionReorg(t *testing.T, oldChainMaker, newChainMaker, pendi expected = append(expected, liveLogs...) - // Calculate address balances - // We check with the balance instead of logs because there is a gap between the ChainReorg occurred and detected, - // so we can't fully control how many logs we'll receive. - // By checking the balance, we can test with the token transfer amount. - balanceDiffer := func(logs []*types.Log) map[common.Address]uint64 { - balances := make(map[common.Address]uint64) - for _, log := range logs { - log := log - from := common.BytesToAddress(log.Topics[1].Bytes()) - to := common.BytesToAddress(log.Topics[2].Bytes()) - amount := common.BytesToHash(log.Data).Big().Uint64() - - if _, ok := balances[from]; !ok { - balances[from] = 0 - } - if _, ok := balances[to]; !ok { - balances[to] = 0 - } - - if log.Removed { // revert - balances[from] += amount - balances[to] -= amount - } else { - balances[from] -= amount - balances[to] += amount - } - } - // Remove zero balances - for addr, balance := range balances { - if balance == 0 { - delete(balances, addr) - } - } - return balances - } - expectedBalance := balanceDiffer(expected) + expectedBalance := calculateBalance(expected) // Subscribe to logs var ( @@ -1056,7 +1057,7 @@ func testLogsSubscriptionReorg(t *testing.T, oldChainMaker, newChainMaker, pendi // logger.Debug("Elog", "i", fmt.Sprintf("%02d", i), "blknum", log.BlockNumber, "index", log.Index, "removed", log.Removed, "to", common.BytesToAddress(log.Topics[2].Bytes()), "amount", common.BytesToHash(log.Data).Big().Uint64()) // } - fetchedBalance := balanceDiffer(fetched) + fetchedBalance := calculateBalance(fetched) if len(fetchedBalance) != len(expectedBalance) { errc <- fmt.Errorf("invalid number of balances, have %d, want %d", len(fetchedBalance), len(expectedBalance)) logger.Info("balance diff", "fetched", fetchedBalance, "expected", expectedBalance) From b187581ee879ee82028ed9b2002653e7a5c56cf5 Mon Sep 17 00:00:00 2001 From: jsvisa Date: Tue, 25 Jul 2023 21:56:00 +0800 Subject: [PATCH 49/70] eth/filters: reuse genesisAlloc Signed-off-by: jsvisa --- eth/filters/filter_system_test.go | 46 ++++++++++++++++--------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index 3d0ea785be..036b9e82d3 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -899,7 +899,21 @@ var ( addr = crypto.PubkeyToAddress(key.PublicKey) contract = common.HexToAddress("0000000000000000000000000000000000031ec7") // Transfer(address indexed from, address indexed to, uint256 value); - topic = common.HexToHash("0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef") + topic = common.HexToHash("0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef") + genesisAlloc = core.GenesisAlloc{ + // // SPDX-License-Identifier: GPL-3.0 + // pragma solidity >=0.7.0 <0.9.0; + // + // contract Token { + // event Transfer(address indexed from, address indexed to, uint256 value); + // function transfer(address to, uint256 value) public returns (bool) { + // emit Transfer(msg.sender, to, value); + // return true; + // } + // } + contract: {Balance: big.NewInt(params.Ether), Code: common.FromHex("0x608060405234801561001057600080fd5b506004361061002b5760003560e01c8063a9059cbb14610030575b600080fd5b61004a6004803603810190610045919061016a565b610060565b60405161005791906101c5565b60405180910390f35b60008273ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff167fddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef846040516100bf91906101ef565b60405180910390a36001905092915050565b600080fd5b600073ffffffffffffffffffffffffffffffffffffffff82169050919050565b6000610101826100d6565b9050919050565b610111816100f6565b811461011c57600080fd5b50565b60008135905061012e81610108565b92915050565b6000819050919050565b61014781610134565b811461015257600080fd5b50565b6000813590506101648161013e565b92915050565b60008060408385031215610181576101806100d1565b5b600061018f8582860161011f565b92505060206101a085828601610155565b9150509250929050565b60008115159050919050565b6101bf816101aa565b82525050565b60006020820190506101da60008301846101b6565b92915050565b6101e981610134565b82525050565b600060208201905061020460008301846101e0565b9291505056fea2646970667358221220b469033f4b77b9565ee84e0a2f04d496b18160d26034d54f9487e57788fd36d564736f6c63430008120033")}, + addr: {Balance: big.NewInt(params.Ether)}, + } ) func makeTx(to int, value int, b *core.BlockGen) *types.Transaction { @@ -912,6 +926,13 @@ func makeTx(to int, value int, b *core.BlockGen) *types.Transaction { func i2h(i int) common.Hash { return common.BigToHash(big.NewInt(int64(i))) } func a2h(a common.Address) common.Hash { return common.HexToHash(a.Hex()) } +func parseTransferLog(log *types.Log) (from, to common.Address, amount uint64) { + from = common.BytesToAddress(log.Topics[1].Bytes()) + to = common.BytesToAddress(log.Topics[2].Bytes()) + amount = common.BytesToHash(log.Data).Big().Uint64() + return +} + // calculateBalance calculates the address balances of all Transfer events // We check with the balance instead of logs because there is a gap between the ChainReorg occurred and detected, // so we can't fully control how many logs we'll receive. @@ -920,9 +941,7 @@ func calculateBalance(logs []*types.Log) map[common.Address]uint64 { balances := make(map[common.Address]uint64) for _, log := range logs { log := log - from := common.BytesToAddress(log.Topics[1].Bytes()) - to := common.BytesToAddress(log.Topics[2].Bytes()) - amount := common.BytesToHash(log.Data).Big().Uint64() + from, to, amount := parseTransferLog(log) if _, ok := balances[from]; !ok { balances[from] = 0 @@ -954,24 +973,7 @@ func testLogsSubscriptionReorg(t *testing.T, oldChainMaker, newChainMaker, pendi db = rawdb.NewMemoryDatabase() backend, sys = newTestFilterSystem(t, db, Config{}) api = NewFilterAPI(sys, false) - genesis = &core.Genesis{ - Config: params.TestChainConfig, - GasLimit: 10000000000000, - Alloc: core.GenesisAlloc{ - // // SPDX-License-Identifier: GPL-3.0 - // pragma solidity >=0.7.0 <0.9.0; - // - // contract Token { - // event Transfer(address indexed from, address indexed to, uint256 value); - // function transfer(address to, uint256 value) public returns (bool) { - // emit Transfer(msg.sender, to, value); - // return true; - // } - // } - contract: {Balance: big.NewInt(params.Ether), Code: common.FromHex("0x608060405234801561001057600080fd5b506004361061002b5760003560e01c8063a9059cbb14610030575b600080fd5b61004a6004803603810190610045919061016a565b610060565b60405161005791906101c5565b60405180910390f35b60008273ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff167fddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef846040516100bf91906101ef565b60405180910390a36001905092915050565b600080fd5b600073ffffffffffffffffffffffffffffffffffffffff82169050919050565b6000610101826100d6565b9050919050565b610111816100f6565b811461011c57600080fd5b50565b60008135905061012e81610108565b92915050565b6000819050919050565b61014781610134565b811461015257600080fd5b50565b6000813590506101648161013e565b92915050565b60008060408385031215610181576101806100d1565b5b600061018f8582860161011f565b92505060206101a085828601610155565b9150509250929050565b60008115159050919050565b6101bf816101aa565b82525050565b60006020820190506101da60008301846101b6565b92915050565b6101e981610134565b82525050565b600060208201905061020460008301846101e0565b9291505056fea2646970667358221220b469033f4b77b9565ee84e0a2f04d496b18160d26034d54f9487e57788fd36d564736f6c63430008120033")}, - addr: {Balance: big.NewInt(params.Ether)}, - }, - } + genesis = &core.Genesis{Config: params.TestChainConfig, Alloc: genesisAlloc, GasLimit: 10000000000000} ) // Hack: GenerateChainWithGenesis creates a new db. From 41a7d00ac8208b84de0b65f0e6c5a2c358c75226 Mon Sep 17 00:00:00 2001 From: jsvisa Date: Wed, 9 Aug 2023 23:27:30 +0800 Subject: [PATCH 50/70] eth/filters: ethclient set toBlock=nil to latest Signed-off-by: jsvisa --- eth/filters/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth/filters/api.go b/eth/filters/api.go index 6184927a99..95b950ead4 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -273,7 +273,7 @@ func (api *FilterAPI) Logs(ctx context.Context, crit FilterCriteria) (*rpc.Subsc // from: blockNum | safe | finalized, to: nil -> historical beginning at `from` to head, then live mode. // Every other case should fail with an error. func (api *FilterAPI) logs(ctx context.Context, notifier notifier, rpcSub *rpc.Subscription, crit FilterCriteria) error { - if crit.ToBlock != nil { + if crit.ToBlock != nil && rpc.BlockNumber(crit.ToBlock.Int64()) != rpc.LatestBlockNumber { return errInvalidToBlock } if crit.FromBlock == nil { From d78c6c74eb891d901b4d3e0ad36e273cccbcf96c Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Thu, 10 Aug 2023 17:51:06 +0200 Subject: [PATCH 51/70] refactors Signed-off-by: jsvisa --- eth/filters/api.go | 82 ++++++++++++++++--------------- eth/filters/filter_system_test.go | 55 ++++++++++++++------- ethclient/ethclient.go | 5 +- 3 files changed, 82 insertions(+), 60 deletions(-) diff --git a/eth/filters/api.go b/eth/filters/api.go index 95b950ead4..f844b90006 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -251,13 +251,14 @@ func (api *FilterAPI) NewHeads(ctx context.Context) (*rpc.Subscription, error) { } // notifier is used for broadcasting data(eg: logs) to rpc receivers -// used in unit testing +// used in unit testing. type notifier interface { Notify(id rpc.ID, data interface{}) error Closed() <-chan interface{} } -// Logs creates a subscription that fires for all new log that match the given filter criteria. +// Logs creates a subscription that fires for all historical +// and new logs that match the given filter criteria. func (api *FilterAPI) Logs(ctx context.Context, crit FilterCriteria) (*rpc.Subscription, error) { notifier, supported := rpc.NotifierFromContext(ctx) if !supported { @@ -268,12 +269,13 @@ func (api *FilterAPI) Logs(ctx context.Context, crit FilterCriteria) (*rpc.Subsc return rpcSub, err } -// logs is the inner implementation of logs filtering. -// from: nil, to: nil -> only live mode. -// from: blockNum | safe | finalized, to: nil -> historical beginning at `from` to head, then live mode. -// Every other case should fail with an error. +// logs is the inner implementation of logs subscription. +// The following criteria are valid: +// * from: nil, to: nil -> yield live logs. +// * from: blockNum | safe | finalized, to: nil -> historical beginning at `from` to head, then live logs. +// * Every other case should fail with an error. func (api *FilterAPI) logs(ctx context.Context, notifier notifier, rpcSub *rpc.Subscription, crit FilterCriteria) error { - if crit.ToBlock != nil && rpc.BlockNumber(crit.ToBlock.Int64()) != rpc.LatestBlockNumber { + if crit.ToBlock != nil { return errInvalidToBlock } if crit.FromBlock == nil { @@ -296,7 +298,7 @@ func (api *FilterAPI) logs(ctx context.Context, notifier notifier, rpcSub *rpc.S return api.histLogs(notifier, rpcSub, int64(from), crit) } -// liveLogs only retrieves live logs +// liveLogs only retrieves live logs. func (api *FilterAPI) liveLogs(notify notifier, rpcSub *rpc.Subscription, crit FilterCriteria) error { matchedLogs := make(chan []*types.Log) logsSub, err := api.events.SubscribeLogs(ethereum.FilterQuery(crit), matchedLogs) @@ -324,7 +326,7 @@ func (api *FilterAPI) liveLogs(notify notifier, rpcSub *rpc.Subscription, crit F return nil } -// histLogs retrieves the logs older than current header. +// histLogs retrieves logs older than current header. func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from int64, crit FilterCriteria) error { // Subscribe the Live logs // if an ChainReorg occurred, @@ -352,10 +354,9 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from // Compose and notify the logs from liveLogs and histLogs go func() { var ( - delivered uint64 - reorgBlock uint64 - liveOnly bool - reorged bool + delivered uint64 + liveOnly bool + reorged bool ) for { select { @@ -380,9 +381,9 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from } continue } - reorgBlock = logs[0].BlockNumber - if reorgBlock <= delivered { - logger.Info("Reorg detected", "reorgBlock", reorgBlock, "delivered", delivered) + reorgBlock := logs[0].BlockNumber + if !reorged && reorgBlock <= delivered { + logger.Debug("Reorg detected", "reorgBlock", reorgBlock, "delivered", delivered) reorged = true } if !reorged { @@ -430,7 +431,29 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from func (api *FilterAPI) doHistLogs(from int64, crit FilterCriteria, histLogs chan<- *types.Log, closeC chan struct{}) error { // The original request ctx will be canceled as soon as the parent goroutine // returns a subscription. Use a new context instead. - ctx := context.Background() + ctx, cancel := context.WithCancel(context.Background()) + + // Fetch logs from a range of blocks. + fetchRange := func(from, to int64) error { + f := api.sys.NewRangeFilter(from, to, crit.Addresses, crit.Topics) + logsCh, errChan := f.rangeLogsAsync(ctx) + for { + select { + case <-closeC: + cancel() + return nil + case log := <-logsCh: + histLogs <- log + case err := <-errChan: + if err != nil { + logger.Error("Error while fetching historical logs", "err", err) + return err + } + // Range filter is done. + return nil + } + } + } for { // Get the latest block header. @@ -440,31 +463,10 @@ func (api *FilterAPI) doHistLogs(from int64, crit FilterCriteria, histLogs chan< } head := header.Number.Int64() if from > head { - logger.Info("Finish historical sync", "from", from, "head", head) + logger.Debug("Finish historical sync", "from", from, "head", head) return nil } - - // Do historical sync beginning at `from` to head. - f := api.sys.NewRangeFilter(from, head, crit.Addresses, crit.Topics) - logsCh, errChan := f.rangeLogsAsync(ctx) - - FORLOOP: - for { - select { - case <-closeC: - return nil - case log := <-logsCh: - histLogs <- log - case err := <-errChan: - // Range filter is done or error, let's also stop the reorgLogs subscribe - if err != nil { - logger.Error("Error while fetching historical logs", "err", err) - return err - } - break FORLOOP - } - } - + fetchRange(from, head) // Move forward to the next batch from = head + 1 } diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index 036b9e82d3..a361487aa7 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -793,7 +793,7 @@ func TestLogsSubscription(t *testing.T) { expectErr error err chan error }{ - // from 0 to latest + // from 0 { FilterCriteria{FromBlock: big.NewInt(0)}, append(allLogs, liveLogs...), newMockNotifier(), &rpc.Subscription{ID: rpc.NewID()}, nil, nil, @@ -803,12 +803,12 @@ func TestLogsSubscription(t *testing.T) { FilterCriteria{FromBlock: big.NewInt(1), ToBlock: big.NewInt(rpc.LatestBlockNumber.Int64())}, nil, newMockNotifier(), &rpc.Subscription{ID: rpc.NewID()}, errInvalidToBlock, nil, }, - // from 2 to latest + // from 2 { FilterCriteria{FromBlock: big.NewInt(2)}, append(allLogs[1:], liveLogs...), newMockNotifier(), &rpc.Subscription{ID: rpc.NewID()}, nil, nil, }, - // from latest to latest + // live logs { FilterCriteria{}, liveLogs, newMockNotifier(), &rpc.Subscription{ID: rpc.NewID()}, nil, nil, @@ -1027,9 +1027,11 @@ func testLogsSubscriptionReorg(t *testing.T, oldChainMaker, newChainMaker, pendi } go func() { - var fetched []*types.Log - - timeout := time.After(3 * time.Second) + var ( + fetched []*types.Log + timeout = time.After(3 * time.Second) + reorged = false + ) fetchLoop: for { select { @@ -1040,11 +1042,12 @@ func testLogsSubscriptionReorg(t *testing.T, oldChainMaker, newChainMaker, pendi // We halt the sender by blocking Notify(). However sender will already prepare // logs for next block and send as soon as Notify is released. So we do reorg // one block earlier than we intend. - if l.BlockNumber == uint64(reorgAt) && l.Index == 0 { + if l.BlockNumber == uint64(reorgAt) && !reorged { // signal reorg reorgSignal <- struct{}{} // wait for reorg to happen <-reorgSignal + reorged = true } case <-timeout: break fetchLoop @@ -1102,26 +1105,40 @@ func TestLogsSubscriptionReorgedSimple(t *testing.T) { t.Parallel() expected := []*types.Log{ - // Original chain until block 3 + // Original chain until block 2 {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(1)}, Data: i2h(11).Bytes(), BlockNumber: 1, Index: 0}, {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(2)}, Data: i2h(12).Bytes(), BlockNumber: 2, Index: 0}, - // New logs for 4 onwards + // New logs for 3 onwards {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(1)}, Data: i2h(103).Bytes(), BlockNumber: 3, Index: 0}, {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(2)}, Data: i2h(104).Bytes(), BlockNumber: 4, Index: 0}, {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(3)}, Data: i2h(105).Bytes(), BlockNumber: 5, Index: 0}, {Address: contract, Topics: []common.Hash{topic, a2h(addr), i2h(4)}, Data: i2h(106).Bytes(), BlockNumber: 6, Index: 0}, } - testLogsSubscriptionReorg(t, func(i int, b *core.BlockGen) { - tx := makeTx(i+1, i+11, b) - b.AddTx(tx) - }, func(i int, b *core.BlockGen) { - tx := makeTx(i+1, i+103, b) - b.AddTx(tx) - }, func(i int, b *core.BlockGen) { - tx := makeTx(i+1, i+21, b) - b.AddTx(tx) - }, 5, 4, 1, 2, expected) + t.Run("reorg-during-historical", func(t *testing.T) { + testLogsSubscriptionReorg(t, func(i int, b *core.BlockGen) { + tx := makeTx(i+1, i+11, b) + b.AddTx(tx) + }, func(i int, b *core.BlockGen) { + tx := makeTx(i+1, i+103, b) + b.AddTx(tx) + }, func(i int, b *core.BlockGen) { + tx := makeTx(i+1, i+21, b) + b.AddTx(tx) + }, 5, 4, 1, 2, expected) + }) + t.Run("reorg-after-historical", func(t *testing.T) { + testLogsSubscriptionReorg(t, func(i int, b *core.BlockGen) { + tx := makeTx(i+1, i+11, b) + b.AddTx(tx) + }, func(i int, b *core.BlockGen) { + tx := makeTx(i+1, i+103, b) + b.AddTx(tx) + }, func(i int, b *core.BlockGen) { + tx := makeTx(i+1, i+21, b) + b.AddTx(tx) + }, 5, 4, 1, 5, expected) + }) } func TestLogsSubscriptionReorgOldMore(t *testing.T) { diff --git a/ethclient/ethclient.go b/ethclient/ethclient.go index 1195929f7d..f4b4a33ee1 100644 --- a/ethclient/ethclient.go +++ b/ethclient/ethclient.go @@ -476,6 +476,9 @@ func (ec *Client) SubscribeFilterLogs(ctx context.Context, q ethereum.FilterQuer if err != nil { return nil, err } + // TODO(s1na): Log subscription prohibits ToBlock being set. + // This will be later re-allowed once it supports historical-only queries. + arg["toBlock"] = nil sub, err := ec.c.EthSubscribe(ctx, ch, "logs", arg) if err != nil { // Defensively prefer returning nil interface explicitly on error-path, instead @@ -486,7 +489,7 @@ func (ec *Client) SubscribeFilterLogs(ctx context.Context, q ethereum.FilterQuer return sub, nil } -func toFilterArg(q ethereum.FilterQuery) (interface{}, error) { +func toFilterArg(q ethereum.FilterQuery) (map[string]interface{}, error) { arg := map[string]interface{}{ "address": q.Addresses, "topics": q.Topics, From 3083f77c87948d4ff02d16c5332a4c23fb3bed12 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Thu, 10 Aug 2023 18:05:39 +0200 Subject: [PATCH 52/70] refactor notifying logs Signed-off-by: jsvisa --- eth/filters/api.go | 37 +++++++++++++++++-------------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/eth/filters/api.go b/eth/filters/api.go index f844b90006..2efc9ed264 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -299,7 +299,7 @@ func (api *FilterAPI) logs(ctx context.Context, notifier notifier, rpcSub *rpc.S } // liveLogs only retrieves live logs. -func (api *FilterAPI) liveLogs(notify notifier, rpcSub *rpc.Subscription, crit FilterCriteria) error { +func (api *FilterAPI) liveLogs(notifier notifier, rpcSub *rpc.Subscription, crit FilterCriteria) error { matchedLogs := make(chan []*types.Log) logsSub, err := api.events.SubscribeLogs(ethereum.FilterQuery(crit), matchedLogs) if err != nil { @@ -310,14 +310,11 @@ func (api *FilterAPI) liveLogs(notify notifier, rpcSub *rpc.Subscription, crit F for { select { case logs := <-matchedLogs: - for _, log := range logs { - log := log - notify.Notify(rpcSub.ID, &log) - } + notifyLogsIf(notifier, rpcSub.ID, logs, nil) case <-rpcSub.Err(): // client send an unsubscribe request logsSub.Unsubscribe() return - case <-notify.Closed(): // connection dropped + case <-notifier.Closed(): // connection dropped logsSub.Unsubscribe() return } @@ -375,10 +372,7 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from continue } if liveOnly { - for _, log := range logs { - log := log - notifier.Notify(rpcSub.ID, &log) - } + notifyLogsIf(notifier, rpcSub.ID, logs, nil) continue } reorgBlock := logs[0].BlockNumber @@ -392,19 +386,13 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from if logs[0].Removed { // Send removed logs notification up until the point // we have delivered logs from old chain. - for _, log := range logs { - if log.BlockNumber <= delivered { - log := log - notifier.Notify(rpcSub.ID, &log) - } - } + notifyLogsIf(notifier, rpcSub.ID, logs, func(log *types.Log) bool { + return log.BlockNumber <= delivered + }) } else { // New logs are emitted for the whole new chain since the reorg block. // Send them all. - for _, log := range logs { - log := log - notifier.Notify(rpcSub.ID, &log) - } + notifyLogsIf(notifier, rpcSub.ID, logs, nil) } case log := <-histLogs: // If ChainReorg is detected, we need to deliver the last block's full logs @@ -472,6 +460,15 @@ func (api *FilterAPI) doHistLogs(from int64, crit FilterCriteria, histLogs chan< } } +func notifyLogsIf(notifier notifier, id rpc.ID, logs []*types.Log, cond func(log *types.Log) bool) { + for _, log := range logs { + if cond == nil || cond(log) { + log := log + notifier.Notify(id, &log) + } + } +} + // FilterCriteria represents a request to create a new filter. // Same as ethereum.FilterQuery but with UnmarshalJSON() method. type FilterCriteria ethereum.FilterQuery From b196709758857a718cf4ed897f9a8f77a17f868a Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Fri, 11 Aug 2023 21:31:06 +0200 Subject: [PATCH 53/70] send logs of each block together Signed-off-by: jsvisa --- eth/filters/api.go | 22 ++++++++++++++-------- eth/filters/filter.go | 2 +- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/eth/filters/api.go b/eth/filters/api.go index 2efc9ed264..a8ae1dbf45 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -335,7 +335,7 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from // 2. if a reorg occurs before the currently delivered block, then we need to stop the historical delivery, and send all replaced logs instead var ( liveLogs = make(chan []*types.Log) - histLogs = make(chan *types.Log) + histLogs = make(chan []*types.Log) histDone = make(chan error) closeC = make(chan struct{}) ) @@ -394,12 +394,18 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from // Send them all. notifyLogsIf(notifier, rpcSub.ID, logs, nil) } - case log := <-histLogs: - // If ChainReorg is detected, we need to deliver the last block's full logs - if !reorged || delivered == log.BlockNumber { - delivered = log.BlockNumber + case logs := <-histLogs: + if len(logs) == 0 { + continue + } + if reorged { + continue + } + for _, log := range logs { notifier.Notify(rpcSub.ID, &log) } + // Assuming batch = all logs of a single block + delivered = logs[0].BlockNumber case <-rpcSub.Err(): // client send an unsubscribe request liveLogsSub.Unsubscribe() closeC <- struct{}{} @@ -416,7 +422,7 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from } // doHistLogs retrieves the logs older than current header, and forward them to the histLogs channel. -func (api *FilterAPI) doHistLogs(from int64, crit FilterCriteria, histLogs chan<- *types.Log, closeC chan struct{}) error { +func (api *FilterAPI) doHistLogs(from int64, crit FilterCriteria, histLogs chan<- []*types.Log, closeC chan struct{}) error { // The original request ctx will be canceled as soon as the parent goroutine // returns a subscription. Use a new context instead. ctx, cancel := context.WithCancel(context.Background()) @@ -430,8 +436,8 @@ func (api *FilterAPI) doHistLogs(from int64, crit FilterCriteria, histLogs chan< case <-closeC: cancel() return nil - case log := <-logsCh: - histLogs <- log + case logs := <-logsCh: + histLogs <- logs case err := <-errChan: if err != nil { logger.Error("Error while fetching historical logs", "err", err) diff --git a/eth/filters/filter.go b/eth/filters/filter.go index 1a9918d0ee..6d0dd321cc 100644 --- a/eth/filters/filter.go +++ b/eth/filters/filter.go @@ -32,7 +32,7 @@ import ( "github.com/ethereum/go-ethereum/rpc" ) -// Filter can be used to retrieve and filter logs. +// Filter can be used to retrieve and filter historical logs. type Filter struct { sys *FilterSystem From d1f2277b5d2b16d62c531ecbcb768ee8a8d80987 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Fri, 11 Aug 2023 21:36:00 +0200 Subject: [PATCH 54/70] fix fetch range err handling Signed-off-by: jsvisa --- eth/filters/api.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/eth/filters/api.go b/eth/filters/api.go index a8ae1dbf45..2e8b390aeb 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -345,7 +345,7 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from } go func() { - histDone <- api.doHistLogs(from, crit, histLogs, closeC) + histDone <- api.doHistLogs(from, crit.Addresses, crit.Topics, histLogs, closeC) }() // Compose and notify the logs from liveLogs and histLogs @@ -377,7 +377,7 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from } reorgBlock := logs[0].BlockNumber if !reorged && reorgBlock <= delivered { - logger.Debug("Reorg detected", "reorgBlock", reorgBlock, "delivered", delivered) + logger.Info("Reorg detected", "reorgBlock", reorgBlock, "delivered", delivered) reorged = true } if !reorged { @@ -422,14 +422,14 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from } // doHistLogs retrieves the logs older than current header, and forward them to the histLogs channel. -func (api *FilterAPI) doHistLogs(from int64, crit FilterCriteria, histLogs chan<- []*types.Log, closeC chan struct{}) error { +func (api *FilterAPI) doHistLogs(from int64, addrs []common.Address, topics [][]common.Hash, histLogs chan<- []*types.Log, closeC chan struct{}) error { // The original request ctx will be canceled as soon as the parent goroutine // returns a subscription. Use a new context instead. ctx, cancel := context.WithCancel(context.Background()) // Fetch logs from a range of blocks. fetchRange := func(from, to int64) error { - f := api.sys.NewRangeFilter(from, to, crit.Addresses, crit.Topics) + f := api.sys.NewRangeFilter(from, to, addrs, topics) logsCh, errChan := f.rangeLogsAsync(ctx) for { select { @@ -457,10 +457,12 @@ func (api *FilterAPI) doHistLogs(from int64, crit FilterCriteria, histLogs chan< } head := header.Number.Int64() if from > head { - logger.Debug("Finish historical sync", "from", from, "head", head) + logger.Info("Finish historical sync", "from", from, "head", head) return nil } - fetchRange(from, head) + if err := fetchRange(from, head); err != nil { + return err + } // Move forward to the next batch from = head + 1 } From 7f30c9e4d1bfe9acd716cc37714ee434c7fafa63 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Mon, 14 Aug 2023 14:16:00 +0300 Subject: [PATCH 55/70] fix future reorged log edge case Signed-off-by: jsvisa --- eth/filters/api.go | 68 ++++++++++++++++++++++++++++------------------ 1 file changed, 42 insertions(+), 26 deletions(-) diff --git a/eth/filters/api.go b/eth/filters/api.go index 2e8b390aeb..af8d59a91e 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -36,11 +36,12 @@ import ( ) var ( - errInvalidTopic = errors.New("invalid topic(s)") - errFilterNotFound = errors.New("filter not found") - errConnectDropped = errors.New("connection dropped") - errInvalidToBlock = errors.New("log subscription does not support history block range") - errInvalidFromBlock = errors.New("from block can be only a number, or \"safe\", or \"finalized\"") + errInvalidTopic = errors.New("invalid topic(s)") + errFilterNotFound = errors.New("filter not found") + errConnectDropped = errors.New("connection dropped") + errInvalidToBlock = errors.New("log subscription does not support history block range") + errInvalidFromBlock = errors.New("from block can be only a number, or \"safe\", or \"finalized\"") + errClientUnsubscribed = errors.New("client unsubscribed") ) // filter is a helper struct that holds meta information over the filter type @@ -337,15 +338,17 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from liveLogs = make(chan []*types.Log) histLogs = make(chan []*types.Log) histDone = make(chan error) - closeC = make(chan struct{}) ) liveLogsSub, err := api.events.SubscribeLogs(ethereum.FilterQuery(crit), liveLogs) if err != nil { return err } + // The original request ctx will be canceled as soon as the parent goroutine + // returns a subscription. Use a new context instead. + ctx, cancel := context.WithCancel(context.Background()) go func() { - histDone <- api.doHistLogs(from, crit.Addresses, crit.Topics, histLogs, closeC) + histDone <- api.doHistLogs(ctx, from, crit.Addresses, crit.Topics, histLogs) }() // Compose and notify the logs from liveLogs and histLogs @@ -354,6 +357,7 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from delivered uint64 liveOnly bool reorged bool + staleHash common.Hash ) for { select { @@ -381,6 +385,10 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from reorged = true } if !reorged { + // Reorg in future. Remember fork point. + if logs[0].Removed && staleHash == (common.Hash{}) { + staleHash = logs[0].BlockHash + } continue } if logs[0].Removed { @@ -401,18 +409,29 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from if reorged { continue } - for _, log := range logs { - notifier.Notify(rpcSub.ID, &log) + if logs[0].BlockHash == staleHash { + // We have reached the fork point and the historical producer + // is emitting old logs because of delay. Restart the process + // from last delivered block. + logger.Info("Restarting historical logs delivery", "from", logs[0].BlockNumber, "delivered", delivered) + liveLogsSub.Unsubscribe() + // Stop hist logs fetcher + cancel() + if err := api.histLogs(notifier, rpcSub, int64(logs[0].BlockNumber), crit); err != nil { + logger.Warn("failed to restart historical logs delivery", "err", err) + } + return } + notifyLogsIf(notifier, rpcSub.ID, logs, nil) // Assuming batch = all logs of a single block delivered = logs[0].BlockNumber case <-rpcSub.Err(): // client send an unsubscribe request liveLogsSub.Unsubscribe() - closeC <- struct{}{} + cancel() return case <-notifier.Closed(): // connection dropped liveLogsSub.Unsubscribe() - closeC <- struct{}{} + cancel() return } } @@ -422,29 +441,22 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from } // doHistLogs retrieves the logs older than current header, and forward them to the histLogs channel. -func (api *FilterAPI) doHistLogs(from int64, addrs []common.Address, topics [][]common.Hash, histLogs chan<- []*types.Log, closeC chan struct{}) error { - // The original request ctx will be canceled as soon as the parent goroutine - // returns a subscription. Use a new context instead. - ctx, cancel := context.WithCancel(context.Background()) - +func (api *FilterAPI) doHistLogs(ctx context.Context, from int64, addrs []common.Address, topics [][]common.Hash, histLogs chan<- []*types.Log) error { // Fetch logs from a range of blocks. fetchRange := func(from, to int64) error { f := api.sys.NewRangeFilter(from, to, addrs, topics) logsCh, errChan := f.rangeLogsAsync(ctx) for { select { - case <-closeC: - cancel() - return nil case logs := <-logsCh: - histLogs <- logs - case err := <-errChan: - if err != nil { - logger.Error("Error while fetching historical logs", "err", err) - return err + select { + case histLogs <- logs: + case <-ctx.Done(): + // Flush out all logs until the range filter voluntarily exits. + continue } - // Range filter is done. - return nil + case err := <-errChan: + return err } } } @@ -461,6 +473,10 @@ func (api *FilterAPI) doHistLogs(from int64, addrs []common.Address, topics [][] return nil } if err := fetchRange(from, head); err != nil { + if errors.Is(err, context.Canceled) { + logger.Info("Historical logs delivery canceled", "from", from, "to", head) + return nil + } return err } // Move forward to the next batch From 3aa0f5bad26e8b57b17b9beaa34553cbe0386f39 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Tue, 29 Aug 2023 17:32:42 +0200 Subject: [PATCH 56/70] minor Signed-off-by: jsvisa --- eth/filters/api.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/eth/filters/api.go b/eth/filters/api.go index af8d59a91e..1fd0307a80 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -353,6 +353,10 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from // Compose and notify the logs from liveLogs and histLogs go func() { + defer func() { + liveLogsSub.Unsubscribe() + cancel() + }() var ( delivered uint64 liveOnly bool @@ -364,7 +368,6 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from case err := <-histDone: if err != nil { logger.Warn("History logs delivery failed", "err", err) - liveLogsSub.Unsubscribe() return } // Else historical logs are all delivered, let's switch to live mode @@ -426,12 +429,8 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from // Assuming batch = all logs of a single block delivered = logs[0].BlockNumber case <-rpcSub.Err(): // client send an unsubscribe request - liveLogsSub.Unsubscribe() - cancel() return case <-notifier.Closed(): // connection dropped - liveLogsSub.Unsubscribe() - cancel() return } } From 743cd0bc326743c4064bc5f0aaa1a9fd3a80061b Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 14 Sep 2023 10:46:20 +0200 Subject: [PATCH 57/70] eth/filters: add some newlines Signed-off-by: jsvisa --- eth/filters/api.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/eth/filters/api.go b/eth/filters/api.go index 1fd0307a80..ef2351613f 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -374,6 +374,7 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from logger.Info("History logs delivery finished, and now enter into live mode", "delivered", delivered) liveOnly = true histLogs = nil + case logs := <-liveLogs: if len(logs) == 0 { continue @@ -405,6 +406,7 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from // Send them all. notifyLogsIf(notifier, rpcSub.ID, logs, nil) } + case logs := <-histLogs: if len(logs) == 0 { continue @@ -428,6 +430,7 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from notifyLogsIf(notifier, rpcSub.ID, logs, nil) // Assuming batch = all logs of a single block delivered = logs[0].BlockNumber + case <-rpcSub.Err(): // client send an unsubscribe request return case <-notifier.Closed(): // connection dropped From be43dcebac91ceb76874ddc7be39c85832347c60 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Mon, 18 Sep 2023 15:41:52 +0200 Subject: [PATCH 58/70] rm reorg param, re-use liveOnly Signed-off-by: jsvisa --- eth/filters/api.go | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/eth/filters/api.go b/eth/filters/api.go index ef2351613f..dd79bee214 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -359,8 +359,12 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from }() var ( delivered uint64 - liveOnly bool - reorged bool + // liveOnly is true when all historical logs are delivered + // and we switch-over to solely returning live logs. + liveOnly bool + // reorged is true when a reorg is detected. + // Question is if its enough to keep one flag (liveMode). + //reorged bool staleHash common.Hash ) for { @@ -379,20 +383,14 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from if len(logs) == 0 { continue } - if liveOnly { - notifyLogsIf(notifier, rpcSub.ID, logs, nil) - continue - } reorgBlock := logs[0].BlockNumber - if !reorged && reorgBlock <= delivered { + if !liveOnly && reorgBlock <= delivered { logger.Info("Reorg detected", "reorgBlock", reorgBlock, "delivered", delivered) - reorged = true + liveOnly = true } - if !reorged { + if !liveOnly && logs[0].Removed && staleHash == (common.Hash{}) { // Reorg in future. Remember fork point. - if logs[0].Removed && staleHash == (common.Hash{}) { - staleHash = logs[0].BlockHash - } + staleHash = logs[0].BlockHash continue } if logs[0].Removed { @@ -411,7 +409,7 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from if len(logs) == 0 { continue } - if reorged { + if liveOnly { continue } if logs[0].BlockHash == staleHash { From 99188a60b81b3eb598064914b9e75fddcac84204 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Mon, 18 Sep 2023 18:34:40 +0200 Subject: [PATCH 59/70] refactor Signed-off-by: jsvisa --- eth/filters/api.go | 64 +++++++++++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 27 deletions(-) diff --git a/eth/filters/api.go b/eth/filters/api.go index dd79bee214..d33367f650 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -358,14 +358,16 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from cancel() }() var ( + // delivered is the block number of the last historical log delivered. delivered uint64 - // liveOnly is true when all historical logs are delivered - // and we switch-over to solely returning live logs. - liveOnly bool - // reorged is true when a reorg is detected. - // Question is if its enough to keep one flag (liveMode). - //reorged bool - staleHash common.Hash + // liveMode is true when either: + // - all historical logs are delivered. + // - or, during history processing a reorg is detected. + liveMode bool + // reorgedBlockHash is the block hash of the reorg point. It is set when + // a reorg is detected in the future. It is used to detect if the history + // processor is sending stale logs. + reorgedBlockHash common.Hash ) for { select { @@ -376,43 +378,51 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from } // Else historical logs are all delivered, let's switch to live mode logger.Info("History logs delivery finished, and now enter into live mode", "delivered", delivered) - liveOnly = true + // TODO: It's theoretically possible that we miss logs due to + // asynchrony between the history processor and the chain subscription. + liveMode = true histLogs = nil case logs := <-liveLogs: if len(logs) == 0 { continue } - reorgBlock := logs[0].BlockNumber - if !liveOnly && reorgBlock <= delivered { - logger.Info("Reorg detected", "reorgBlock", reorgBlock, "delivered", delivered) - liveOnly = true - } - if !liveOnly && logs[0].Removed && staleHash == (common.Hash{}) { - // Reorg in future. Remember fork point. - staleHash = logs[0].BlockHash - continue - } - if logs[0].Removed { - // Send removed logs notification up until the point - // we have delivered logs from old chain. + // TODO: further reorgs are possible during history processing. + if !liveMode && logs[0].BlockNumber <= delivered { + // History is being processed and a reorg is encountered. + // From this point we ignore historical logs coming in and + // only send logs from the chain subscription. + logger.Info("Reorg detected", "reorgBlock", logs[0].BlockNumber, "delivered", delivered) + liveMode = true + // On reorg the chain will send first the removed logs. + // Send them up to the block we've delivered historical logs for. notifyLogsIf(notifier, rpcSub.ID, logs, func(log *types.Log) bool { return log.BlockNumber <= delivered }) - } else { - // New logs are emitted for the whole new chain since the reorg block. - // Send them all. - notifyLogsIf(notifier, rpcSub.ID, logs, nil) + continue } + if !liveMode { + if logs[0].Removed && reorgedBlockHash == (common.Hash{}) { + // Reorg in future. Remember fork point. + reorgedBlockHash = logs[0].BlockHash + } + // Implicit cases: + // - there was a reorg in future and blockchain is sending logs from the new chain. + // - history is still being processed and blockchain sends logs from the tip. + continue + } + // New logs are emitted for the whole new chain since the reorg block. + // Send them all. + notifyLogsIf(notifier, rpcSub.ID, logs, nil) case logs := <-histLogs: if len(logs) == 0 { continue } - if liveOnly { + if liveMode { continue } - if logs[0].BlockHash == staleHash { + if logs[0].BlockHash == reorgedBlockHash { // We have reached the fork point and the historical producer // is emitting old logs because of delay. Restart the process // from last delivered block. From b274824681bd460ca5bb90ea7747fdd5811d5e91 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Wed, 20 Sep 2023 00:02:32 +0200 Subject: [PATCH 60/70] track delivered blocks Signed-off-by: jsvisa --- eth/filters/api.go | 72 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 59 insertions(+), 13 deletions(-) diff --git a/eth/filters/api.go b/eth/filters/api.go index d33367f650..cfd48bc7cb 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -28,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/common/lru" "github.com/ethereum/go-ethereum/core/history" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/internal/ethapi" @@ -44,6 +45,11 @@ var ( errClientUnsubscribed = errors.New("client unsubscribed") ) +const ( + // maxTrackedBlocks is the number of block hashes that will be tracked by subscription. + maxTrackedBlocks = 32 * 1024 +) + // filter is a helper struct that holds meta information over the filter type // and associated subscription in the event system. type filter struct { @@ -312,6 +318,7 @@ func (api *FilterAPI) liveLogs(notifier notifier, rpcSub *rpc.Subscription, crit select { case logs := <-matchedLogs: notifyLogsIf(notifier, rpcSub.ID, logs, nil) + case <-rpcSub.Err(): // client send an unsubscribe request logsSub.Unsubscribe() return @@ -360,14 +367,21 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from var ( // delivered is the block number of the last historical log delivered. delivered uint64 + // liveMode is true when either: // - all historical logs are delivered. // - or, during history processing a reorg is detected. liveMode bool + // reorgedBlockHash is the block hash of the reorg point. It is set when // a reorg is detected in the future. It is used to detect if the history // processor is sending stale logs. reorgedBlockHash common.Hash + + // hashes is used to track the hashes of the blocks that have been delivered. + // It is used as a guard to prevent duplicate logs as well as inaccurate "removed" + // logs being delivered during a reorg. + hashes = lru.NewBasicLRU[common.Hash, struct{}](maxTrackedBlocks) ) for { select { @@ -394,12 +408,6 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from // only send logs from the chain subscription. logger.Info("Reorg detected", "reorgBlock", logs[0].BlockNumber, "delivered", delivered) liveMode = true - // On reorg the chain will send first the removed logs. - // Send them up to the block we've delivered historical logs for. - notifyLogsIf(notifier, rpcSub.ID, logs, func(log *types.Log) bool { - return log.BlockNumber <= delivered - }) - continue } if !liveMode { if logs[0].Removed && reorgedBlockHash == (common.Hash{}) { @@ -411,9 +419,8 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from // - history is still being processed and blockchain sends logs from the tip. continue } - // New logs are emitted for the whole new chain since the reorg block. - // Send them all. - notifyLogsIf(notifier, rpcSub.ID, logs, nil) + // Removed logs from reorged chain, replacing logs or logs from tip of the chain. + notifyLogsIf(notifier, rpcSub.ID, logs, &hashes) case logs := <-histLogs: if len(logs) == 0 { @@ -435,7 +442,7 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from } return } - notifyLogsIf(notifier, rpcSub.ID, logs, nil) + notifyLogsIf(notifier, rpcSub.ID, logs, &hashes) // Assuming batch = all logs of a single block delivered = logs[0].BlockNumber @@ -494,9 +501,48 @@ func (api *FilterAPI) doHistLogs(ctx context.Context, from int64, addrs []common } } -func notifyLogsIf(notifier notifier, id rpc.ID, logs []*types.Log, cond func(log *types.Log) bool) { - for _, log := range logs { - if cond == nil || cond(log) { +// notifyLogsIf sends logs to the notifier if the condition is met. +// It assumes all logs of the same block are either all removed or all added. +func notifyLogsIf(notifier notifier, id rpc.ID, logs []*types.Log, hashes *lru.BasicLRU[common.Hash, struct{}]) { + // Iterate logs and batch them by block hash. + type batch struct { + start int + end int + hash common.Hash + removed bool + } + var ( + batches = make([]batch, 0) + h common.Hash + ) + for i, log := range logs { + if h == log.BlockHash { + // Skip logs of seen block + continue + } + if len(batches) > 0 { + batches[len(batches)-1].end = i + } + batches = append(batches, batch{start: i, hash: log.BlockHash, removed: log.Removed}) + h = log.BlockHash + } + // Close off last batch. + if batches[len(batches)-1].end == 0 { + batches[len(batches)-1].end = len(logs) + } + for _, batch := range batches { + if hashes != nil { + // During reorgs it's possible that logs from the new chain have been delivered. + // Avoid sending removed logs from the old chain and duplicate logs from new chain. + if batch.removed && !hashes.Contains(batch.hash) { + continue + } + if !batch.removed && hashes.Contains(batch.hash) { + continue + } + hashes.Add(batch.hash, struct{}{}) + } + for _, log := range logs[batch.start:batch.end] { log := log notifier.Notify(id, &log) } From dccdc11afc79572c554946ba21bc0282c387816c Mon Sep 17 00:00:00 2001 From: jsvisa Date: Tue, 9 Sep 2025 09:20:01 +0800 Subject: [PATCH 61/70] resolve conflicts --- eth/filters/api.go | 65 +++++++++++++++++++++---------- eth/filters/filter_system.go | 6 +-- eth/filters/filter_system_test.go | 62 ++++++++++++++++++++++------- eth/filters/filter_test.go | 2 +- 4 files changed, 95 insertions(+), 40 deletions(-) diff --git a/eth/filters/api.go b/eth/filters/api.go index cfd48bc7cb..445fa54650 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -37,17 +37,30 @@ import ( ) var ( - errInvalidTopic = errors.New("invalid topic(s)") - errFilterNotFound = errors.New("filter not found") - errConnectDropped = errors.New("connection dropped") - errInvalidToBlock = errors.New("log subscription does not support history block range") - errInvalidFromBlock = errors.New("from block can be only a number, or \"safe\", or \"finalized\"") - errClientUnsubscribed = errors.New("client unsubscribed") + errInvalidTopic = errors.New("invalid topic(s)") + errFilterNotFound = errors.New("filter not found") + errConnectDropped = errors.New("connection dropped") + errInvalidToBlock = errors.New("log subscription does not support history block range") + errInvalidFromBlock = errors.New("from block can be only a number, or \"safe\", or \"finalized\"") + errClientUnsubscribed = errors.New("client unsubscribed") + errExceedMaxTopics = errors.New("exceeds max topics") + errExceedMaxAddresses = errors.New("exceeds max addresses") + errBlockHashWithRange = errors.New("cannot specify block hash with block range") + errInvalidBlockRange = errors.New("invalid block range") + errUnknownBlock = errors.New("unknown block") + errPendingLogsUnsupported = errors.New("pending logs are not supported") + errInvalidFromTo = errors.New("invalid from and to block combination: from > to") ) const ( // maxTrackedBlocks is the number of block hashes that will be tracked by subscription. maxTrackedBlocks = 32 * 1024 + // maxTopics is the maximum number of topics that can be specified in a filter + maxTopics = 4 + // maxSubTopics is the maximum number of sub-topics that can be specified in a single topic position + maxSubTopics = 1000 + // maxAddresses is the maximum number of addresses that can be specified in a filter + maxAddresses = 1000 ) // filter is a helper struct that holds meta information over the filter type @@ -261,7 +274,6 @@ func (api *FilterAPI) NewHeads(ctx context.Context) (*rpc.Subscription, error) { // used in unit testing. type notifier interface { Notify(id rpc.ID, data interface{}) error - Closed() <-chan interface{} } // Logs creates a subscription that fires for all historical @@ -322,9 +334,6 @@ func (api *FilterAPI) liveLogs(notifier notifier, rpcSub *rpc.Subscription, crit case <-rpcSub.Err(): // client send an unsubscribe request logsSub.Unsubscribe() return - case <-notifier.Closed(): // connection dropped - logsSub.Unsubscribe() - return } } }() @@ -448,8 +457,6 @@ func (api *FilterAPI) histLogs(notifier notifier, rpcSub *rpc.Subscription, from case <-rpcSub.Err(): // client send an unsubscribe request return - case <-notifier.Closed(): // connection dropped - return } } }() @@ -462,20 +469,36 @@ func (api *FilterAPI) doHistLogs(ctx context.Context, from int64, addrs []common // Fetch logs from a range of blocks. fetchRange := func(from, to int64) error { f := api.sys.NewRangeFilter(from, to, addrs, topics) - logsCh, errChan := f.rangeLogsAsync(ctx) - for { - select { - case logs := <-logsCh: + logs, err := f.Logs(ctx) + if err != nil { + return err + } + // Group logs by block and send them in batches + var currentBlockLogs []*types.Log + var currentBlockNumber uint64 + + for _, log := range logs { + if currentBlockNumber != log.BlockNumber && len(currentBlockLogs) > 0 { + // Send the current block's logs select { - case histLogs <- logs: + case histLogs <- currentBlockLogs: case <-ctx.Done(): - // Flush out all logs until the range filter voluntarily exits. - continue + return ctx.Err() } - case err := <-errChan: - return err + currentBlockLogs = nil + } + currentBlockLogs = append(currentBlockLogs, log) + currentBlockNumber = log.BlockNumber + } + // Send remaining logs if any + if len(currentBlockLogs) > 0 { + select { + case histLogs <- currentBlockLogs: + case <-ctx.Done(): + return ctx.Err() } } + return nil } for { diff --git a/eth/filters/filter_system.go b/eth/filters/filter_system.go index c5f8318826..f635217402 100644 --- a/eth/filters/filter_system.go +++ b/eth/filters/filter_system.go @@ -39,10 +39,6 @@ import ( "github.com/ethereum/go-ethereum/rpc" ) -var ( - errInvalidFromTo = errors.New("invalid from and to block combination: from > to") -) - // Config represents the configuration of the filter system. type Config struct { LogCacheSize int // maximum number of cached blocks (default: 32) @@ -156,6 +152,8 @@ const ( UnknownSubscription Type = iota // LogsSubscription queries for new or removed (chain reorg) logs LogsSubscription + // MinedAndPendingLogsSubscription queries for both mined and pending logs + MinedAndPendingLogsSubscription // PendingTransactionsSubscription queries for pending transactions entering // the pending state PendingTransactionsSubscription diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index a361487aa7..2dab89e02a 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -32,7 +32,6 @@ import ( "github.com/ethereum/go-ethereum/core/filtermaps" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" @@ -40,7 +39,7 @@ import ( logger "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" - "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/triedb" ) type testBackend struct { @@ -185,7 +184,34 @@ func (b *testBackend) forwardLogEvents(logCh chan []*types.Log, removedLogCh cha }() } -func newTestFilterSystem(t testing.TB, db ethdb.Database, cfg Config) (*testBackend, *FilterSystem) { +func (b *testBackend) startFilterMaps(history uint64, disabled bool, params filtermaps.Params) { + head := b.CurrentBlock() + chainView := filtermaps.NewChainView(b, head.Number.Uint64(), head.Hash()) + config := filtermaps.Config{ + History: history, + Disabled: disabled, + ExportFileName: "", + } + b.fm, _ = filtermaps.NewFilterMaps(b.db, chainView, 0, 0, params, config) + b.fm.Start() + b.fm.WaitIdle() +} + +func (b *testBackend) stopFilterMaps() { + b.fm.Stop() + b.fm = nil +} + +func (b *testBackend) setPending(block *types.Block, receipts types.Receipts) { + b.pendingBlock = block + b.pendingReceipts = receipts +} + +func (b *testBackend) HistoryPruningCutoff() uint64 { + return 0 +} + +func newTestFilterSystem(db ethdb.Database, cfg Config) (*testBackend, *FilterSystem) { backend := &testBackend{db: db} sys := NewFilterSystem(backend, cfg) return backend, sys @@ -717,6 +743,7 @@ func TestLogsSubscription(t *testing.T) { var ( db = rawdb.NewMemoryDatabase() + tdb = triedb.NewDatabase(db, triedb.HashDefaults) signer = types.HomesteadSigner{} key, _ = crypto.GenerateKey() addr = crypto.PubkeyToAddress(key.PublicKey) @@ -742,7 +769,7 @@ func TestLogsSubscription(t *testing.T) { // Hack: GenerateChainWithGenesis creates a new db. // Commit the genesis manually and use GenerateChain. - _, err := genesis.Commit(db, trie.NewDatabase(db)) + _, err := genesis.Commit(db, tdb) if err != nil { t.Fatal(err) } @@ -752,7 +779,7 @@ func TestLogsSubscription(t *testing.T) { tx, _ := types.SignTx(types.NewTx(&types.LegacyTx{Nonce: uint64(i), To: &contract, Value: big.NewInt(0), Gas: 46000, GasPrice: b.BaseFee(), Data: common.FromHex(data)}), signer, key) b.AddTx(tx) }) - bc, err := core.NewBlockChain(db, nil, genesis, nil, ethash.NewFaker(), vm.Config{}, nil, new(uint64)) + bc, err := core.NewBlockChain(db, genesis, ethash.NewFaker(), nil) if err != nil { t.Fatal(err) } @@ -771,12 +798,15 @@ func TestLogsSubscription(t *testing.T) { }) liveLogs := preceipts[0][0].Logs var ( - backend, sys = newTestFilterSystem(t, db, Config{}) - api = NewFilterAPI(sys, false) + backend, sys = newTestFilterSystem(db, Config{}) + api = NewFilterAPI(sys) // Transfer(address indexed from, address indexed to, uint256 value); topic = common.HexToHash("0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef") ) + backend.startFilterMaps(0, false, filtermaps.RangeTestParams) + defer backend.stopFilterMaps() + i2h := func(i int) common.Hash { return common.BigToHash(big.NewInt(int64(i))) } allLogs := []*types.Log{ @@ -886,9 +916,8 @@ func TestLogsSubscription(t *testing.T) { backend.logsFeed.Send(liveLogs) for i := range testCases { - err := <-testCases[i].err - if err != nil { - t.Fatalf("test %d failed: %v", i, err) + if err := <-testCases[i].err; err != nil { + t.Errorf("test %d failed: %v", i, err) } } } @@ -971,19 +1000,20 @@ func calculateBalance(logs []*types.Log) map[common.Address]uint64 { func testLogsSubscriptionReorg(t *testing.T, oldChainMaker, newChainMaker, pendingMaker func(i int, b *core.BlockGen), oldChainLen, newChainLen, forkAt, reorgAt int, expected []*types.Log) { var ( db = rawdb.NewMemoryDatabase() - backend, sys = newTestFilterSystem(t, db, Config{}) - api = NewFilterAPI(sys, false) + tdb = triedb.NewDatabase(db, triedb.HashDefaults) + backend, sys = newTestFilterSystem(db, Config{}) + api = NewFilterAPI(sys) genesis = &core.Genesis{Config: params.TestChainConfig, Alloc: genesisAlloc, GasLimit: 10000000000000} ) // Hack: GenerateChainWithGenesis creates a new db. // Commit the genesis manually and use GenerateChain. - _, err := genesis.Commit(db, trie.NewDatabase(db)) + _, err := genesis.Commit(db, tdb) if err != nil { t.Fatal(err) } oldChain, _ := core.GenerateChain(genesis.Config, genesis.ToBlock(), ethash.NewFaker(), db, oldChainLen, oldChainMaker) - bc, err := core.NewBlockChain(db, nil, genesis, nil, ethash.NewFaker(), vm.Config{}, nil, new(uint64)) + bc, err := core.NewBlockChain(db, genesis, ethash.NewFaker(), nil) if err != nil { t.Fatal(err) } @@ -1001,6 +1031,10 @@ func testLogsSubscriptionReorg(t *testing.T, oldChainMaker, newChainMaker, pendi if err != nil { t.Fatal(err) } + + backend.startFilterMaps(0, false, filtermaps.RangeTestParams) + defer backend.stopFilterMaps() + newChain, _ := core.GenerateChain(genesis.Config, oldChain[forkAt], ethash.NewFaker(), db, newChainLen, newChainMaker) logger.Info("oldChain/newChain", "oldChain", fmt.Sprintf("%d..%d", oldChain[0].Number(), oldChain[len(oldChain)-1].Number()), "newChain", fmt.Sprintf("%d..%d", newChain[0].Number(), newChain[len(newChain)-1].Number())) diff --git a/eth/filters/filter_test.go b/eth/filters/filter_test.go index e99b2f6caa..b05b0d888c 100644 --- a/eth/filters/filter_test.go +++ b/eth/filters/filter_test.go @@ -456,7 +456,7 @@ func TestRangeLogs(t *testing.T) { ) expEvent := func(expEvent int, expFirst, expAfterLast uint64) { - exp := rangeLogsTestEvent{expEvent, common.NewRange[uint64](expFirst, expAfterLast-expFirst)} + exp := rangeLogsTestEvent{expEvent, common.NewRange(expFirst, expAfterLast-expFirst)} event++ ev := <-filter.rangeLogsTestHook if ev != exp { From 1682d4d66bb1227d11c86a846a22fdef59abc146 Mon Sep 17 00:00:00 2001 From: jsvisa Date: Tue, 9 Sep 2025 14:43:19 +0800 Subject: [PATCH 62/70] fix tests --- eth/filters/filter_system_test.go | 74 ++++++++++--------------------- 1 file changed, 24 insertions(+), 50 deletions(-) diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index 2dab89e02a..56dfffbebc 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -712,14 +712,6 @@ func TestPendingTxFilterDeadlock(t *testing.T) { } } -func flattenLogs(pl [][]*types.Log) []*types.Log { - var logs []*types.Log - for _, l := range pl { - logs = append(logs, l...) - } - return logs -} - type mockNotifier struct { c chan interface{} } @@ -733,38 +725,16 @@ func (n *mockNotifier) Notify(id rpc.ID, data interface{}) error { return nil } -func (n *mockNotifier) Closed() <-chan interface{} { - return nil -} - // TestLogsSubscription tests if a rpc subscription receives the correct logs func TestLogsSubscription(t *testing.T) { t.Parallel() var ( - db = rawdb.NewMemoryDatabase() - tdb = triedb.NewDatabase(db, triedb.HashDefaults) - signer = types.HomesteadSigner{} - key, _ = crypto.GenerateKey() - addr = crypto.PubkeyToAddress(key.PublicKey) - contract = common.HexToAddress("0000000000000000000000000000000000031ec7") - genesis = &core.Genesis{ - Config: params.TestChainConfig, - Alloc: core.GenesisAlloc{ - // // SPDX-License-Identifier: GPL-3.0 - // pragma solidity >=0.7.0 <0.9.0; - // - // contract Token { - // event Transfer(address indexed from, address indexed to, uint256 value); - // function transfer(address to, uint256 value) public returns (bool) { - // emit Transfer(msg.sender, to, value); - // return true; - // } - // } - contract: {Balance: big.NewInt(params.Ether), Code: common.FromHex("0x608060405234801561001057600080fd5b506004361061002b5760003560e01c8063a9059cbb14610030575b600080fd5b61004a6004803603810190610045919061016a565b610060565b60405161005791906101c5565b60405180910390f35b60008273ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff167fddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef846040516100bf91906101ef565b60405180910390a36001905092915050565b600080fd5b60073ffffffffffffffffffffffffffffffffffffffff82169050919050565b6000610101826100d6565b9050919050565b610111816100f6565b811461011c57600080fd5b50565b60008135905061012e81610108565b92915050565b6000819050919050565b61014781610134565b811461015257600080fd5b50565b6000813590506101648161013e565b92915050565b60008060408385031215610181576101806100d1565b5b600061018f8582860161011f565b92505060206101a085828601610155565b9150509250929050565b60008115159050919050565b6101bf816101aa565b82525050565b60006020820190506101da60008301846101b6565b92915050565b6101e981610134565b82525050565b600060208201905061020460008301846101e0565b9291505056fea2646970667358221220b469033f4b77b9565ee84e0a2f04d496b18160d26034d54f9487e57788fd36d564736f6c63430008120033")}, - addr: {Balance: big.NewInt(params.Ether)}, - }, - } + db = rawdb.NewMemoryDatabase() + tdb = triedb.NewDatabase(db, triedb.HashDefaults) + genesis = &core.Genesis{Config: params.TestChainConfig, Alloc: genesisAlloc} + backend, sys = newTestFilterSystem(db, Config{}) + api = NewFilterAPI(sys) ) // Hack: GenerateChainWithGenesis creates a new db. @@ -783,11 +753,24 @@ func TestLogsSubscription(t *testing.T) { if err != nil { t.Fatal(err) } + + // Hack: FilterSystem is using mock backend, i.e. blockchain events will be not received + // by it. Forward them manually. + var ( + bcLogsFeed = make(chan []*types.Log) + bcRmLogsFeed = make(chan core.RemovedLogsEvent) + ) + backend.forwardLogEvents(bcLogsFeed, bcRmLogsFeed) + bc.SubscribeLogsEvent(bcLogsFeed) + _, err = bc.InsertChain(blocks) if err != nil { t.Fatal(err) } + backend.startFilterMaps(0, false, filtermaps.RangeTestParams) + defer backend.stopFilterMaps() + // Generate pending block, logs for which // will be sent to subscription feed. _, preceipts := core.GenerateChain(genesis.Config, blocks[len(blocks)-1], ethash.NewFaker(), db, 1, func(i int, gen *core.BlockGen) { @@ -797,23 +780,12 @@ func TestLogsSubscription(t *testing.T) { gen.AddTx(tx) }) liveLogs := preceipts[0][0].Logs - var ( - backend, sys = newTestFilterSystem(db, Config{}) - api = NewFilterAPI(sys) - // Transfer(address indexed from, address indexed to, uint256 value); - topic = common.HexToHash("0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef") - ) - - backend.startFilterMaps(0, false, filtermaps.RangeTestParams) - defer backend.stopFilterMaps() - - i2h := func(i int) common.Hash { return common.BigToHash(big.NewInt(int64(i))) } allLogs := []*types.Log{ - {Address: contract, Topics: []common.Hash{topic, common.HexToHash(addr.Hex()), i2h(1)}, Data: i2h(11).Bytes(), BlockNumber: 1, BlockHash: blocks[0].Hash(), TxHash: blocks[0].Transactions()[0].Hash()}, - {Address: contract, Topics: []common.Hash{topic, common.HexToHash(addr.Hex()), i2h(2)}, Data: i2h(12).Bytes(), BlockNumber: 2, BlockHash: blocks[1].Hash(), TxHash: blocks[1].Transactions()[0].Hash()}, - {Address: contract, Topics: []common.Hash{topic, common.HexToHash(addr.Hex()), i2h(3)}, Data: i2h(13).Bytes(), BlockNumber: 3, BlockHash: blocks[2].Hash(), TxHash: blocks[2].Transactions()[0].Hash()}, - {Address: contract, Topics: []common.Hash{topic, common.HexToHash(addr.Hex()), i2h(4)}, Data: i2h(14).Bytes(), BlockNumber: 4, BlockHash: blocks[3].Hash(), TxHash: blocks[3].Transactions()[0].Hash()}, + {Address: contract, Topics: []common.Hash{topic, addrHash, i2h(1)}, Data: i2b(11), BlockNumber: 1, BlockTimestamp: 10, BlockHash: blocks[0].Hash(), TxHash: blocks[0].Transactions()[0].Hash()}, + {Address: contract, Topics: []common.Hash{topic, addrHash, i2h(2)}, Data: i2b(12), BlockNumber: 2, BlockTimestamp: 20, BlockHash: blocks[1].Hash(), TxHash: blocks[1].Transactions()[0].Hash()}, + {Address: contract, Topics: []common.Hash{topic, addrHash, i2h(3)}, Data: i2b(13), BlockNumber: 3, BlockTimestamp: 30, BlockHash: blocks[2].Hash(), TxHash: blocks[2].Transactions()[0].Hash()}, + {Address: contract, Topics: []common.Hash{topic, addrHash, i2h(4)}, Data: i2b(14), BlockNumber: 4, BlockTimestamp: 40, BlockHash: blocks[3].Hash(), TxHash: blocks[3].Transactions()[0].Hash()}, } testCases := []struct { crit FilterCriteria @@ -926,6 +898,7 @@ var ( signer = types.HomesteadSigner{} key, _ = crypto.GenerateKey() addr = crypto.PubkeyToAddress(key.PublicKey) + addrHash = common.HexToHash(addr.Hex()) contract = common.HexToAddress("0000000000000000000000000000000000031ec7") // Transfer(address indexed from, address indexed to, uint256 value); topic = common.HexToHash("0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef") @@ -954,6 +927,7 @@ func makeTx(to int, value int, b *core.BlockGen) *types.Transaction { func i2h(i int) common.Hash { return common.BigToHash(big.NewInt(int64(i))) } func a2h(a common.Address) common.Hash { return common.HexToHash(a.Hex()) } +func i2b(i int) []byte { return i2h(i).Bytes() } func parseTransferLog(log *types.Log) (from, to common.Address, amount uint64) { from = common.BytesToAddress(log.Topics[1].Bytes()) From 7e9ce630c9e0728d99f8e5b203e9f6822610ebf0 Mon Sep 17 00:00:00 2001 From: jsvisa Date: Tue, 9 Sep 2025 15:24:53 +0800 Subject: [PATCH 63/70] improve --- eth/filters/api.go | 8 +++--- eth/filters/filter.go | 2 +- eth/filters/filter_system_test.go | 48 +++++++++++++++---------------- 3 files changed, 29 insertions(+), 29 deletions(-) diff --git a/eth/filters/api.go b/eth/filters/api.go index 445fa54650..3b5a9ff0f5 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -53,14 +53,14 @@ var ( ) const ( + // The maximum number of addresses allowed in a filter criteria + maxAddresses = 1000 + // The maximum number of topic criteria allowed, vm.LOG4 - vm.LOG0 + maxTopics = 4 // maxTrackedBlocks is the number of block hashes that will be tracked by subscription. maxTrackedBlocks = 32 * 1024 - // maxTopics is the maximum number of topics that can be specified in a filter - maxTopics = 4 // maxSubTopics is the maximum number of sub-topics that can be specified in a single topic position maxSubTopics = 1000 - // maxAddresses is the maximum number of addresses that can be specified in a filter - maxAddresses = 1000 ) // filter is a helper struct that holds meta information over the filter type diff --git a/eth/filters/filter.go b/eth/filters/filter.go index 6d0dd321cc..1a9918d0ee 100644 --- a/eth/filters/filter.go +++ b/eth/filters/filter.go @@ -32,7 +32,7 @@ import ( "github.com/ethereum/go-ethereum/rpc" ) -// Filter can be used to retrieve and filter historical logs. +// Filter can be used to retrieve and filter logs. type Filter struct { sys *FilterSystem diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index 56dfffbebc..287116d08e 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -725,6 +725,30 @@ func (n *mockNotifier) Notify(id rpc.ID, data interface{}) error { return nil } +var ( + signer = types.HomesteadSigner{} + key, _ = crypto.GenerateKey() + addr = crypto.PubkeyToAddress(key.PublicKey) + addrHash = common.HexToHash(addr.Hex()) + contract = common.HexToAddress("0000000000000000000000000000000000031ec7") + // Transfer(address indexed from, address indexed to, uint256 value); + topic = common.HexToHash("0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef") + genesisAlloc = core.GenesisAlloc{ + // // SPDX-License-Identifier: GPL-3.0 + // pragma solidity >=0.7.0 <0.9.0; + // + // contract Token { + // event Transfer(address indexed from, address indexed to, uint256 value); + // function transfer(address to, uint256 value) public returns (bool) { + // emit Transfer(msg.sender, to, value); + // return true; + // } + // } + contract: {Balance: big.NewInt(params.Ether), Code: common.FromHex("0x608060405234801561001057600080fd5b506004361061002b5760003560e01c8063a9059cbb14610030575b600080fd5b61004a6004803603810190610045919061016a565b610060565b60405161005791906101c5565b60405180910390f35b60008273ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff167fddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef846040516100bf91906101ef565b60405180910390a36001905092915050565b600080fd5b600073ffffffffffffffffffffffffffffffffffffffff82169050919050565b6000610101826100d6565b9050919050565b610111816100f6565b811461011c57600080fd5b50565b60008135905061012e81610108565b92915050565b6000819050919050565b61014781610134565b811461015257600080fd5b50565b6000813590506101648161013e565b92915050565b60008060408385031215610181576101806100d1565b5b600061018f8582860161011f565b92505060206101a085828601610155565b9150509250929050565b60008115159050919050565b6101bf816101aa565b82525050565b60006020820190506101da60008301846101b6565b92915050565b6101e981610134565b82525050565b600060208201905061020460008301846101e0565b9291505056fea2646970667358221220b469033f4b77b9565ee84e0a2f04d496b18160d26034d54f9487e57788fd36d564736f6c63430008120033")}, + addr: {Balance: big.NewInt(params.Ether)}, + } +) + // TestLogsSubscription tests if a rpc subscription receives the correct logs func TestLogsSubscription(t *testing.T) { t.Parallel() @@ -894,30 +918,6 @@ func TestLogsSubscription(t *testing.T) { } } -var ( - signer = types.HomesteadSigner{} - key, _ = crypto.GenerateKey() - addr = crypto.PubkeyToAddress(key.PublicKey) - addrHash = common.HexToHash(addr.Hex()) - contract = common.HexToAddress("0000000000000000000000000000000000031ec7") - // Transfer(address indexed from, address indexed to, uint256 value); - topic = common.HexToHash("0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef") - genesisAlloc = core.GenesisAlloc{ - // // SPDX-License-Identifier: GPL-3.0 - // pragma solidity >=0.7.0 <0.9.0; - // - // contract Token { - // event Transfer(address indexed from, address indexed to, uint256 value); - // function transfer(address to, uint256 value) public returns (bool) { - // emit Transfer(msg.sender, to, value); - // return true; - // } - // } - contract: {Balance: big.NewInt(params.Ether), Code: common.FromHex("0x608060405234801561001057600080fd5b506004361061002b5760003560e01c8063a9059cbb14610030575b600080fd5b61004a6004803603810190610045919061016a565b610060565b60405161005791906101c5565b60405180910390f35b60008273ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff167fddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef846040516100bf91906101ef565b60405180910390a36001905092915050565b600080fd5b600073ffffffffffffffffffffffffffffffffffffffff82169050919050565b6000610101826100d6565b9050919050565b610111816100f6565b811461011c57600080fd5b50565b60008135905061012e81610108565b92915050565b6000819050919050565b61014781610134565b811461015257600080fd5b50565b6000813590506101648161013e565b92915050565b60008060408385031215610181576101806100d1565b5b600061018f8582860161011f565b92505060206101a085828601610155565b9150509250929050565b60008115159050919050565b6101bf816101aa565b82525050565b60006020820190506101da60008301846101b6565b92915050565b6101e981610134565b82525050565b600060208201905061020460008301846101e0565b9291505056fea2646970667358221220b469033f4b77b9565ee84e0a2f04d496b18160d26034d54f9487e57788fd36d564736f6c63430008120033")}, - addr: {Balance: big.NewInt(params.Ether)}, - } -) - func makeTx(to int, value int, b *core.BlockGen) *types.Transaction { // transfer(address to, uint256 value) data := fmt.Sprintf("0xa9059cbb%s%s", common.HexToHash(common.BigToAddress(big.NewInt(int64(to))).Hex()).String()[2:], common.BytesToHash([]byte{byte(value)}).String()[2:]) From f223b5e79891f2eaa1982e6bcad4f36b5eb57fb3 Mon Sep 17 00:00:00 2001 From: jsvisa Date: Tue, 9 Sep 2025 15:55:14 +0800 Subject: [PATCH 64/70] eth/filters: rm not used error variables --- eth/filters/api.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/eth/filters/api.go b/eth/filters/api.go index 3b5a9ff0f5..ff379a8b17 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -39,10 +39,8 @@ import ( var ( errInvalidTopic = errors.New("invalid topic(s)") errFilterNotFound = errors.New("filter not found") - errConnectDropped = errors.New("connection dropped") errInvalidToBlock = errors.New("log subscription does not support history block range") errInvalidFromBlock = errors.New("from block can be only a number, or \"safe\", or \"finalized\"") - errClientUnsubscribed = errors.New("client unsubscribed") errExceedMaxTopics = errors.New("exceeds max topics") errExceedMaxAddresses = errors.New("exceeds max addresses") errBlockHashWithRange = errors.New("cannot specify block hash with block range") From 9d717a1d8cb4459f2cd2eb3b532a6ac171980eb3 Mon Sep 17 00:00:00 2001 From: jsvisa Date: Tue, 9 Sep 2025 16:57:51 +0800 Subject: [PATCH 65/70] wip --- eth/filters/api.go | 18 +++++++++--------- eth/filters/filter_system.go | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/eth/filters/api.go b/eth/filters/api.go index ff379a8b17..4697b60e89 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -39,15 +39,15 @@ import ( var ( errInvalidTopic = errors.New("invalid topic(s)") errFilterNotFound = errors.New("filter not found") - errInvalidToBlock = errors.New("log subscription does not support history block range") - errInvalidFromBlock = errors.New("from block can be only a number, or \"safe\", or \"finalized\"") - errExceedMaxTopics = errors.New("exceeds max topics") - errExceedMaxAddresses = errors.New("exceeds max addresses") - errBlockHashWithRange = errors.New("cannot specify block hash with block range") - errInvalidBlockRange = errors.New("invalid block range") + errInvalidBlockRange = errors.New("invalid block range params") errUnknownBlock = errors.New("unknown block") + errBlockHashWithRange = errors.New("can't specify fromBlock/toBlock with blockHash") errPendingLogsUnsupported = errors.New("pending logs are not supported") - errInvalidFromTo = errors.New("invalid from and to block combination: from > to") + errExceedMaxTopics = errors.New("exceed max topics") + errExceedMaxAddresses = errors.New("exceed max addresses") + errInvalidFromBlock = errors.New("from block can be only a number, or \"safe\", or \"finalized\"") + errInvalidToBlock = errors.New("log subscription does not support history block range") + errInvalidFromToBlock = errors.New("invalid from and to block combination: from > to") ) const ( @@ -55,10 +55,10 @@ const ( maxAddresses = 1000 // The maximum number of topic criteria allowed, vm.LOG4 - vm.LOG0 maxTopics = 4 + // The maximum number of allowed topics within a topic criteria + maxSubTopics = 1000 // maxTrackedBlocks is the number of block hashes that will be tracked by subscription. maxTrackedBlocks = 32 * 1024 - // maxSubTopics is the maximum number of sub-topics that can be specified in a single topic position - maxSubTopics = 1000 ) // filter is a helper struct that holds meta information over the filter type diff --git a/eth/filters/filter_system.go b/eth/filters/filter_system.go index f635217402..cf0213c069 100644 --- a/eth/filters/filter_system.go +++ b/eth/filters/filter_system.go @@ -337,7 +337,7 @@ func (es *EventSystem) SubscribeLogs(crit ethereum.FilterQuery, logs chan []*typ if from >= 0 && to == rpc.LatestBlockNumber { return es.subscribeLogs(crit, logs), nil } - return nil, errInvalidFromTo + return nil, errInvalidFromToBlock } // subscribeMinedPendingLogs creates a subscription that returned mined and From cf7252e2bc3214d8b46688c9a7a7844da25380a0 Mon Sep 17 00:00:00 2001 From: jsvisa Date: Tue, 9 Sep 2025 17:49:49 +0800 Subject: [PATCH 66/70] use maketx --- eth/filters/filter_system_test.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index 287116d08e..7a945b64f9 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -768,9 +768,7 @@ func TestLogsSubscription(t *testing.T) { t.Fatal(err) } blocks, _ := core.GenerateChain(genesis.Config, genesis.ToBlock(), ethash.NewFaker(), db, 4, func(i int, b *core.BlockGen) { - // transfer(address to, uint256 value) - data := fmt.Sprintf("0xa9059cbb%s%s", common.HexToHash(common.BigToAddress(big.NewInt(int64(i + 1))).Hex()).String()[2:], common.BytesToHash([]byte{byte(i + 11)}).String()[2:]) - tx, _ := types.SignTx(types.NewTx(&types.LegacyTx{Nonce: uint64(i), To: &contract, Value: big.NewInt(0), Gas: 46000, GasPrice: b.BaseFee(), Data: common.FromHex(data)}), signer, key) + tx := makeTx(i+1, i+11, b) b.AddTx(tx) }) bc, err := core.NewBlockChain(db, genesis, ethash.NewFaker(), nil) From ba67bef6a7bf598385db6908b2985abd80c42289 Mon Sep 17 00:00:00 2001 From: jsvisa Date: Tue, 9 Sep 2025 19:57:07 +0800 Subject: [PATCH 67/70] use single pointer instead Signed-off-by: jsvisa --- eth/filters/api.go | 3 +-- eth/filters/filter_system_test.go | 5 ++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/eth/filters/api.go b/eth/filters/api.go index 4697b60e89..45010db199 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -564,8 +564,7 @@ func notifyLogsIf(notifier notifier, id rpc.ID, logs []*types.Log, hashes *lru.B hashes.Add(batch.hash, struct{}{}) } for _, log := range logs[batch.start:batch.end] { - log := log - notifier.Notify(id, &log) + notifier.Notify(id, log) } } } diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index 7a945b64f9..8057f524f6 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -880,7 +880,7 @@ func TestLogsSubscription(t *testing.T) { for { select { case log := <-tt.notifier.c: - fetched = append(fetched, *log.(**types.Log)) + fetched = append(fetched, log.(*types.Log)) case <-timeout: break fetchLoop } @@ -941,7 +941,6 @@ func parseTransferLog(log *types.Log) (from, to common.Address, amount uint64) { func calculateBalance(logs []*types.Log) map[common.Address]uint64 { balances := make(map[common.Address]uint64) for _, log := range logs { - log := log from, to, amount := parseTransferLog(log) if _, ok := balances[from]; !ok { @@ -1042,7 +1041,7 @@ func testLogsSubscriptionReorg(t *testing.T, oldChainMaker, newChainMaker, pendi for { select { case log := <-notifier.c: - l := *log.(**types.Log) + l := log.(*types.Log) fetched = append(fetched, l) // We halt the sender by blocking Notify(). However sender will already prepare From 2079198ce2e3bfee45617f916775a53a530876df Mon Sep 17 00:00:00 2001 From: jsvisa Date: Wed, 10 Sep 2025 02:37:28 +0000 Subject: [PATCH 68/70] eth/filter: handle historical in chunk Signed-off-by: jsvisa --- eth/filters/api.go | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/eth/filters/api.go b/eth/filters/api.go index 45010db199..370fcb8f88 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -59,6 +59,9 @@ const ( maxSubTopics = 1000 // maxTrackedBlocks is the number of block hashes that will be tracked by subscription. maxTrackedBlocks = 32 * 1024 + // histLogChunkSize is the maximum number of blocks to process in a single chunk + // when fetching historical logs to improve responsiveness + histLogChunkSize = 4096 ) // filter is a helper struct that holds meta information over the filter type @@ -510,15 +513,28 @@ func (api *FilterAPI) doHistLogs(ctx context.Context, from int64, addrs []common logger.Info("Finish historical sync", "from", from, "head", head) return nil } - if err := fetchRange(from, head); err != nil { + + to := min(from+histLogChunkSize-1, head) + + logger.Debug("Processing historical logs chunk", "from", from, "to", to, "head", head) + if err := fetchRange(from, to); err != nil { if errors.Is(err, context.Canceled) { - logger.Info("Historical logs delivery canceled", "from", from, "to", head) + logger.Warn("Historical logs delivery canceled", "from", from, "to", to) return nil } return err } - // Move forward to the next batch - from = head + 1 + + // Move forward to the next chunk + from = to + 1 + + // Allow other goroutines to run and check for cancellation + select { + case <-ctx.Done(): + logger.Warn("Historical logs delivery canceled", "from", from, "to", to) + return ctx.Err() + default: + } } } From 7f90a0fff7a9fa5f956c27e57f333eb86b4c2871 Mon Sep 17 00:00:00 2001 From: jsvisa Date: Thu, 11 Sep 2025 15:58:46 +0800 Subject: [PATCH 69/70] ethclient: set from block to nil if q.from is nil --- ethclient/ethclient.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ethclient/ethclient.go b/ethclient/ethclient.go index f4b4a33ee1..ffe5d79075 100644 --- a/ethclient/ethclient.go +++ b/ethclient/ethclient.go @@ -479,6 +479,10 @@ func (ec *Client) SubscribeFilterLogs(ctx context.Context, q ethereum.FilterQuer // TODO(s1na): Log subscription prohibits ToBlock being set. // This will be later re-allowed once it supports historical-only queries. arg["toBlock"] = nil + // For subscriptions, if FromBlock is nil, we want only live logs, not historical logs from genesis + if q.FromBlock == nil { + arg["fromBlock"] = "latest" + } sub, err := ec.c.EthSubscribe(ctx, ch, "logs", arg) if err != nil { // Defensively prefer returning nil interface explicitly on error-path, instead From 0859bd75b7f4bc214b226cc5e23d748ce5b426e1 Mon Sep 17 00:00:00 2001 From: jsvisa Date: Thu, 11 Sep 2025 16:02:23 +0800 Subject: [PATCH 70/70] eth: latest subscription --- eth/filters/api.go | 8 +++++--- eth/filters/filter_system_test.go | 5 +++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/eth/filters/api.go b/eth/filters/api.go index 370fcb8f88..2360164884 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -45,7 +45,7 @@ var ( errPendingLogsUnsupported = errors.New("pending logs are not supported") errExceedMaxTopics = errors.New("exceed max topics") errExceedMaxAddresses = errors.New("exceed max addresses") - errInvalidFromBlock = errors.New("from block can be only a number, or \"safe\", or \"finalized\"") + errInvalidFromBlock = errors.New(`from block can be only a number, or "latest", "safe", "finalized"`) errInvalidToBlock = errors.New("log subscription does not support history block range") errInvalidFromToBlock = errors.New("invalid from and to block combination: from > to") ) @@ -291,7 +291,7 @@ func (api *FilterAPI) Logs(ctx context.Context, crit FilterCriteria) (*rpc.Subsc // logs is the inner implementation of logs subscription. // The following criteria are valid: -// * from: nil, to: nil -> yield live logs. +// * from: nil | latest, to: nil -> yield live logs. // * from: blockNum | safe | finalized, to: nil -> historical beginning at `from` to head, then live logs. // * Every other case should fail with an error. func (api *FilterAPI) logs(ctx context.Context, notifier notifier, rpcSub *rpc.Subscription, crit FilterCriteria) error { @@ -303,8 +303,10 @@ func (api *FilterAPI) logs(ctx context.Context, notifier notifier, rpcSub *rpc.S } from := rpc.BlockNumber(crit.FromBlock.Int64()) switch from { - case rpc.LatestBlockNumber, rpc.PendingBlockNumber: + case rpc.PendingBlockNumber: return errInvalidFromBlock + case rpc.LatestBlockNumber: + return api.liveLogs(notifier, rpcSub, crit) case rpc.SafeBlockNumber, rpc.FinalizedBlockNumber: header, err := api.sys.backend.HeaderByNumber(ctx, from) if err != nil { diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index 8057f524f6..5c8233b044 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -847,6 +847,11 @@ func TestLogsSubscription(t *testing.T) { FilterCriteria{FromBlock: big.NewInt(-101)}, nil, newMockNotifier(), &rpc.Subscription{ID: rpc.NewID()}, errInvalidFromBlock, nil, }, + // from "latest" - should work like live logs only + { + FilterCriteria{FromBlock: big.NewInt(rpc.LatestBlockNumber.Int64())}, + liveLogs, newMockNotifier(), &rpc.Subscription{ID: rpc.NewID()}, nil, nil, + }, } // subscribe logs