From 4a8eaeb1d179336be443c4411b072357892a2b3f Mon Sep 17 00:00:00 2001 From: zsfelfoldi Date: Mon, 28 Jul 2025 23:41:15 +0200 Subject: [PATCH] core/filtermaps: implement log indexer as core.Indexer --- common/range.go | 12 +- core/filtermaps/chain_view.go | 31 +- core/filtermaps/checkpoints.go | 2 +- core/filtermaps/checkpoints_holesky.json | 52 +- core/filtermaps/checkpoints_mainnet.json | 580 ++++++------- core/filtermaps/checkpoints_sepolia.json | 194 ++--- core/filtermaps/filtermaps.go | 889 -------------------- core/filtermaps/index_view.go | 274 +++++++ core/filtermaps/indexer.go | 989 ++++++++++++++--------- core/filtermaps/indexer_test.go | 637 --------------- core/filtermaps/map_database.go | 575 +++++++++++++ core/filtermaps/map_database_test.go | 79 ++ core/filtermaps/map_renderer.go | 815 ------------------- core/filtermaps/map_storage.go | 711 ++++++++++++++++ core/filtermaps/map_storage_test.go | 150 ++++ core/filtermaps/matcher.go | 171 ++-- core/filtermaps/matcher_backend.go | 201 ----- core/filtermaps/matcher_test.go | 2 + core/filtermaps/math.go | 327 ++++++-- core/filtermaps/math_test.go | 2 +- core/filtermaps/memory_map.go | 171 ++++ core/filtermaps/test_helpers.go | 226 ++++++ core/rawdb/accessors_indexes.go | 42 +- core/rawdb/schema.go | 11 +- eth/api_backend.go | 6 +- eth/backend.go | 83 +- eth/filters/filter.go | 347 +++----- eth/filters/filter_system.go | 4 +- eth/filters/filter_system_test.go | 40 +- eth/filters/filter_test.go | 185 +---- internal/ethapi/api_test.go | 4 +- internal/ethapi/backend.go | 4 +- internal/ethapi/transaction_args_test.go | 4 +- 33 files changed, 3774 insertions(+), 4046 deletions(-) delete mode 100644 core/filtermaps/filtermaps.go create mode 100644 core/filtermaps/index_view.go delete mode 100644 core/filtermaps/indexer_test.go create mode 100644 core/filtermaps/map_database.go create mode 100644 core/filtermaps/map_database_test.go delete mode 100644 core/filtermaps/map_renderer.go create mode 100644 core/filtermaps/map_storage.go create mode 100644 core/filtermaps/map_storage_test.go delete mode 100644 core/filtermaps/matcher_backend.go create mode 100644 core/filtermaps/memory_map.go create mode 100644 core/filtermaps/test_helpers.go diff --git a/common/range.go b/common/range.go index c3a26ea7f5..aefd0c16d9 100644 --- a/common/range.go +++ b/common/range.go @@ -27,7 +27,11 @@ type Range[T uint32 | uint64] struct { // NewRange creates a new range based of first element and number of elements. func NewRange[T uint32 | uint64](first, count T) Range[T] { - return Range[T]{first, first + count} + afterLast := first + count + if afterLast < first { + panic("range overflow") + } + return Range[T]{first, afterLast} } // First returns the first element of the range. @@ -97,6 +101,12 @@ func (r Range[T]) Intersection(q Range[T]) Range[T] { // Union returns the union of two ranges. Panics for gapped ranges. func (r Range[T]) Union(q Range[T]) Range[T] { + if r.IsEmpty() { + return q + } + if q.IsEmpty() { + return r + } if max(r.first, q.first) > min(r.afterLast, q.afterLast) { panic("cannot create union; gap between ranges") } diff --git a/core/filtermaps/chain_view.go b/core/filtermaps/chain_view.go index 35c5ed22a5..bbc60fe313 100644 --- a/core/filtermaps/chain_view.go +++ b/core/filtermaps/chain_view.go @@ -32,12 +32,11 @@ type blockchain interface { GetRawReceipts(hash common.Hash, number uint64) types.Receipts } -// ChainView represents an immutable view of a chain with a block id and a set +// ChainView represents an immutable view of a chain with a block hash and a set // of receipts associated to each block number and a block hash associated with // all block numbers except the head block. This is because in the future // ChainView might represent a view where the head block is currently being -// created. Block id is a unique identifier that can also be calculated for the -// head block. +// created. // Note that the view's head does not have to be the current canonical head // of the underlying blockchain, it should only possess the block headers // and receipts up until the expected chain view head. @@ -79,21 +78,6 @@ func (cv *ChainView) BlockHash(number uint64) common.Hash { return cv.blockHash(number) } -// BlockId returns the unique block id belonging to the given block number. -// Note that it is currently equal to the block hash. In the future it might -// be a different id for future blocks if the log index root becomes part of -// consensus and therefore rendering the index with the new head will happen -// before the hash of that new head is available. -func (cv *ChainView) BlockId(number uint64) common.Hash { - cv.lock.Lock() - defer cv.lock.Unlock() - - if number > cv.headNumber { - panic("invalid block number") - } - return cv.blockHash(number) -} - // Header returns the block header at the given block number. func (cv *ChainView) Header(number uint64) *types.Header { return cv.chain.GetHeader(cv.BlockHash(number), number) @@ -128,7 +112,7 @@ func (cv *ChainView) SharedRange(cv2 *ChainView) common.Range[uint64] { return common.Range[uint64]{} } sharedLen := min(cv.headNumber, cv2.headNumber) + 1 - for sharedLen > 0 && cv.BlockId(sharedLen-1) != cv2.BlockId(sharedLen-1) { + for sharedLen > 0 && cv.BlockHash(sharedLen-1) != cv2.BlockHash(sharedLen-1) { sharedLen-- } return common.NewRange(0, sharedLen) @@ -139,7 +123,7 @@ func equalViews(cv1, cv2 *ChainView) bool { if cv1 == nil || cv2 == nil { return false } - return cv1.headNumber == cv2.headNumber && cv1.BlockId(cv1.headNumber) == cv2.BlockId(cv2.headNumber) + return cv1.headNumber == cv2.headNumber && cv1.BlockHash(cv1.headNumber) == cv2.BlockHash(cv2.headNumber) } // matchViews returns true if the two chain views are equivalent up until the @@ -152,12 +136,7 @@ func matchViews(cv1, cv2 *ChainView, number uint64) bool { if cv1.headNumber < number || cv2.headNumber < number { return false } - var h1, h2 common.Hash - if number == cv1.headNumber || number == cv2.headNumber { - h1, h2 = cv1.BlockId(number), cv2.BlockId(number) - } else { - h1, h2 = cv1.BlockHash(number), cv2.BlockHash(number) - } + h1, h2 := cv1.BlockHash(number), cv2.BlockHash(number) return h1 == h2 && h1 != common.Hash{} } diff --git a/core/filtermaps/checkpoints.go b/core/filtermaps/checkpoints.go index face4ea5d3..a3d86d685e 100644 --- a/core/filtermaps/checkpoints.go +++ b/core/filtermaps/checkpoints.go @@ -33,7 +33,7 @@ type checkpointList []epochCheckpoint // be initialized at the epoch boundary. type epochCheckpoint struct { BlockNumber uint64 // block that generated the last log value of the given epoch - BlockId common.Hash + BlockHash common.Hash FirstIndex uint64 // first log value index of the given block } diff --git a/core/filtermaps/checkpoints_holesky.json b/core/filtermaps/checkpoints_holesky.json index 481c71a114..31948d48b3 100644 --- a/core/filtermaps/checkpoints_holesky.json +++ b/core/filtermaps/checkpoints_holesky.json @@ -1,28 +1,28 @@ [ -{"blockNumber": 814410, "blockId": "0x6c38f0d4ff2c23ae187f581cebb0963c0ec2dbf051b289de1582c369c98a3244", "firstIndex": 67107349}, -{"blockNumber": 914268, "blockId": "0xeff161573a11eb6e2ab930674a5245b2c5ffc5e9e63093503344ec6fa60578a3", "firstIndex": 134216064}, -{"blockNumber": 1048866, "blockId": "0xebd3b95415dad9ab7f2b1d25e48c791e299063c09427bbde54217029c3e215ad", "firstIndex": 201325914}, -{"blockNumber": 1144433, "blockId": "0x0c895438ef12a2c835b4372c209e40a499ba2b18ed0e658846813784de391392", "firstIndex": 268434935}, -{"blockNumber": 1230406, "blockId": "0xd3512e7241efc9853e46b1ad565403d302f62457b6dd84917c31ff375fabf487", "firstIndex": 335544096}, -{"blockNumber": 1309104, "blockId": "0xcf108c6e002e7a5657bf7587adde03edf5f20d03e95b66a194eb8080daaf4f97", "firstIndex": 402653143}, -{"blockNumber": 1380499, "blockId": "0x984b0f8b6bf06f1240bbca8c1df9bef3880bedafd5b5c2a55cbc2f64c9efb974", "firstIndex": 469759822}, -{"blockNumber": 1476950, "blockId": "0x8323b528bb8d80a96b172de92452be0f45078a9ca311cb4be6675f089e25a426", "firstIndex": 536870335}, -{"blockNumber": 1533506, "blockId": "0x737dc416070aa3b522a0c2ab813a70c1820f062c718f27597394f309f0004106", "firstIndex": 603979618}, -{"blockNumber": 1613764, "blockId": "0x9d6a5a505afdda86b1ebcc2b10cb687a1f89c2e2b74315945a3ddc6a0b3c9cd6", "firstIndex": 671088253}, -{"blockNumber": 1719073, "blockId": "0x17bf6a51d9c55a908c3e9e652f80b2aa464b049c697abf3bd5a537e461aff0b9", "firstIndex": 738197366}, -{"blockNumber": 1973133, "blockId": "0x14b288d70e688de3334d09283670301aacfed21c193da8a56fd63767f8177705", "firstIndex": 805305624}, -{"blockNumber": 2274761, "blockId": "0xf02790b273b04219e001d2fcc93e8e47708cb228364e96ad754bc8050d08a9aa", "firstIndex": 872414871}, -{"blockNumber": 2530470, "blockId": "0x157ea98b1a7e66dd3f045da8e25219800671f9a000211d3d94006efd33d89a80", "firstIndex": 939523234}, -{"blockNumber": 2781706, "blockId": "0xa723663a584e280700d2302eba03d1b3b6549333db70c6152576d33e2911f594", "firstIndex": 1006632936}, -{"blockNumber": 3101669, "blockId": "0xa6e66a18fed64d33178c7088f273c404be597662c34f6e6884cf2e24f3ea4ace", "firstIndex": 1073741353}, -{"blockNumber": 3221725, "blockId": "0xe771f897dece48b1583cc1d1d10de8015da57407eb1fdf239fdbe46eaab85143", "firstIndex": 1140850137}, -{"blockNumber": 3357164, "blockId": "0x6252d0aa54c79623b0680069c88d7b5c47983f0d5c4845b6c811b8d9b5e8ff3c", "firstIndex": 1207959453}, -{"blockNumber": 3447019, "blockId": "0xeb7d585e1e063f3cc05ed399fbf6c2df63c271f62f030acb804e9fb1e74b6dc1", "firstIndex": 1275067542}, -{"blockNumber": 3546397, "blockId": "0xdabdef7defa4281180a57c5af121877b82274f15ccf074ea0096146f4c246df2", "firstIndex": 1342176778}, -{"blockNumber": 3867885, "blockId": "0x8be069dd7a3e2ffb869ee164d11b28555233d2510b134ab9d5484fdae55d2225", "firstIndex": 1409285539}, -{"blockNumber": 3935446, "blockId": "0xc91a61bc215bbcccc3020c62e5c8153162df0d8bcc59813d74671b2d24903ed7", "firstIndex": 1476394742}, -{"blockNumber": 3989508, "blockId": "0xc85dec36a767e44237842ef51915944c2a49780c8c394a3aa6cfb013c99cf58b", "firstIndex": 1543503452}, -{"blockNumber": 4057078, "blockId": "0xccdb79f6705629cb6ab1667a1244938f60911236549143fcff23a3989213e67e", "firstIndex": 1610612030}, -{"blockNumber": 4126499, "blockId": "0x92f2ef21fc911e87e81e38373d5f2915587b9648a0ab3cf4fcfe3e5aaffe7b85", "firstIndex": 1677720416}, -{"blockNumber": 4239335, "blockId": "0x64fbd22965eb583a584552b7edb9b7ce26fb6aad247c1063d0d5a4d11cbcc58c", "firstIndex": 1744830176} +{"blockNumber": 814410, "blockHash": "0x6c38f0d4ff2c23ae187f581cebb0963c0ec2dbf051b289de1582c369c98a3244", "firstIndex": 67107349}, +{"blockNumber": 914268, "blockHash": "0xeff161573a11eb6e2ab930674a5245b2c5ffc5e9e63093503344ec6fa60578a3", "firstIndex": 134216064}, +{"blockNumber": 1048866, "blockHash": "0xebd3b95415dad9ab7f2b1d25e48c791e299063c09427bbde54217029c3e215ad", "firstIndex": 201325914}, +{"blockNumber": 1144433, "blockHash": "0x0c895438ef12a2c835b4372c209e40a499ba2b18ed0e658846813784de391392", "firstIndex": 268434935}, +{"blockNumber": 1230406, "blockHash": "0xd3512e7241efc9853e46b1ad565403d302f62457b6dd84917c31ff375fabf487", "firstIndex": 335544096}, +{"blockNumber": 1309104, "blockHash": "0xcf108c6e002e7a5657bf7587adde03edf5f20d03e95b66a194eb8080daaf4f97", "firstIndex": 402653143}, +{"blockNumber": 1380499, "blockHash": "0x984b0f8b6bf06f1240bbca8c1df9bef3880bedafd5b5c2a55cbc2f64c9efb974", "firstIndex": 469759822}, +{"blockNumber": 1476950, "blockHash": "0x8323b528bb8d80a96b172de92452be0f45078a9ca311cb4be6675f089e25a426", "firstIndex": 536870335}, +{"blockNumber": 1533506, "blockHash": "0x737dc416070aa3b522a0c2ab813a70c1820f062c718f27597394f309f0004106", "firstIndex": 603979618}, +{"blockNumber": 1613764, "blockHash": "0x9d6a5a505afdda86b1ebcc2b10cb687a1f89c2e2b74315945a3ddc6a0b3c9cd6", "firstIndex": 671088253}, +{"blockNumber": 1719073, "blockHash": "0x17bf6a51d9c55a908c3e9e652f80b2aa464b049c697abf3bd5a537e461aff0b9", "firstIndex": 738197366}, +{"blockNumber": 1973133, "blockHash": "0x14b288d70e688de3334d09283670301aacfed21c193da8a56fd63767f8177705", "firstIndex": 805305624}, +{"blockNumber": 2274761, "blockHash": "0xf02790b273b04219e001d2fcc93e8e47708cb228364e96ad754bc8050d08a9aa", "firstIndex": 872414871}, +{"blockNumber": 2530470, "blockHash": "0x157ea98b1a7e66dd3f045da8e25219800671f9a000211d3d94006efd33d89a80", "firstIndex": 939523234}, +{"blockNumber": 2781706, "blockHash": "0xa723663a584e280700d2302eba03d1b3b6549333db70c6152576d33e2911f594", "firstIndex": 1006632936}, +{"blockNumber": 3101669, "blockHash": "0xa6e66a18fed64d33178c7088f273c404be597662c34f6e6884cf2e24f3ea4ace", "firstIndex": 1073741353}, +{"blockNumber": 3221725, "blockHash": "0xe771f897dece48b1583cc1d1d10de8015da57407eb1fdf239fdbe46eaab85143", "firstIndex": 1140850137}, +{"blockNumber": 3357164, "blockHash": "0x6252d0aa54c79623b0680069c88d7b5c47983f0d5c4845b6c811b8d9b5e8ff3c", "firstIndex": 1207959453}, +{"blockNumber": 3447019, "blockHash": "0xeb7d585e1e063f3cc05ed399fbf6c2df63c271f62f030acb804e9fb1e74b6dc1", "firstIndex": 1275067542}, +{"blockNumber": 3546397, "blockHash": "0xdabdef7defa4281180a57c5af121877b82274f15ccf074ea0096146f4c246df2", "firstIndex": 1342176778}, +{"blockNumber": 3867885, "blockHash": "0x8be069dd7a3e2ffb869ee164d11b28555233d2510b134ab9d5484fdae55d2225", "firstIndex": 1409285539}, +{"blockNumber": 3935446, "blockHash": "0xc91a61bc215bbcccc3020c62e5c8153162df0d8bcc59813d74671b2d24903ed7", "firstIndex": 1476394742}, +{"blockNumber": 3989508, "blockHash": "0xc85dec36a767e44237842ef51915944c2a49780c8c394a3aa6cfb013c99cf58b", "firstIndex": 1543503452}, +{"blockNumber": 4057078, "blockHash": "0xccdb79f6705629cb6ab1667a1244938f60911236549143fcff23a3989213e67e", "firstIndex": 1610612030}, +{"blockNumber": 4126499, "blockHash": "0x92f2ef21fc911e87e81e38373d5f2915587b9648a0ab3cf4fcfe3e5aaffe7b85", "firstIndex": 1677720416}, +{"blockNumber": 4239335, "blockHash": "0x64fbd22965eb583a584552b7edb9b7ce26fb6aad247c1063d0d5a4d11cbcc58c", "firstIndex": 1744830176} ] diff --git a/core/filtermaps/checkpoints_mainnet.json b/core/filtermaps/checkpoints_mainnet.json index 2ea065ddb7..bb84950f86 100644 --- a/core/filtermaps/checkpoints_mainnet.json +++ b/core/filtermaps/checkpoints_mainnet.json @@ -1,292 +1,292 @@ [ -{"blockNumber": 4166212, "blockId": "0xd94b724fc1c7dceb3251b51b81f7a0f3220ae5b9add1e62917004f14f2a3532c", "firstIndex": 67108840}, -{"blockNumber": 4513996, "blockId": "0xc0fd25fef5888609d05fa8c5620d8e18c31cdd675dd4ee926ff966668f0e5d76", "firstIndex": 134217480}, -{"blockNumber": 4817399, "blockId": "0x8573c3166fb7f716e97cf6109d382f86441238e0b85828944a072ae8583a6cfa", "firstIndex": 201326547}, -{"blockNumber": 5087706, "blockId": "0xcd2dfae45901e299b25faa04c10df60983d5c0a3d0e478e789c7aeec980cf691", "firstIndex": 268435422}, -{"blockNumber": 5306085, "blockId": "0xe8026984c85873f2b24018a7e0bdf8c2df136b2fc49666b4addf0d2e067890da", "firstIndex": 335544018}, -{"blockNumber": 5509898, "blockId": "0x41bc63dc8184f9a7d4b9a5a554e00eee32fee60ccfa2339264be11270ac28de1", "firstIndex": 402652789}, -{"blockNumber": 5670367, "blockId": "0xe61c2ba463f2f458817ed1bf0ff9fb9d07e9d7354f46b48175b2d784b817bf3b", "firstIndex": 469761896}, -{"blockNumber": 5826113, "blockId": "0xfbbb1fc5e03a5562bf6b7f1799d7a572d9ef66c7fd0e0b9f4fed7262767b5c86", "firstIndex": 536870852}, -{"blockNumber": 5953008, "blockId": "0x6bbdccd094928e846de3c615a33f2952e3259afaa1929f4ee241a56907d0593b", "firstIndex": 603979150}, -{"blockNumber": 6102812, "blockId": "0x2403f2502088a1fa26c1294f5245032fe3b7f3a8bb14c12d0ba8d577156bbc1b", "firstIndex": 671087962}, -{"blockNumber": 6276672, "blockId": "0xeaa05a0e0574fbb164244628a7d14d9d47f1692941098aa737059d44104401df", "firstIndex": 738197399}, -{"blockNumber": 6448662, "blockId": "0xe6f097fa18a2cef425514e863659e42e1aa8d74df49cb0bed5877e82a08d1f84", "firstIndex": 805306056}, -{"blockNumber": 6655925, "blockId": "0xee5c9d87b06dc0c2e850fead07bcd13f85d45ad6b5a6e7ebee53cc8618772786", "firstIndex": 872415229}, -{"blockNumber": 6873878, "blockId": "0x9a0ea103146da21fa1d9cab7ff609ec2c5e7f1856288a1f744239588e33904c2", "firstIndex": 939524080}, -{"blockNumber": 7080893, "blockId": "0x35ad45055d7a1a3ef94efeb05e4122757bcc126ff1bc2b4ac72dc78ab3c0fd75", "firstIndex": 1006632327}, -{"blockNumber": 7266955, "blockId": "0x7b80f70520f2c16eb5890f4963af8f60446f50af82b589ceaf009d09e704b302", "firstIndex": 1073741608}, -{"blockNumber": 7466649, "blockId": "0xea8980b5c692c729013f9cc4c9f66e94d5ef2c5caad23f370be0953b6d85f32c", "firstIndex": 1140850335}, -{"blockNumber": 7661736, "blockId": "0x615cdb03dbf29a22d59bbc769c03f3647c2c9abb7d1767f3f6529708209e7073", "firstIndex": 1207959232}, -{"blockNumber": 7834494, "blockId": "0x05b7dd13e2eb8a833e4b278f9151ca01b294ae5fc5617769422ec85ffb446215", "firstIndex": 1275068098}, -{"blockNumber": 7989998, "blockId": "0x4b5026add7f2fc2c02993b82c61bceba1b75cfcb8a78c5112cc2832bc0413be6", "firstIndex": 1342177025}, -{"blockNumber": 8142950, "blockId": "0x9f56b7753a0e9964973f3e096e1385215e3aef232b448359e5bec05046b016a3", "firstIndex": 1409285545}, -{"blockNumber": 8297598, "blockId": "0x2ddfeb161fe34ff73c4a995bb94256c3b522b39348169edb97bb3d876ccdf77d", "firstIndex": 1476394898}, -{"blockNumber": 8465061, "blockId": "0xfdbe5435da5ae9fb34a9154e5ebe515afac9a128e383f798f1e69952be404fcd", "firstIndex": 1543503805}, -{"blockNumber": 8655740, "blockId": "0x305137325a39a44aba997e214c05f32d965658f7e3fe20322f2e57e72f239977", "firstIndex": 1610612406}, -{"blockNumber": 8807102, "blockId": "0xc32f9fd9a43d2b502dbcc4a683ffe324fca9a871753478bcb1f4fb9573da96a5", "firstIndex": 1677721343}, -{"blockNumber": 8911110, "blockId": "0x806eabee7dc429b57afcb6c0b72b61728b4f13c62ee9942f1813790463f9ab5b", "firstIndex": 1744830172}, -{"blockNumber": 8960262, "blockId": "0xa69f24032e9d5761565e726d484926a0932c6ca7fdb4c04a55bfc5e8f5879c4d", "firstIndex": 1811938729}, -{"blockNumber": 9085906, "blockId": "0x922b4d6aeca64517f5048fa1ffa98d3e813bc415716a1763d07cf2bdea236896", "firstIndex": 1879048062}, -{"blockNumber": 9230838, "blockId": "0xd892f513c48c95a859ae59c4e00ec4be0d684c549456c90e9c36669c11092f50", "firstIndex": 1946156580}, -{"blockNumber": 9390449, "blockId": "0xcc0ced78fa66b570f338d129794d574560a23958e28d0d5288003d4542cec734", "firstIndex": 2013264376}, -{"blockNumber": 9515554, "blockId": "0xc06d04737813c3e2c8e910b2d4543c5c684c7bfe235bb501445f68fd0663f3d9", "firstIndex": 2080374749}, -{"blockNumber": 9659367, "blockId": "0x765fa33d9418b1ed648e88b497b4bf4aa66990e6413dfc155525cda61e3cc813", "firstIndex": 2147483344}, -{"blockNumber": 9793940, "blockId": "0x9527dcbd85a100c51aee4ba90828fee23312f5f041859f3416a54fd87a437ce2", "firstIndex": 2214591669}, -{"blockNumber": 9923756, "blockId": "0xc4da5fdd2de077459fe84f721d7fc01ef69683e03e634b0c5443186486792246", "firstIndex": 2281700949}, -{"blockNumber": 10042325, "blockId": "0x6f554f195d20e4ba1262f73cf2c1cd44b14a71befe5ef0a5318fc309f8cde9fa", "firstIndex": 2348810131}, -{"blockNumber": 10168768, "blockId": "0xdcdf88ec2c197d552343c80d01375a35d6462aff39c5bd976160f7176d2add64", "firstIndex": 2415918723}, -{"blockNumber": 10289467, "blockId": "0x5ece23babcf6b8838695b891b044c3dcf8120cdc15acd83018e96ffc590f6cd0", "firstIndex": 2483027720}, -{"blockNumber": 10386601, "blockId": "0xd398ee129aea1247019fa39dc2f5083819687d8f5ad17ec39ba1803d178fbcda", "firstIndex": 2550136043}, -{"blockNumber": 10479586, "blockId": "0x61c1cccd60a546eacc29f2ae2a6facab402b4f35e2986c6629b302e6b1ebca3e", "firstIndex": 2617245577}, -{"blockNumber": 10562579, "blockId": "0x14fd6a079050e3e110140372605c050945d6db03746fa5b84a8718cb2a0e9b86", "firstIndex": 2684353802}, -{"blockNumber": 10641438, "blockId": "0x1998dcae09111b2c1bf6d0592e2f0b4a0143fe7db7115ff94b2b73b74542cee4", "firstIndex": 2751463334}, -{"blockNumber": 10717094, "blockId": "0x9ce45a150bb6bd13687d0692c86716911ffc6aff5831a03f4015bdffb85f5093", "firstIndex": 2818571144}, -{"blockNumber": 10784494, "blockId": "0xf65c936cf578e673696477f7db7189b47222fdf7ed13096564b9d47b15c86eb9", "firstIndex": 2885680709}, -{"blockNumber": 10848591, "blockId": "0x84ea8e9d5789302ebe88888115488f91cfb75fd89e282dfbd29ba9d6225ab0ca", "firstIndex": 2952789491}, -{"blockNumber": 10909102, "blockId": "0x22275047f643f4a3b82c349274a53d56a00da25beac89655877fb8cbfa3182c8", "firstIndex": 3019898024}, -{"blockNumber": 10972828, "blockId": "0x38e8acfec19bb53ab9fe70dd3577ccd8bd7c41b4d468b8f099a19bf06ce56518", "firstIndex": 3087006561}, -{"blockNumber": 11036532, "blockId": "0x478dac03f520e987a6e845cac1b3a756d69e4f5257f20758b8fc38456e7b6884", "firstIndex": 3154116463}, -{"blockNumber": 11102457, "blockId": "0x1a36616b03351ebe481330fc86578f57d9da5c0dda59f4406d49b53a2fa9b6f6", "firstIndex": 3221224961}, -{"blockNumber": 11168092, "blockId": "0xb23af483ca6dbc2fb8d2fada1f8189de2e155758ed9ecbbb20c234a30c7e093c", "firstIndex": 3288333510}, -{"blockNumber": 11233640, "blockId": "0xdc4c80d8370823773425b31e70b3af59582b556514857d1e9a9b4c53b880a424", "firstIndex": 3355442901}, -{"blockNumber": 11300454, "blockId": "0xdbd4d8e9c1756093037f431330db4604e5a3e67bb74f425f667633d5298fe34c", "firstIndex": 3422551962}, -{"blockNumber": 11367550, "blockId": "0x9e00816bd5b39353657a99421d278091599a39f246bbf8c0898f83e6d70b7c7c", "firstIndex": 3489660743}, -{"blockNumber": 11431820, "blockId": "0xd32d5e5871ce71acc9dcc7cf9d8cb9d964646182529c327a0ddf617814e27d74", "firstIndex": 3556769150}, -{"blockNumber": 11497031, "blockId": "0xed823054d4b293294e3301d3131e2ae9f0ce63c9f07d2d80fc144e42d535d61d", "firstIndex": 3623878513}, -{"blockNumber": 11560027, "blockId": "0x7098177efe77cfb5912dc63069fe366033111041ca5c539907a89e439c861d8f", "firstIndex": 3690987092}, -{"blockNumber": 11625049, "blockId": "0x61fccb47104a076b905d390498a1b38c317348bb1567e69aa1b670bed7a6a13f", "firstIndex": 3758096043}, -{"blockNumber": 11690298, "blockId": "0xaf6f9e9c92db9e155f1ffb35eb68765c09de1206c353c9a56a5d5fa53e2d88d2", "firstIndex": 3825204430}, -{"blockNumber": 11755003, "blockId": "0xf9efb54f08a2b59086a607541005928f1bda54fa2c46fcc48352fa6aa461c03a", "firstIndex": 3892313541}, -{"blockNumber": 11819591, "blockId": "0xa551deb3bd2ec0ca2683cb76adb8fb295d171a3787aa5ff071bd1fa94d3af82e", "firstIndex": 3959422322}, -{"blockNumber": 11883278, "blockId": "0x60c7a359d4aeb09b63b64487fa7794aee87c41b11446f69ba83e24d5bce6947f", "firstIndex": 4026531239}, -{"blockNumber": 11948436, "blockId": "0x1c46fb22653e5f9eb9503fe532151519845d05b121291766715973e027eb9c0c", "firstIndex": 4093639741}, -{"blockNumber": 12013084, "blockId": "0x0b8d1da9e70c4494ba61190390c891230846e69d9b0be605df8b04c4199ffd1b", "firstIndex": 4160749561}, -{"blockNumber": 12078612, "blockId": "0xb049af260be25c6153b67c227689b0556d8fb7d6033c3f61b8218f5c4ac5917a", "firstIndex": 4227857310}, -{"blockNumber": 12143549, "blockId": "0x685dce41efbe2bab0ac9a4b9026d9e5faa3e01ae4637c8181cd2f1eac326fe6a", "firstIndex": 4294966232}, -{"blockNumber": 12207918, "blockId": "0x53644ea8f781c6270f1136ed4bbcb956c98f12a4793c2f56f49b3105282b13ea", "firstIndex": 4362075410}, -{"blockNumber": 12272370, "blockId": "0x503523e1a01cb7d68be0eb22f36280c0ee7a1fc802061a704e326fbb8efcfe27", "firstIndex": 4429184831}, -{"blockNumber": 12329342, "blockId": "0xe6a1bf78dd827def078e23c768cf6e96666ee06b5fdc113380a4769d85f4bd33", "firstIndex": 4496293151}, -{"blockNumber": 12382311, "blockId": "0x352fe793c16b1d8aea195af86ee9aec3dbaa83e7a32872a0d222a77e96fc0e57", "firstIndex": 4563402486}, -{"blockNumber": 12436970, "blockId": "0x90276349edf0941bb3327dc1d749d0371fe550eaddafb5141fcacdd281de0976", "firstIndex": 4630510187}, -{"blockNumber": 12489954, "blockId": "0x71894b1e1608aeba777662a6055e601ce6f17950dbe3844c3601e0f88cbd5848", "firstIndex": 4697619586}, -{"blockNumber": 12541655, "blockId": "0xc34c7299041df72a81c4b86e81639d2dfcebfae603e287fc143881a636ce1188", "firstIndex": 4764728948}, -{"blockNumber": 12597319, "blockId": "0x4c4dc71d8d2f1223ad0469ddfe09fa8934f64e3712ed83708d6e1b9c64eceebb", "firstIndex": 4831837390}, -{"blockNumber": 12651863, "blockId": "0x406a642c8364fb61cbd896512a2d19ecdba3932ab09ac67cf48d91aa1a03f24e", "firstIndex": 4898946362}, -{"blockNumber": 12706391, "blockId": "0xa7fc3739898f7ff5733896313b248fb0b306a132681609b8b9e2671b42dac3a1", "firstIndex": 4966054748}, -{"blockNumber": 12762840, "blockId": "0x5a1048966dfa55a48248517c45683ce342671bf544117d2bac13d8e343563136", "firstIndex": 5033164303}, -{"blockNumber": 12816597, "blockId": "0x97f5b87f422870a8ffeff70c12f4377dde96d248aa912cb64ebfaea51dc086b6", "firstIndex": 5100272541}, -{"blockNumber": 12872315, "blockId": "0x432f7df7973fdea4e5abe2c09b634ca3b724e37a19d5ec23ca5810e4854c6b33", "firstIndex": 5167382243}, -{"blockNumber": 12929620, "blockId": "0xa4524e3b4c5dadac91df2f29696532cfed377f47469fe7cfabf10baac945eea2", "firstIndex": 5234490748}, -{"blockNumber": 12988653, "blockId": "0x9be70e31aeea164c370d8d183a1af45120693b13d360edc78214535c40ef1d63", "firstIndex": 5301600248}, -{"blockNumber": 13049074, "blockId": "0x3076857bd17959c2a3df18357eab3ecbb7ca25457785b07074d2e885eb566fc2", "firstIndex": 5368707746}, -{"blockNumber": 13108823, "blockId": "0x67b2764f6b53779e3d683def2c7659c6d92814152574073d24f10cd67200a22b", "firstIndex": 5435817680}, -{"blockNumber": 13175389, "blockId": "0x0d41e34dbca58dec45e0509b6cbf027b2377220845a85caebbf3a14b8505e8ce", "firstIndex": 5502925085}, -{"blockNumber": 13237624, "blockId": "0x723e0818d54ab1b1c53efc2f2f448c16c1f943af2033575c612c3c71228d4ddf", "firstIndex": 5570034948}, -{"blockNumber": 13298658, "blockId": "0xd7f9f477b20bcf1379ccaaae701c9658a02b177861e0af9e873c63fe8b854f4b", "firstIndex": 5637144562}, -{"blockNumber": 13361166, "blockId": "0x9f9e4130093269d59045f91344a13eb2786023e02099f53dd44854d50a7f5ffb", "firstIndex": 5704253260}, -{"blockNumber": 13421716, "blockId": "0x28752f35ad2adeba342ef6aa51bca1c0abc757c4f712cf620c8a5e6a8576636e", "firstIndex": 5771361775}, -{"blockNumber": 13480516, "blockId": "0x43f4ecf24c17b7a77621e2ca09c88c53b3407fb018f4bb3a032c212015b4e18d", "firstIndex": 5838469616}, -{"blockNumber": 13535688, "blockId": "0x99ffcd9982219a4be45fa5f15622b288afe4f85da536956d39dccac1ec416444", "firstIndex": 5905579739}, -{"blockNumber": 13590487, "blockId": "0xff7b74464a318037e0670c0be7a5b1b79418d1a42c7a3587c46adc1ef720c819", "firstIndex": 5972688013}, -{"blockNumber": 13646596, "blockId": "0xd555ce5f2b65ffeec9b9e3e75a56f5f23d7353a1014bcc861efd7df6e5dc2c92", "firstIndex": 6039797525}, -{"blockNumber": 13702902, "blockId": "0x3e8feab590c6631c63931fee83bf7e10234890bdf377b1b374fb70221b9ac9c0", "firstIndex": 6106905641}, -{"blockNumber": 13760908, "blockId": "0x18e96fa585c57aad1f2a8dc9639d9f4fe53c3f38894862471b7cec17f97f3cb1", "firstIndex": 6174014330}, -{"blockNumber": 13819340, "blockId": "0x25384bf2cef178e30c4d9e0abae59357055241fc715133b2bb078e8278e5296a", "firstIndex": 6241123906}, -{"blockNumber": 13877817, "blockId": "0x4a84a0a7035dbfed0e1281fb1df162a195f2c516686a2e2a6ad605e6f2b3cf57", "firstIndex": 6308232690}, -{"blockNumber": 13935260, "blockId": "0x6c14a8e563614edbad66698ebe911e997e27844f9d33c7069795379540e71a44", "firstIndex": 6375341148}, -{"blockNumber": 13993914, "blockId": "0xc0991d1bc431c5215fb0218b0b541f2b4fae0994775e5eddb016965fe5b4a0c7", "firstIndex": 6442448947}, -{"blockNumber": 14051007, "blockId": "0x854803ce364b6fe4446800e74448a5ee883e7d472e8a0347f1ad13bd347fa69f", "firstIndex": 6509558964}, -{"blockNumber": 14109063, "blockId": "0xa3fadce7ab18a5fedcfed9e3b1f0cf2fb32151172c362327c602343bc26dde8f", "firstIndex": 6576668017}, -{"blockNumber": 14166701, "blockId": "0xaa2443a38e2b0d6b01cf4b73de54d2b7e92c99475bd0802beebd86845c126d1b", "firstIndex": 6643774648}, -{"blockNumber": 14222367, "blockId": "0xa27c78134c1fbf432c8caee7c9d2ef323fdde519a065804c9c8da6ca1615e04f", "firstIndex": 6710886238}, -{"blockNumber": 14277029, "blockId": "0x5d0ed08b65db01e382c339dcb2715b2927fb472b0c1f4256841c386d785c3f3a", "firstIndex": 6777995247}, -{"blockNumber": 14330955, "blockId": "0x3e5014d82c416549acfe2e26b13ff5ddf29ad74943ef5b94a65eca0a4d72d82e", "firstIndex": 6845102969}, -{"blockNumber": 14384139, "blockId": "0x720e1fd687292019c3fa1812c39c73d20fde783c0196f43378a51cb4e0b46d65", "firstIndex": 6912212078}, -{"blockNumber": 14437534, "blockId": "0xe4342ab6f363066d9c2514a870241ceb63df232d33659fdb3928faf433360bb4", "firstIndex": 6979321493}, -{"blockNumber": 14489073, "blockId": "0xeb2f9923d816e773460eccfa42eb4ff2bbfc353b756ba274e98dc85c64ab1536", "firstIndex": 7046428879}, -{"blockNumber": 14541559, "blockId": "0x55cc33721fa9022964f47c30f4146fee7cc80f5b4158d7ecea0ff661cbba3487", "firstIndex": 7113538857}, -{"blockNumber": 14594432, "blockId": "0x676441bdc1ec430a9fbc05e56a0437a475606a253f9fdffc86f5f6286c159b5d", "firstIndex": 7180647956}, -{"blockNumber": 14644942, "blockId": "0x03232a1d3c9b2f1f0c2a4b780c83bce66238d9abdeed71c2a8af9e97a53011b0", "firstIndex": 7247754981}, -{"blockNumber": 14695793, "blockId": "0x6017f6d1d79188692e1a1a18de5f93b83b97905c8ff5bed5cef4b2e9f4d4d41b", "firstIndex": 7314863646}, -{"blockNumber": 14746186, "blockId": "0x332c8ac0b2960fb1627407d45443201a0f6de7421b52af077b32fa6e018e5401", "firstIndex": 7381974499}, -{"blockNumber": 14798403, "blockId": "0x4507ae1d9f27a3eada874dcf04112a5d4ee7155701afd3fb857fe58201a2dede", "firstIndex": 7449082752}, -{"blockNumber": 14848093, "blockId": "0xff98c820c66c3beaaa3b195158c94f099148573a7f032295ba7156c13f78eaa8", "firstIndex": 7516191978}, -{"blockNumber": 14897508, "blockId": "0x49c31d3433b197585f733d7c382ae16d9d78bf59c333a9b76362cc6c5fed270c", "firstIndex": 7583300663}, -{"blockNumber": 14950638, "blockId": "0xf1a1609206788977841927271808fe1c42d059026385d025bf4cea0c0a05966b", "firstIndex": 7650408992}, -{"blockNumber": 15003958, "blockId": "0xf9e6b41108d907672f29ba1df97ed83aca2ba933335e6b5ccfb5d4001807b33f", "firstIndex": 7717519105}, -{"blockNumber": 15056783, "blockId": "0x126a5a5c22e878a4c42bea890d48cfe00edf7fbc27d805c88a5269540bc94a42", "firstIndex": 7784628173}, -{"blockNumber": 15108163, "blockId": "0x5f8494ccacefce9fef8a95a52247f2670ed30bb5a3b3deb85c645e72c115d506", "firstIndex": 7851735261}, -{"blockNumber": 15159385, "blockId": "0x0fed6c5aafb53e9cdb0d8620c3eef9b619fb3ae9559296618f634eddae33f0c6", "firstIndex": 7918844299}, -{"blockNumber": 15210872, "blockId": "0x37bdb94b57044b67fbd9b9a7cdc728fcfd2066cf1904cab478fd42d41ec6e1f0", "firstIndex": 7985954119}, -{"blockNumber": 15264226, "blockId": "0xc173ad8a791e2293efd8e0b2c9f088fac49233aabb1e72db6a045aa9904f4728", "firstIndex": 8053061840}, -{"blockNumber": 15315693, "blockId": "0x985cf97bf28f0becb2b16abfe629a5f520596e45fc23f4936bc2f6298b8145f9", "firstIndex": 8120172048}, -{"blockNumber": 15364606, "blockId": "0x593362ef8a82140f3dd3ff13d0ce50287dcb0b8b3d31ffcc7813c6be16f7438d", "firstIndex": 8187281275}, -{"blockNumber": 15412727, "blockId": "0x5a6ff10acc48e9e8bcf963b042012ea094628b9488b8bf569783b3297c67eedd", "firstIndex": 8254388575}, -{"blockNumber": 15462642, "blockId": "0x137f15c5d112b0fdc6e4c84393032030646122c3b0d86286143a1ab8bfacbac4", "firstIndex": 8321497214}, -{"blockNumber": 15509781, "blockId": "0xe58a8ca223b7c03755d046a6212bc33fa9814f73efe52804d8eea7ae4b4bd7f9", "firstIndex": 8388606496}, -{"blockNumber": 15558625, "blockId": "0x4127d527b58c7685a1237d228508ea77d40f33dbeffb5cc30e47e83a4b6152a5", "firstIndex": 8455714910}, -{"blockNumber": 15604075, "blockId": "0x7d1c1bccfe5c23a1edd81246e6e55277f271cffef30bde22c9d1202f764dd96f", "firstIndex": 8522823324}, -{"blockNumber": 15651742, "blockId": "0x732e5203a504fd97510325c1fb876476ca132831e78182367cd4b4efae1232c3", "firstIndex": 8589934155}, -{"blockNumber": 15700810, "blockId": "0xe031decb5e42a69dafcc814441227085508cf51c9c77f35d06487db19bd05a6a", "firstIndex": 8657041374}, -{"blockNumber": 15762855, "blockId": "0xbdff9b1e193c44f35bf38945d2523c74ef59c4f352efc93fad02ef34eef1c5f0", "firstIndex": 8724150924}, -{"blockNumber": 15814304, "blockId": "0x524a226d9f48e37db00e252c7f6a1350d1829d38fe83c756021cc11c56d3b2dd", "firstIndex": 8791259373}, -{"blockNumber": 15865514, "blockId": "0x4e4ad46f3ae7fb2a2e7cb94aa804929cb7353c79c78cc2cbcee710da5f0ec4b5", "firstIndex": 8858369571}, -{"blockNumber": 15920411, "blockId": "0xa39e4cbd8e84b427e44b8d06b5b1149652f3af5fa88633b802665b8677772bc5", "firstIndex": 8925477782}, -{"blockNumber": 15974193, "blockId": "0x4a3f82935e6fcda6ce7636001989b65b79422d362e60833e01185a7960b2fa55", "firstIndex": 8992587369}, -{"blockNumber": 16032739, "blockId": "0x7f50bb76dfd01125e0a4a7595106e43329eb8bd02e819be2ade4248a403c714a", "firstIndex": 9059696611}, -{"blockNumber": 16090896, "blockId": "0xdc4db130973ef793d2223844e35122a2477e642d86c69616fb3604d2a79805c4", "firstIndex": 9126804425}, -{"blockNumber": 16150783, "blockId": "0xb46294fe1cbcbbdff7309ebf0bf481e78e2589d386b95fa4c781c437e02e0aec", "firstIndex": 9193912139}, -{"blockNumber": 16207278, "blockId": "0x8a3af0387572b56989bb6c98d8860e07b5cdefb2ebe7b1dc58be07c24a53e056", "firstIndex": 9261022215}, -{"blockNumber": 16262389, "blockId": "0xec5a03d88cfc1bf8c278ddce7ace5032c3f7ed5032b4575df54779ec1abca131", "firstIndex": 9328130680}, -{"blockNumber": 16319525, "blockId": "0x74ae702375e242205fe4986448b0ee494d4a38ccc813981489c4b0488e58a949", "firstIndex": 9395240779}, -{"blockNumber": 16373441, "blockId": "0x5f404555cf45c547ef0fc044f2701bc7a28ce1e432de06e0264dbfd1c17447d0", "firstIndex": 9462347671}, -{"blockNumber": 16423343, "blockId": "0xe4af890b6c4dd3b10af2e6c08f14563bd2a3121b22a3cc6a0db29d438c1dcc59", "firstIndex": 9529457704}, -{"blockNumber": 16474687, "blockId": "0x019a8b374cf150fcda1c8af2a3e7ba63603ae81744d6343897c4b17ee58411d5", "firstIndex": 9596566169}, -{"blockNumber": 16525504, "blockId": "0xad394ce7e84e1c904cc5dd6dabd51b1a575c0b2fb006f10f272875b7f921904e", "firstIndex": 9663675287}, -{"blockNumber": 16574051, "blockId": "0x098e2dca2eda0d700aa3127bdfb679ce16e3ca297a305978c6ecfff044269511", "firstIndex": 9730784563}, -{"blockNumber": 16622457, "blockId": "0x8627f1237731c1127116898724f1b6420b8e7bcd8e18a0ea568f192bbd2cd55d", "firstIndex": 9797893241}, -{"blockNumber": 16672410, "blockId": "0x3e32b46faa670b191b89bea84a68022d5ea8b90008b388a948ff27f871d17292", "firstIndex": 9865002964}, -{"blockNumber": 16719964, "blockId": "0x1668c3ea6cb256f306e93851cc2c00612f85df5cf7b5527ffce14c89088d7fdf", "firstIndex": 9932111708}, -{"blockNumber": 16769014, "blockId": "0xed9a6f88e47b5bfe4d5bc92e1c1bfeb252f716f0990ffa1f907655c0f5e3a365", "firstIndex": 9999220312}, -{"blockNumber": 16818134, "blockId": "0x93504ba617500698a1b46f3110cd26827b8efe468190b2a91d3595784e34a1b6", "firstIndex": 10066329167}, -{"blockNumber": 16869369, "blockId": "0x94ca4b395a5c36c4e9d07cbb08271f0680455af4e4659906b647e16abebd7d4e", "firstIndex": 10133438415}, -{"blockNumber": 16921428, "blockId": "0x66650775bff4cb95de56099c10b06f3dfe302a731b391f406d1aa551121308ba", "firstIndex": 10200547148}, -{"blockNumber": 16974677, "blockId": "0x58d5fc4cd45047dac10d55a036e0a4448faacc1041b29c4c7fb41a6e9394c3ff", "firstIndex": 10267655623}, -{"blockNumber": 17031050, "blockId": "0x9d15b831979e95fc4149ecae46478cac1e54470b277d57d7102d19a72f28eeba", "firstIndex": 10334764441}, -{"blockNumber": 17086163, "blockId": "0x978ff48863fcdb50b274ea4220e7635c13c41bae91912db9e24205fb917fde85", "firstIndex": 10401873176}, -{"blockNumber": 17141086, "blockId": "0x4084d442499893361731f17534944d260cccd8e07d41c52d7160910555770b90", "firstIndex": 10468982462}, -{"blockNumber": 17190735, "blockId": "0x42ed86fb42def23870df6ec23204f557669ec7cd8de59c9f3ef9ab90704e4b11", "firstIndex": 10536090726}, -{"blockNumber": 17237026, "blockId": "0x9a49d943f82a7335f8049df4d953080f25920e3b098f6a2d872520166b225961", "firstIndex": 10603199938}, -{"blockNumber": 17287007, "blockId": "0xac3d189676d24eadc62adcf97d8730a001a7766fe3ac27b18e745ca9727f6c46", "firstIndex": 10670308186}, -{"blockNumber": 17338498, "blockId": "0xe50e251aa0a3209fa8a0030b237cf8357b08eeb80492a432c8aa5969a820ee42", "firstIndex": 10737417623}, -{"blockNumber": 17389178, "blockId": "0x1e18cedcfa6c81bc845c9136fc92f9f27ca8fff8847289bb139f374f0f6ba733", "firstIndex": 10804526769}, -{"blockNumber": 17442038, "blockId": "0x07f47ebbee99938fb96beb1ffadf8266f71d0d73f393172f0e24facd40d45c97", "firstIndex": 10871635940}, -{"blockNumber": 17497589, "blockId": "0x247a5e852e36e46f7897ad1875e93d810ff4434bb835b469afae3a3d3d4a9c9a", "firstIndex": 10938744251}, -{"blockNumber": 17554111, "blockId": "0x420e420a1b599e4dc3bd115652b4d6f5eedc391086ef064f97a9cbffc02b62f9", "firstIndex": 11005853686}, -{"blockNumber": 17608282, "blockId": "0xb2ba644d3593d3c832f932f14ffec532e26e9e1cdcd8fbe7477f46e238c6bffd", "firstIndex": 11072962303}, -{"blockNumber": 17664091, "blockId": "0x6d0bfff8145652a617bfe03ab87e3af09219cdb822c9c214b42b16f20742c1eb", "firstIndex": 11140070260}, -{"blockNumber": 17714917, "blockId": "0x61509544b5174d3ff5d48816ff54f2929c8a400ef21454116056baa136b607b6", "firstIndex": 11207179013}, -{"blockNumber": 17763850, "blockId": "0x8da83012621f42670601406c902ce6e48e77c6285dcc6983e108e69b3a8d5d2d", "firstIndex": 11274288732}, -{"blockNumber": 17814186, "blockId": "0xc37bda200856589c220dc206e31186d6c12761fd193f5bc7123cf331fbbed7f5", "firstIndex": 11341397343}, -{"blockNumber": 17864461, "blockId": "0x99af43893dc9f54c55abbdbd7b000e27248145371e9b00db116dfb961b36749c", "firstIndex": 11408506122}, -{"blockNumber": 17913212, "blockId": "0x9b642236d63d756fcd3404713bc6abe97804b46a9e665ae5f1d9aa48e693fe78", "firstIndex": 11475614347}, -{"blockNumber": 17961497, "blockId": "0xaec9b1e851cf51646673ec7698e1d5964d44eb04729ad11385f6ae84d497070a", "firstIndex": 11542723713}, -{"blockNumber": 18010859, "blockId": "0xb15fb7fc308516cad5f1dadfb7dd92fd3fe612308f67154cf2a241990aa15bc4", "firstIndex": 11609833204}, -{"blockNumber": 18061052, "blockId": "0xd8ea94b8b8a7380d21dc433abb69973f3de2eeb339f4aaffdb79f85a5d86929d", "firstIndex": 11676938565}, -{"blockNumber": 18111466, "blockId": "0x5b5cc6d182bffaf06692a7efda8cad5d4742fdbce5c1146b3be3c5be10ea046a", "firstIndex": 11744048539}, -{"blockNumber": 18166001, "blockId": "0x901d8401c795542c75069cd917781fa587ad416da0a5298f144fae2f14305705", "firstIndex": 11811159949}, -{"blockNumber": 18218505, "blockId": "0x5894285023f34ec6fd501cd90b605e118b9688ca7c01082289bf69a6987142bc", "firstIndex": 11878268395}, -{"blockNumber": 18271037, "blockId": "0x3f0fce315fd993d966b6c599ebd360462466e67046d512dc430ac07f75243bae", "firstIndex": 11945377050}, -{"blockNumber": 18322778, "blockId": "0xfa6536b3b06e7b579e53a8d01368182808c950586e39e12485be3145c3acbf43", "firstIndex": 12012486036}, -{"blockNumber": 18372240, "blockId": "0x5531859d5b8a4093020e2fb971808d0eeaddd7d61dc58d70f34923081f97336f", "firstIndex": 12079595358}, -{"blockNumber": 18421630, "blockId": "0xfeadff676fe8f04d0362b77ea2d7c3ea59b05d68cc6b2cc95cfe751558b1bfa7", "firstIndex": 12146703872}, -{"blockNumber": 18471472, "blockId": "0x352b0a17e4160d334b876eb180d686fd67c7ddc40a78219b0bc4b59f5ae5912d", "firstIndex": 12213813174}, -{"blockNumber": 18522088, "blockId": "0xf7d408309246089531b47c787049c7ffd2820c1b87b4804033a458bc193a3432", "firstIndex": 12280921181}, -{"blockNumber": 18572725, "blockId": "0x44126d4a5b43594b09d26e0b0a5dc859c074865cb9b11ab78c80f28406fd0240", "firstIndex": 12348029615}, -{"blockNumber": 18623223, "blockId": "0xd3aec1f98cea30e870d0f530194e9a72e225eb45eefe435e38da5967eb2fa470", "firstIndex": 12415139522}, -{"blockNumber": 18675228, "blockId": "0x24deaef2508038bddf8e9020f95128ae34eb1786f786c3f98df0bdf95f3e97cc", "firstIndex": 12482248123}, -{"blockNumber": 18725495, "blockId": "0x2e5a7edd7337b20e862741e116680a472089426dbc2b0eb5f1c984f8d6da935b", "firstIndex": 12549357121}, -{"blockNumber": 18778174, "blockId": "0xe734a1084fa5eeff427bba8e8d3623181a30dd9bad128fc22de9b15dffd0e549", "firstIndex": 12616466131}, -{"blockNumber": 18834807, "blockId": "0x0924a55ef4f9c2605261433c5b236d47645484efa44b34dc9f15b7c2e6b47a74", "firstIndex": 12683575000}, -{"blockNumber": 18883054, "blockId": "0xef638ec491a1cf685b22bf4072e41710440d6133a53e31250762825247d3eb64", "firstIndex": 12750683712}, -{"blockNumber": 18933405, "blockId": "0x6e9eefc2e121d13e6c04544ea370fe8543afeb36fef3a8e2ea68afb079738c8c", "firstIndex": 12817792510}, -{"blockNumber": 18988033, "blockId": "0x4c98c0dbb6a7217609c4b277fe7ff9e2c32c8d000ee628a3f37b2b296e1ca421", "firstIndex": 12884901389}, -{"blockNumber": 19041096, "blockId": "0x61a219a23d111cd07861ff47b89d4b2840c0ab5421d5d55a6f1fe43da3f33b98", "firstIndex": 12952010147}, -{"blockNumber": 19088914, "blockId": "0xefb26ba58447e50e9b820e0b0d16c08da45793a824af12a2e1f38b6f46b7e36a", "firstIndex": 13019118891}, -{"blockNumber": 19140413, "blockId": "0x91dbb13623fc4d5a4e462dfa0a03c92bf5301948dd4b2c3e5fddd2bc88856d47", "firstIndex": 13086228137}, -{"blockNumber": 19191876, "blockId": "0xdd7f97af7ad206b2636656ed4722fe1c47c6ba9fa5531066785acbe4f8d4c4ad", "firstIndex": 13153336551}, -{"blockNumber": 19237651, "blockId": "0xf1e875f314314abe3654b667c6443c772dd67c525c52877b06515ea8fc5a391b", "firstIndex": 13220444812}, -{"blockNumber": 19291032, "blockId": "0x9eda269ac47957b619d2a8c408b5efc8cfb168a46bfde351f43325fa337fce47", "firstIndex": 13287553967}, -{"blockNumber": 19344225, "blockId": "0x62859b78d45f589f679fc6a3bf266a2e0aa906fd61d787755ec51cfadc05eaf6", "firstIndex": 13354663338}, -{"blockNumber": 19394709, "blockId": "0x7cda3d193bcfe8ddbc89b85e98c814dbbe31a54ccab52ab7738ecd003061f3c8", "firstIndex": 13421772549}, -{"blockNumber": 19442850, "blockId": "0xd95f3f8bab5b91c3e1f848c9455936850bf92cf6a5086b817f6f81031bc0f81e", "firstIndex": 13488880704}, -{"blockNumber": 19488177, "blockId": "0x3a1a9623ee332d6decada9359cf84eeafb81fffa997d6cf4f3efcc92e98e9310", "firstIndex": 13555989570}, -{"blockNumber": 19534365, "blockId": "0x5c7a8ee7cbba4894ea412cc335895a9a7d3369d35779bcdc97a6c24b6a81a7ba", "firstIndex": 13623097446}, -{"blockNumber": 19579409, "blockId": "0x4280f5f76cb0f9947f9da1cb9de6c4b6e65b37b3a3c174f5fb29d360cf389ffb", "firstIndex": 13690206790}, -{"blockNumber": 19621406, "blockId": "0x0a98ca72e97fd295c7c73263b17185d74dbe427f41c616a5d5454faa5a1c63a0", "firstIndex": 13757316653}, -{"blockNumber": 19664868, "blockId": "0xc8c53d89b7a849861f3daaf9503e6397a0340cecb085850c3636bee6ea2b7189", "firstIndex": 13824424608}, -{"blockNumber": 19709011, "blockId": "0xf65babc88ef6a9eb2ffce9a6844adf895995f65a2facd1956f5df17ca68dbf94", "firstIndex": 13891534141}, -{"blockNumber": 19755170, "blockId": "0xf903882449eeb86d69af33b6797099275e82d8787334a02a42a2168dbd739b95", "firstIndex": 13958642627}, -{"blockNumber": 19803666, "blockId": "0xa8a975f570db082a3771790bb1fe405667dde4674ff49e31092687fa92727e36", "firstIndex": 14025752433}, -{"blockNumber": 19847730, "blockId": "0x89d479a4b4d9faa7af3e3f61d13a7a9c80a11c4adf7cd018859eb0ddba30bdd5", "firstIndex": 14092860598}, -{"blockNumber": 19894138, "blockId": "0x740278e6dc79cfd2b030780868a1b8e1520dbae0e8d3ee0a3ef43c84c83c6e0a", "firstIndex": 14159952129}, -{"blockNumber": 19938334, "blockId": "0x05ae7ce9813235c19baa57658bc5c8e160130328a1fd2cc58d2baf2d9a4ae097", "firstIndex": 14227078393}, -{"blockNumber": 19985138, "blockId": "0x511c8876a93a77fa473ae0d29e25d8163162c09b7212f506be0ea73840ea3971", "firstIndex": 14294187397}, -{"blockNumber": 20028221, "blockId": "0x64379baedf429cfa65df076142a7d9909d7e369d6aa6b8e161bb686944d169ff", "firstIndex": 14361296446}, -{"blockNumber": 20071555, "blockId": "0x4f43d8b1091d2bd0c6b080ce51bce08aae6be605e00889be3cdd17cb80614ff8", "firstIndex": 14428405409}, -{"blockNumber": 20113611, "blockId": "0xebbf87409ef6732a53513414926b0aa998a7269f9e9717a3acbe96a33c2de937", "firstIndex": 14495514255}, -{"blockNumber": 20156628, "blockId": "0x70218108c0669f42c4ac1afff80947cf4a0a2fd8e9300b31d45bbe330a5159c9", "firstIndex": 14562622257}, -{"blockNumber": 20199886, "blockId": "0x1d6aaa4b041eb0cc9caed34347bb943e50a139caf8998d781ff4f06d4aa727dc", "firstIndex": 14629732017}, -{"blockNumber": 20244148, "blockId": "0x37fed57b6945a815ec56c84c6261e90cf9f3883d9cbffc62b70d9722c1f0f363", "firstIndex": 14696839730}, -{"blockNumber": 20288249, "blockId": "0x1d9fe664431e093c4ef6139ad35fea767a2e66825e605cfea9e91d1738665619", "firstIndex": 14763948803}, -{"blockNumber": 20333358, "blockId": "0x1ac7c1362791507057b7e87d7118b4f61c2919c8ebdb498ecf7d64a59ba3d2e2", "firstIndex": 14831058814}, -{"blockNumber": 20376855, "blockId": "0xf6a19eac8a402f591079a4aa7ff57349c350ec96d683a031e4d85c05bc0541b2", "firstIndex": 14898167076}, -{"blockNumber": 20421474, "blockId": "0xe8767aa7516b632dc8c0fed3a366273ef14bd7b8c2212998b7105fb80b4c0d9b", "firstIndex": 14965276285}, -{"blockNumber": 20466974, "blockId": "0xafa2d2e6c9594762c163c339069cb15aa9fd27d301268892e12f99a3f2912ade", "firstIndex": 15032384674}, -{"blockNumber": 20512183, "blockId": "0xc1c51f9372d781d7ac0a1a7aa052503bbde3e1706bf65f225b0f6a6eed93d055", "firstIndex": 15099493832}, -{"blockNumber": 20557251, "blockId": "0xa905087c15e2b12b19cd23f20089b471cd9815dcff73842d2043e13a276fcc55", "firstIndex": 15166601921}, -{"blockNumber": 20595654, "blockId": "0x9e7772dd946f65e7569f34f6896d8e597ac8ff88700b871de21ff578c0c995e6", "firstIndex": 15233711474}, -{"blockNumber": 20638340, "blockId": "0xc08b46af83db6f0e5d1acad7020bacd59d0f18df72651d32c9b60edd0351a886", "firstIndex": 15300820584}, -{"blockNumber": 20683359, "blockId": "0x7ec93a1f1d211c24d93ce7e344634efef680a6388374cf3831d250ecd63711f9", "firstIndex": 15367929168}, -{"blockNumber": 20728401, "blockId": "0x4ad4124c2595abcf4477cac4eb70c0c9372bbd48f2066c7ac5b8e9f94817ab48", "firstIndex": 15435038303}, -{"blockNumber": 20771272, "blockId": "0x68fd6b12fdb4add8b2957105a21b571c7dcb114c6c2ca02f5fa40730a108f864", "firstIndex": 15502146509}, -{"blockNumber": 20814862, "blockId": "0x4e7f640bf3c135b2442ca4c744023021af163e8d45e8d776fb7051c49d79fcff", "firstIndex": 15569256173}, -{"blockNumber": 20857627, "blockId": "0x9b023bffed12530b5a68db263bb621a147e0cf02b30ec39a82b82a1fdeb7ca36", "firstIndex": 15636364955}, -{"blockNumber": 20896662, "blockId": "0x8c3adf694b8520ed1edd38282b2f58cf383426176628b5be6330a56882e2a07e", "firstIndex": 15703473445}, -{"blockNumber": 20939156, "blockId": "0x0649e2559f767da945a674c9d6c700628109e18d0757ad33496a228a827413a4", "firstIndex": 15770581979}, -{"blockNumber": 20981054, "blockId": "0xebf0ecca671b833f77df70106cf1171d4dd3fa0f062607e4845569073997f1cd", "firstIndex": 15837691319}, -{"blockNumber": 21022980, "blockId": "0x07479520c08a727c42e3616aca7fc4435773f4ef277d84369190263159606d71", "firstIndex": 15904799391}, -{"blockNumber": 21068139, "blockId": "0x669cfd03b293a61815588cb9cef928a512d328c1fe9191fe6b281573f429f8b8", "firstIndex": 15971908604}, -{"blockNumber": 21112368, "blockId": "0x98957aedaf8b0483be208e8d16a3e6d73ff217e732f3649650671c92c8fb57a5", "firstIndex": 16039007815}, -{"blockNumber": 21155756, "blockId": "0xc301e40988fbe19b1254ab3abe2f78fe0cd35259898be02476179a041926f777", "firstIndex": 16106127130}, -{"blockNumber": 21200717, "blockId": "0x8d35b199ee65c5ccf4ca0fc22b50282850cd88b72580679f1568eea67d3431e8", "firstIndex": 16173235593}, -{"blockNumber": 21244396, "blockId": "0xe7ec084eb8a7125fe26c4ef64fdbb3e0a727b2d882fb1e4fdad1d7fa3dc55fd2", "firstIndex": 16240343803}, -{"blockNumber": 21290031, "blockId": "0x8bb31b883d30875b7ed8a01c35a6f51e603b8d974c4e74af20fb9c723e5ac5df", "firstIndex": 16307452772}, -{"blockNumber": 21335893, "blockId": "0xd0976b7c946a7556e607e47a02d2cfcccd259508e2828b9c4faa5c7105a597a8", "firstIndex": 16374562657}, -{"blockNumber": 21378611, "blockId": "0x72d42953f013aeb7c84bb22c9e07828b92f73885b20c208af612874df8fe85ed", "firstIndex": 16441671192}, -{"blockNumber": 21421343, "blockId": "0x663b3cddcf3f7cfa59ed29b4894e43fa784c07576cfec897ff1a3838235ecb50", "firstIndex": 16508779766}, -{"blockNumber": 21466598, "blockId": "0xe04ebd60289dbcc09a27e053e7b240a59e928e2a76a85a6fa4044ee387c46d18", "firstIndex": 16575888836}, -{"blockNumber": 21511717, "blockId": "0x5ea9863a0b504f381010ff383d60a56390ba9cef162d9d5a077b1884d2adac98", "firstIndex": 16642998234}, -{"blockNumber": 21550050, "blockId": "0x2a7b3e635c80521887ca3ebda04541090ec720dd817cd06fcba61e7ae0f78a4f", "firstIndex": 16710106733}, -{"blockNumber": 21592440, "blockId": "0xc7095268d0d53cb5a069c2ea7107cc42172bb1b817f04154938e6b52bdbb369c", "firstIndex": 16777215261}, -{"blockNumber": 21636007, "blockId": "0x55f463b932053ebfc02728a12c2cdfb2cb4700a115cde686c9ab22bb4c336ecb", "firstIndex": 16844324844}, -{"blockNumber": 21680799, "blockId": "0x4521d473aa7f5150f920c66fe639b20f30cb373b32e425fcb7dcce9803b506be", "firstIndex": 16911431255}, -{"blockNumber": 21725330, "blockId": "0x793876007ce3594df603628766b139a02273ceee43d5b3ca2d0ca2f13a5fc5d9", "firstIndex": 16978541626}, -{"blockNumber": 21771310, "blockId": "0xd0db23bb28d6e39917348b1879dd560c73354ea6d7d0bbee0b7a8fcbcede1091", "firstIndex": 17045650902}, -{"blockNumber": 21811348, "blockId": "0x8165948c08db34aeb8cc2a3e331a00af2d09fbff9c466019fb2f2892ca3f7b71", "firstIndex": 17112760024}, -{"blockNumber": 21851615, "blockId": "0x3083e8f4fbb4d726ef19b1a5bce20e639a0418eac384f9967f24c7697a7e2dfc", "firstIndex": 17179868870}, -{"blockNumber": 21894041, "blockId": "0x9b0776a020374e4985f6a7e1199596bedd387ff085f65224d724ab11955c56fe", "firstIndex": 17246977342}, -{"blockNumber": 21932218, "blockId": "0xe9c50eddc21759d3ab790e2babba56f4b8d38eb739f1b561bc337a1ba6e15393", "firstIndex": 17314085541}, -{"blockNumber": 21959188, "blockId": "0x6b9c482cc4822af2ad62a359b923062556ab6a046d1a39a8243b764848690601", "firstIndex": 17381193886}, -{"blockNumber": 21994911, "blockId": "0x0d99ef9dd42dd1c62fc5249eb4f618e7672ab93fa0ba7545c77360371cf972e5", "firstIndex": 17448302795}, -{"blockNumber": 22026007, "blockId": "0xf7cdc7f6694f2c2ed31fa8a607f65cfa59d0dd7d7424ab5c017f959ae2982c71", "firstIndex": 17515413036}, -{"blockNumber": 22059890, "blockId": "0x82b768a0dddefda2eefd3a33596ea2be075312f1dd4b01f6b0d377faca2af98b", "firstIndex": 17582521768}, -{"blockNumber": 22090784, "blockId": "0xf97c2eaf9a550360ac24000c0ff17ffa388a2bdd6f73f2f36718e332edfa107a", "firstIndex": 17649630983}, -{"blockNumber": 22121157, "blockId": "0xa790025235db782e899f23d8b09663ec2d74ec149e4125d62989f98829b08e2d", "firstIndex": 17716724973}, -{"blockNumber": 22148056, "blockId": "0xbe25ac4f1bdd89a7db5782bf1157e6c4378d80220d67a029397853ef16cd1a4c", "firstIndex": 17783838366}, -{"blockNumber": 22168652, "blockId": "0x6ae43618c915e636794e2cc2d75dde9992766881c7405fe6479c045ed4bee57e", "firstIndex": 17850956277}, -{"blockNumber": 22190975, "blockId": "0x9437121647899a4b7b84d67fbea7cc6ff967481c2eab4328ccd86e2cefe19420", "firstIndex": 17918066140}, -{"blockNumber": 22234357, "blockId": "0x036030830134f9224160d5a0b62da35ec7813dc8855d554bd22e9d38545243ed", "firstIndex": 17985175075}, -{"blockNumber": 22276736, "blockId": "0x5ceb96d98aa1b4c1c2f2fa253ae9cdb1b04e0420c11bf795400e8762c0a1635c", "firstIndex": 18052284344}, -{"blockNumber": 22321282, "blockId": "0x8a601ebf6a757020c6d43a978f0bd2c150c4acc1ffdd50c7ee88afc78b0c11f8", "firstIndex": 18119392242}, -{"blockNumber": 22349231, "blockId": "0xb751c026a92ba5be95ad7ea4e2729a175b0d0e11a4c108f47cab232b4715d1a2", "firstIndex": 18186501218}, -{"blockNumber": 22377469, "blockId": "0xa47916860a22f7e26761ec2d7f717410791bd3ed0237b2f6266750214c7bbf08", "firstIndex": 18253610249}, -{"blockNumber": 22422685, "blockId": "0x8beaee39603af55fad222730f556c840c41cd76a5eef0bad367ac94d3b86c7aa", "firstIndex": 18320716377}, -{"blockNumber": 22462378, "blockId": "0x6dba9c5d2949f5a6a072267b590e8b15e6fb157a0fc22719387f1fd6bfcd8d5d", "firstIndex": 18387828426}, -{"blockNumber": 22500185, "blockId": "0x2484c380df0a8f7edfdf8d917570d23fab8499aea80c35b6cf4e5fe1e34106e9", "firstIndex": 18454936227}, -{"blockNumber": 22539624, "blockId": "0xd418071906803d25afc3842a6a6468ad3b5fea27107b314ce4e2ccf08b478acf", "firstIndex": 18522044531}, -{"blockNumber": 22577021, "blockId": "0xff222982693f3ff60d2097822171f80a6ddd979080aeb7e995bfb1b973497c84", "firstIndex": 18589154438}, -{"blockNumber": 22614525, "blockId": "0x9868da1fea2ffca3f67e35570f02eb5707b27f6967ea4a109eb4ddbf24566efd", "firstIndex": 18656264174}, -{"blockNumber": 22652848, "blockId": "0x060a911da11ab0f1dda307f5196e622d23901d198925749e70ab58a439477c5a", "firstIndex": 18723372617}, -{"blockNumber": 22692432, "blockId": "0x6a937f2c283aba8c778c1f5ef340b225fd820f8a7dfa6f24f5fe541994f32f2d", "firstIndex": 18790480232}, -{"blockNumber": 22731200, "blockId": "0x00d57a9e7a2dad252436fe9f0382c6a8860d301a9f9ffe6d7ac64c82b95300f8", "firstIndex": 18857590076}, -{"blockNumber": 22769000, "blockId": "0xa48db20307c19c373ef2d31d85088ea14b8df0450491c31982504c87b04edbc0", "firstIndex": 18924699130}, -{"blockNumber": 22808126, "blockId": "0x1419c64ff003edca0586f1c8ec3063da5c54c57ff826cfb34bc866cc18949653", "firstIndex": 18991807807}, -{"blockNumber": 22845231, "blockId": "0x691f87217e61c5d7ae9ad53a44d30e1ab6b1cc3f2b689b9fbf7c38fbacacfe3e", "firstIndex": 19058917062}, -{"blockNumber": 22884189, "blockId": "0x7f102d44c0ea7803f5b0e1a98a6abf0e8383eb99fb114d6f7b4591753ce8bba3", "firstIndex": 19126024122}, -{"blockNumber": 22920923, "blockId": "0x04fe6179495016fc3fe56d8ef5311c360a5761a898262173849c3494fdd73d92", "firstIndex": 19193134595}, -{"blockNumber": 22958100, "blockId": "0xe38e0ff7b0c4065ca42ea577bc32f2566ca46f2ddeedcc4bc1f8fb00e7f26329", "firstIndex": 19260242424}, -{"blockNumber": 22988600, "blockId": "0x04ca74758b22e0ea54b8c992022ff21c16a2af9c45144c3b0f80de921a7eee82", "firstIndex": 19327351273}, -{"blockNumber": 23018392, "blockId": "0x61cc979b00bc97b48356f986a5b9ec997d674bc904c2a2e4b0f17de08e50b3bb", "firstIndex": 19394459627}, -{"blockNumber": 23048524, "blockId": "0x489de15d95739ede4ab15e8b5151d80d4dc85ae10e7be800b1a4723094a678df", "firstIndex": 19461570073} +{"blockNumber": 4166212, "blockHash": "0xd94b724fc1c7dceb3251b51b81f7a0f3220ae5b9add1e62917004f14f2a3532c", "firstIndex": 67108840}, +{"blockNumber": 4513996, "blockHash": "0xc0fd25fef5888609d05fa8c5620d8e18c31cdd675dd4ee926ff966668f0e5d76", "firstIndex": 134217480}, +{"blockNumber": 4817399, "blockHash": "0x8573c3166fb7f716e97cf6109d382f86441238e0b85828944a072ae8583a6cfa", "firstIndex": 201326547}, +{"blockNumber": 5087706, "blockHash": "0xcd2dfae45901e299b25faa04c10df60983d5c0a3d0e478e789c7aeec980cf691", "firstIndex": 268435422}, +{"blockNumber": 5306085, "blockHash": "0xe8026984c85873f2b24018a7e0bdf8c2df136b2fc49666b4addf0d2e067890da", "firstIndex": 335544018}, +{"blockNumber": 5509898, "blockHash": "0x41bc63dc8184f9a7d4b9a5a554e00eee32fee60ccfa2339264be11270ac28de1", "firstIndex": 402652789}, +{"blockNumber": 5670367, "blockHash": "0xe61c2ba463f2f458817ed1bf0ff9fb9d07e9d7354f46b48175b2d784b817bf3b", "firstIndex": 469761896}, +{"blockNumber": 5826113, "blockHash": "0xfbbb1fc5e03a5562bf6b7f1799d7a572d9ef66c7fd0e0b9f4fed7262767b5c86", "firstIndex": 536870852}, +{"blockNumber": 5953008, "blockHash": "0x6bbdccd094928e846de3c615a33f2952e3259afaa1929f4ee241a56907d0593b", "firstIndex": 603979150}, +{"blockNumber": 6102812, "blockHash": "0x2403f2502088a1fa26c1294f5245032fe3b7f3a8bb14c12d0ba8d577156bbc1b", "firstIndex": 671087962}, +{"blockNumber": 6276672, "blockHash": "0xeaa05a0e0574fbb164244628a7d14d9d47f1692941098aa737059d44104401df", "firstIndex": 738197399}, +{"blockNumber": 6448662, "blockHash": "0xe6f097fa18a2cef425514e863659e42e1aa8d74df49cb0bed5877e82a08d1f84", "firstIndex": 805306056}, +{"blockNumber": 6655925, "blockHash": "0xee5c9d87b06dc0c2e850fead07bcd13f85d45ad6b5a6e7ebee53cc8618772786", "firstIndex": 872415229}, +{"blockNumber": 6873878, "blockHash": "0x9a0ea103146da21fa1d9cab7ff609ec2c5e7f1856288a1f744239588e33904c2", "firstIndex": 939524080}, +{"blockNumber": 7080893, "blockHash": "0x35ad45055d7a1a3ef94efeb05e4122757bcc126ff1bc2b4ac72dc78ab3c0fd75", "firstIndex": 1006632327}, +{"blockNumber": 7266955, "blockHash": "0x7b80f70520f2c16eb5890f4963af8f60446f50af82b589ceaf009d09e704b302", "firstIndex": 1073741608}, +{"blockNumber": 7466649, "blockHash": "0xea8980b5c692c729013f9cc4c9f66e94d5ef2c5caad23f370be0953b6d85f32c", "firstIndex": 1140850335}, +{"blockNumber": 7661736, "blockHash": "0x615cdb03dbf29a22d59bbc769c03f3647c2c9abb7d1767f3f6529708209e7073", "firstIndex": 1207959232}, +{"blockNumber": 7834494, "blockHash": "0x05b7dd13e2eb8a833e4b278f9151ca01b294ae5fc5617769422ec85ffb446215", "firstIndex": 1275068098}, +{"blockNumber": 7989998, "blockHash": "0x4b5026add7f2fc2c02993b82c61bceba1b75cfcb8a78c5112cc2832bc0413be6", "firstIndex": 1342177025}, +{"blockNumber": 8142950, "blockHash": "0x9f56b7753a0e9964973f3e096e1385215e3aef232b448359e5bec05046b016a3", "firstIndex": 1409285545}, +{"blockNumber": 8297598, "blockHash": "0x2ddfeb161fe34ff73c4a995bb94256c3b522b39348169edb97bb3d876ccdf77d", "firstIndex": 1476394898}, +{"blockNumber": 8465061, "blockHash": "0xfdbe5435da5ae9fb34a9154e5ebe515afac9a128e383f798f1e69952be404fcd", "firstIndex": 1543503805}, +{"blockNumber": 8655740, "blockHash": "0x305137325a39a44aba997e214c05f32d965658f7e3fe20322f2e57e72f239977", "firstIndex": 1610612406}, +{"blockNumber": 8807102, "blockHash": "0xc32f9fd9a43d2b502dbcc4a683ffe324fca9a871753478bcb1f4fb9573da96a5", "firstIndex": 1677721343}, +{"blockNumber": 8911110, "blockHash": "0x806eabee7dc429b57afcb6c0b72b61728b4f13c62ee9942f1813790463f9ab5b", "firstIndex": 1744830172}, +{"blockNumber": 8960262, "blockHash": "0xa69f24032e9d5761565e726d484926a0932c6ca7fdb4c04a55bfc5e8f5879c4d", "firstIndex": 1811938729}, +{"blockNumber": 9085906, "blockHash": "0x922b4d6aeca64517f5048fa1ffa98d3e813bc415716a1763d07cf2bdea236896", "firstIndex": 1879048062}, +{"blockNumber": 9230838, "blockHash": "0xd892f513c48c95a859ae59c4e00ec4be0d684c549456c90e9c36669c11092f50", "firstIndex": 1946156580}, +{"blockNumber": 9390449, "blockHash": "0xcc0ced78fa66b570f338d129794d574560a23958e28d0d5288003d4542cec734", "firstIndex": 2013264376}, +{"blockNumber": 9515554, "blockHash": "0xc06d04737813c3e2c8e910b2d4543c5c684c7bfe235bb501445f68fd0663f3d9", "firstIndex": 2080374749}, +{"blockNumber": 9659367, "blockHash": "0x765fa33d9418b1ed648e88b497b4bf4aa66990e6413dfc155525cda61e3cc813", "firstIndex": 2147483344}, +{"blockNumber": 9793940, "blockHash": "0x9527dcbd85a100c51aee4ba90828fee23312f5f041859f3416a54fd87a437ce2", "firstIndex": 2214591669}, +{"blockNumber": 9923756, "blockHash": "0xc4da5fdd2de077459fe84f721d7fc01ef69683e03e634b0c5443186486792246", "firstIndex": 2281700949}, +{"blockNumber": 10042325, "blockHash": "0x6f554f195d20e4ba1262f73cf2c1cd44b14a71befe5ef0a5318fc309f8cde9fa", "firstIndex": 2348810131}, +{"blockNumber": 10168768, "blockHash": "0xdcdf88ec2c197d552343c80d01375a35d6462aff39c5bd976160f7176d2add64", "firstIndex": 2415918723}, +{"blockNumber": 10289467, "blockHash": "0x5ece23babcf6b8838695b891b044c3dcf8120cdc15acd83018e96ffc590f6cd0", "firstIndex": 2483027720}, +{"blockNumber": 10386601, "blockHash": "0xd398ee129aea1247019fa39dc2f5083819687d8f5ad17ec39ba1803d178fbcda", "firstIndex": 2550136043}, +{"blockNumber": 10479586, "blockHash": "0x61c1cccd60a546eacc29f2ae2a6facab402b4f35e2986c6629b302e6b1ebca3e", "firstIndex": 2617245577}, +{"blockNumber": 10562579, "blockHash": "0x14fd6a079050e3e110140372605c050945d6db03746fa5b84a8718cb2a0e9b86", "firstIndex": 2684353802}, +{"blockNumber": 10641438, "blockHash": "0x1998dcae09111b2c1bf6d0592e2f0b4a0143fe7db7115ff94b2b73b74542cee4", "firstIndex": 2751463334}, +{"blockNumber": 10717094, "blockHash": "0x9ce45a150bb6bd13687d0692c86716911ffc6aff5831a03f4015bdffb85f5093", "firstIndex": 2818571144}, +{"blockNumber": 10784494, "blockHash": "0xf65c936cf578e673696477f7db7189b47222fdf7ed13096564b9d47b15c86eb9", "firstIndex": 2885680709}, +{"blockNumber": 10848591, "blockHash": "0x84ea8e9d5789302ebe88888115488f91cfb75fd89e282dfbd29ba9d6225ab0ca", "firstIndex": 2952789491}, +{"blockNumber": 10909102, "blockHash": "0x22275047f643f4a3b82c349274a53d56a00da25beac89655877fb8cbfa3182c8", "firstIndex": 3019898024}, +{"blockNumber": 10972828, "blockHash": "0x38e8acfec19bb53ab9fe70dd3577ccd8bd7c41b4d468b8f099a19bf06ce56518", "firstIndex": 3087006561}, +{"blockNumber": 11036532, "blockHash": "0x478dac03f520e987a6e845cac1b3a756d69e4f5257f20758b8fc38456e7b6884", "firstIndex": 3154116463}, +{"blockNumber": 11102457, "blockHash": "0x1a36616b03351ebe481330fc86578f57d9da5c0dda59f4406d49b53a2fa9b6f6", "firstIndex": 3221224961}, +{"blockNumber": 11168092, "blockHash": "0xb23af483ca6dbc2fb8d2fada1f8189de2e155758ed9ecbbb20c234a30c7e093c", "firstIndex": 3288333510}, +{"blockNumber": 11233640, "blockHash": "0xdc4c80d8370823773425b31e70b3af59582b556514857d1e9a9b4c53b880a424", "firstIndex": 3355442901}, +{"blockNumber": 11300454, "blockHash": "0xdbd4d8e9c1756093037f431330db4604e5a3e67bb74f425f667633d5298fe34c", "firstIndex": 3422551962}, +{"blockNumber": 11367550, "blockHash": "0x9e00816bd5b39353657a99421d278091599a39f246bbf8c0898f83e6d70b7c7c", "firstIndex": 3489660743}, +{"blockNumber": 11431820, "blockHash": "0xd32d5e5871ce71acc9dcc7cf9d8cb9d964646182529c327a0ddf617814e27d74", "firstIndex": 3556769150}, +{"blockNumber": 11497031, "blockHash": "0xed823054d4b293294e3301d3131e2ae9f0ce63c9f07d2d80fc144e42d535d61d", "firstIndex": 3623878513}, +{"blockNumber": 11560027, "blockHash": "0x7098177efe77cfb5912dc63069fe366033111041ca5c539907a89e439c861d8f", "firstIndex": 3690987092}, +{"blockNumber": 11625049, "blockHash": "0x61fccb47104a076b905d390498a1b38c317348bb1567e69aa1b670bed7a6a13f", "firstIndex": 3758096043}, +{"blockNumber": 11690298, "blockHash": "0xaf6f9e9c92db9e155f1ffb35eb68765c09de1206c353c9a56a5d5fa53e2d88d2", "firstIndex": 3825204430}, +{"blockNumber": 11755003, "blockHash": "0xf9efb54f08a2b59086a607541005928f1bda54fa2c46fcc48352fa6aa461c03a", "firstIndex": 3892313541}, +{"blockNumber": 11819591, "blockHash": "0xa551deb3bd2ec0ca2683cb76adb8fb295d171a3787aa5ff071bd1fa94d3af82e", "firstIndex": 3959422322}, +{"blockNumber": 11883278, "blockHash": "0x60c7a359d4aeb09b63b64487fa7794aee87c41b11446f69ba83e24d5bce6947f", "firstIndex": 4026531239}, +{"blockNumber": 11948436, "blockHash": "0x1c46fb22653e5f9eb9503fe532151519845d05b121291766715973e027eb9c0c", "firstIndex": 4093639741}, +{"blockNumber": 12013084, "blockHash": "0x0b8d1da9e70c4494ba61190390c891230846e69d9b0be605df8b04c4199ffd1b", "firstIndex": 4160749561}, +{"blockNumber": 12078612, "blockHash": "0xb049af260be25c6153b67c227689b0556d8fb7d6033c3f61b8218f5c4ac5917a", "firstIndex": 4227857310}, +{"blockNumber": 12143549, "blockHash": "0x685dce41efbe2bab0ac9a4b9026d9e5faa3e01ae4637c8181cd2f1eac326fe6a", "firstIndex": 4294966232}, +{"blockNumber": 12207918, "blockHash": "0x53644ea8f781c6270f1136ed4bbcb956c98f12a4793c2f56f49b3105282b13ea", "firstIndex": 4362075410}, +{"blockNumber": 12272370, "blockHash": "0x503523e1a01cb7d68be0eb22f36280c0ee7a1fc802061a704e326fbb8efcfe27", "firstIndex": 4429184831}, +{"blockNumber": 12329342, "blockHash": "0xe6a1bf78dd827def078e23c768cf6e96666ee06b5fdc113380a4769d85f4bd33", "firstIndex": 4496293151}, +{"blockNumber": 12382311, "blockHash": "0x352fe793c16b1d8aea195af86ee9aec3dbaa83e7a32872a0d222a77e96fc0e57", "firstIndex": 4563402486}, +{"blockNumber": 12436970, "blockHash": "0x90276349edf0941bb3327dc1d749d0371fe550eaddafb5141fcacdd281de0976", "firstIndex": 4630510187}, +{"blockNumber": 12489954, "blockHash": "0x71894b1e1608aeba777662a6055e601ce6f17950dbe3844c3601e0f88cbd5848", "firstIndex": 4697619586}, +{"blockNumber": 12541655, "blockHash": "0xc34c7299041df72a81c4b86e81639d2dfcebfae603e287fc143881a636ce1188", "firstIndex": 4764728948}, +{"blockNumber": 12597319, "blockHash": "0x4c4dc71d8d2f1223ad0469ddfe09fa8934f64e3712ed83708d6e1b9c64eceebb", "firstIndex": 4831837390}, +{"blockNumber": 12651863, "blockHash": "0x406a642c8364fb61cbd896512a2d19ecdba3932ab09ac67cf48d91aa1a03f24e", "firstIndex": 4898946362}, +{"blockNumber": 12706391, "blockHash": "0xa7fc3739898f7ff5733896313b248fb0b306a132681609b8b9e2671b42dac3a1", "firstIndex": 4966054748}, +{"blockNumber": 12762840, "blockHash": "0x5a1048966dfa55a48248517c45683ce342671bf544117d2bac13d8e343563136", "firstIndex": 5033164303}, +{"blockNumber": 12816597, "blockHash": "0x97f5b87f422870a8ffeff70c12f4377dde96d248aa912cb64ebfaea51dc086b6", "firstIndex": 5100272541}, +{"blockNumber": 12872315, "blockHash": "0x432f7df7973fdea4e5abe2c09b634ca3b724e37a19d5ec23ca5810e4854c6b33", "firstIndex": 5167382243}, +{"blockNumber": 12929620, "blockHash": "0xa4524e3b4c5dadac91df2f29696532cfed377f47469fe7cfabf10baac945eea2", "firstIndex": 5234490748}, +{"blockNumber": 12988653, "blockHash": "0x9be70e31aeea164c370d8d183a1af45120693b13d360edc78214535c40ef1d63", "firstIndex": 5301600248}, +{"blockNumber": 13049074, "blockHash": "0x3076857bd17959c2a3df18357eab3ecbb7ca25457785b07074d2e885eb566fc2", "firstIndex": 5368707746}, +{"blockNumber": 13108823, "blockHash": "0x67b2764f6b53779e3d683def2c7659c6d92814152574073d24f10cd67200a22b", "firstIndex": 5435817680}, +{"blockNumber": 13175389, "blockHash": "0x0d41e34dbca58dec45e0509b6cbf027b2377220845a85caebbf3a14b8505e8ce", "firstIndex": 5502925085}, +{"blockNumber": 13237624, "blockHash": "0x723e0818d54ab1b1c53efc2f2f448c16c1f943af2033575c612c3c71228d4ddf", "firstIndex": 5570034948}, +{"blockNumber": 13298658, "blockHash": "0xd7f9f477b20bcf1379ccaaae701c9658a02b177861e0af9e873c63fe8b854f4b", "firstIndex": 5637144562}, +{"blockNumber": 13361166, "blockHash": "0x9f9e4130093269d59045f91344a13eb2786023e02099f53dd44854d50a7f5ffb", "firstIndex": 5704253260}, +{"blockNumber": 13421716, "blockHash": "0x28752f35ad2adeba342ef6aa51bca1c0abc757c4f712cf620c8a5e6a8576636e", "firstIndex": 5771361775}, +{"blockNumber": 13480516, "blockHash": "0x43f4ecf24c17b7a77621e2ca09c88c53b3407fb018f4bb3a032c212015b4e18d", "firstIndex": 5838469616}, +{"blockNumber": 13535688, "blockHash": "0x99ffcd9982219a4be45fa5f15622b288afe4f85da536956d39dccac1ec416444", "firstIndex": 5905579739}, +{"blockNumber": 13590487, "blockHash": "0xff7b74464a318037e0670c0be7a5b1b79418d1a42c7a3587c46adc1ef720c819", "firstIndex": 5972688013}, +{"blockNumber": 13646596, "blockHash": "0xd555ce5f2b65ffeec9b9e3e75a56f5f23d7353a1014bcc861efd7df6e5dc2c92", "firstIndex": 6039797525}, +{"blockNumber": 13702902, "blockHash": "0x3e8feab590c6631c63931fee83bf7e10234890bdf377b1b374fb70221b9ac9c0", "firstIndex": 6106905641}, +{"blockNumber": 13760908, "blockHash": "0x18e96fa585c57aad1f2a8dc9639d9f4fe53c3f38894862471b7cec17f97f3cb1", "firstIndex": 6174014330}, +{"blockNumber": 13819340, "blockHash": "0x25384bf2cef178e30c4d9e0abae59357055241fc715133b2bb078e8278e5296a", "firstIndex": 6241123906}, +{"blockNumber": 13877817, "blockHash": "0x4a84a0a7035dbfed0e1281fb1df162a195f2c516686a2e2a6ad605e6f2b3cf57", "firstIndex": 6308232690}, +{"blockNumber": 13935260, "blockHash": "0x6c14a8e563614edbad66698ebe911e997e27844f9d33c7069795379540e71a44", "firstIndex": 6375341148}, +{"blockNumber": 13993914, "blockHash": "0xc0991d1bc431c5215fb0218b0b541f2b4fae0994775e5eddb016965fe5b4a0c7", "firstIndex": 6442448947}, +{"blockNumber": 14051007, "blockHash": "0x854803ce364b6fe4446800e74448a5ee883e7d472e8a0347f1ad13bd347fa69f", "firstIndex": 6509558964}, +{"blockNumber": 14109063, "blockHash": "0xa3fadce7ab18a5fedcfed9e3b1f0cf2fb32151172c362327c602343bc26dde8f", "firstIndex": 6576668017}, +{"blockNumber": 14166701, "blockHash": "0xaa2443a38e2b0d6b01cf4b73de54d2b7e92c99475bd0802beebd86845c126d1b", "firstIndex": 6643774648}, +{"blockNumber": 14222367, "blockHash": "0xa27c78134c1fbf432c8caee7c9d2ef323fdde519a065804c9c8da6ca1615e04f", "firstIndex": 6710886238}, +{"blockNumber": 14277029, "blockHash": "0x5d0ed08b65db01e382c339dcb2715b2927fb472b0c1f4256841c386d785c3f3a", "firstIndex": 6777995247}, +{"blockNumber": 14330955, "blockHash": "0x3e5014d82c416549acfe2e26b13ff5ddf29ad74943ef5b94a65eca0a4d72d82e", "firstIndex": 6845102969}, +{"blockNumber": 14384139, "blockHash": "0x720e1fd687292019c3fa1812c39c73d20fde783c0196f43378a51cb4e0b46d65", "firstIndex": 6912212078}, +{"blockNumber": 14437534, "blockHash": "0xe4342ab6f363066d9c2514a870241ceb63df232d33659fdb3928faf433360bb4", "firstIndex": 6979321493}, +{"blockNumber": 14489073, "blockHash": "0xeb2f9923d816e773460eccfa42eb4ff2bbfc353b756ba274e98dc85c64ab1536", "firstIndex": 7046428879}, +{"blockNumber": 14541559, "blockHash": "0x55cc33721fa9022964f47c30f4146fee7cc80f5b4158d7ecea0ff661cbba3487", "firstIndex": 7113538857}, +{"blockNumber": 14594432, "blockHash": "0x676441bdc1ec430a9fbc05e56a0437a475606a253f9fdffc86f5f6286c159b5d", "firstIndex": 7180647956}, +{"blockNumber": 14644942, "blockHash": "0x03232a1d3c9b2f1f0c2a4b780c83bce66238d9abdeed71c2a8af9e97a53011b0", "firstIndex": 7247754981}, +{"blockNumber": 14695793, "blockHash": "0x6017f6d1d79188692e1a1a18de5f93b83b97905c8ff5bed5cef4b2e9f4d4d41b", "firstIndex": 7314863646}, +{"blockNumber": 14746186, "blockHash": "0x332c8ac0b2960fb1627407d45443201a0f6de7421b52af077b32fa6e018e5401", "firstIndex": 7381974499}, +{"blockNumber": 14798403, "blockHash": "0x4507ae1d9f27a3eada874dcf04112a5d4ee7155701afd3fb857fe58201a2dede", "firstIndex": 7449082752}, +{"blockNumber": 14848093, "blockHash": "0xff98c820c66c3beaaa3b195158c94f099148573a7f032295ba7156c13f78eaa8", "firstIndex": 7516191978}, +{"blockNumber": 14897508, "blockHash": "0x49c31d3433b197585f733d7c382ae16d9d78bf59c333a9b76362cc6c5fed270c", "firstIndex": 7583300663}, +{"blockNumber": 14950638, "blockHash": "0xf1a1609206788977841927271808fe1c42d059026385d025bf4cea0c0a05966b", "firstIndex": 7650408992}, +{"blockNumber": 15003958, "blockHash": "0xf9e6b41108d907672f29ba1df97ed83aca2ba933335e6b5ccfb5d4001807b33f", "firstIndex": 7717519105}, +{"blockNumber": 15056783, "blockHash": "0x126a5a5c22e878a4c42bea890d48cfe00edf7fbc27d805c88a5269540bc94a42", "firstIndex": 7784628173}, +{"blockNumber": 15108163, "blockHash": "0x5f8494ccacefce9fef8a95a52247f2670ed30bb5a3b3deb85c645e72c115d506", "firstIndex": 7851735261}, +{"blockNumber": 15159385, "blockHash": "0x0fed6c5aafb53e9cdb0d8620c3eef9b619fb3ae9559296618f634eddae33f0c6", "firstIndex": 7918844299}, +{"blockNumber": 15210872, "blockHash": "0x37bdb94b57044b67fbd9b9a7cdc728fcfd2066cf1904cab478fd42d41ec6e1f0", "firstIndex": 7985954119}, +{"blockNumber": 15264226, "blockHash": "0xc173ad8a791e2293efd8e0b2c9f088fac49233aabb1e72db6a045aa9904f4728", "firstIndex": 8053061840}, +{"blockNumber": 15315693, "blockHash": "0x985cf97bf28f0becb2b16abfe629a5f520596e45fc23f4936bc2f6298b8145f9", "firstIndex": 8120172048}, +{"blockNumber": 15364606, "blockHash": "0x593362ef8a82140f3dd3ff13d0ce50287dcb0b8b3d31ffcc7813c6be16f7438d", "firstIndex": 8187281275}, +{"blockNumber": 15412727, "blockHash": "0x5a6ff10acc48e9e8bcf963b042012ea094628b9488b8bf569783b3297c67eedd", "firstIndex": 8254388575}, +{"blockNumber": 15462642, "blockHash": "0x137f15c5d112b0fdc6e4c84393032030646122c3b0d86286143a1ab8bfacbac4", "firstIndex": 8321497214}, +{"blockNumber": 15509781, "blockHash": "0xe58a8ca223b7c03755d046a6212bc33fa9814f73efe52804d8eea7ae4b4bd7f9", "firstIndex": 8388606496}, +{"blockNumber": 15558625, "blockHash": "0x4127d527b58c7685a1237d228508ea77d40f33dbeffb5cc30e47e83a4b6152a5", "firstIndex": 8455714910}, +{"blockNumber": 15604075, "blockHash": "0x7d1c1bccfe5c23a1edd81246e6e55277f271cffef30bde22c9d1202f764dd96f", "firstIndex": 8522823324}, +{"blockNumber": 15651742, "blockHash": "0x732e5203a504fd97510325c1fb876476ca132831e78182367cd4b4efae1232c3", "firstIndex": 8589934155}, +{"blockNumber": 15700810, "blockHash": "0xe031decb5e42a69dafcc814441227085508cf51c9c77f35d06487db19bd05a6a", "firstIndex": 8657041374}, +{"blockNumber": 15762855, "blockHash": "0xbdff9b1e193c44f35bf38945d2523c74ef59c4f352efc93fad02ef34eef1c5f0", "firstIndex": 8724150924}, +{"blockNumber": 15814304, "blockHash": "0x524a226d9f48e37db00e252c7f6a1350d1829d38fe83c756021cc11c56d3b2dd", "firstIndex": 8791259373}, +{"blockNumber": 15865514, "blockHash": "0x4e4ad46f3ae7fb2a2e7cb94aa804929cb7353c79c78cc2cbcee710da5f0ec4b5", "firstIndex": 8858369571}, +{"blockNumber": 15920411, "blockHash": "0xa39e4cbd8e84b427e44b8d06b5b1149652f3af5fa88633b802665b8677772bc5", "firstIndex": 8925477782}, +{"blockNumber": 15974193, "blockHash": "0x4a3f82935e6fcda6ce7636001989b65b79422d362e60833e01185a7960b2fa55", "firstIndex": 8992587369}, +{"blockNumber": 16032739, "blockHash": "0x7f50bb76dfd01125e0a4a7595106e43329eb8bd02e819be2ade4248a403c714a", "firstIndex": 9059696611}, +{"blockNumber": 16090896, "blockHash": "0xdc4db130973ef793d2223844e35122a2477e642d86c69616fb3604d2a79805c4", "firstIndex": 9126804425}, +{"blockNumber": 16150783, "blockHash": "0xb46294fe1cbcbbdff7309ebf0bf481e78e2589d386b95fa4c781c437e02e0aec", "firstIndex": 9193912139}, +{"blockNumber": 16207278, "blockHash": "0x8a3af0387572b56989bb6c98d8860e07b5cdefb2ebe7b1dc58be07c24a53e056", "firstIndex": 9261022215}, +{"blockNumber": 16262389, "blockHash": "0xec5a03d88cfc1bf8c278ddce7ace5032c3f7ed5032b4575df54779ec1abca131", "firstIndex": 9328130680}, +{"blockNumber": 16319525, "blockHash": "0x74ae702375e242205fe4986448b0ee494d4a38ccc813981489c4b0488e58a949", "firstIndex": 9395240779}, +{"blockNumber": 16373441, "blockHash": "0x5f404555cf45c547ef0fc044f2701bc7a28ce1e432de06e0264dbfd1c17447d0", "firstIndex": 9462347671}, +{"blockNumber": 16423343, "blockHash": "0xe4af890b6c4dd3b10af2e6c08f14563bd2a3121b22a3cc6a0db29d438c1dcc59", "firstIndex": 9529457704}, +{"blockNumber": 16474687, "blockHash": "0x019a8b374cf150fcda1c8af2a3e7ba63603ae81744d6343897c4b17ee58411d5", "firstIndex": 9596566169}, +{"blockNumber": 16525504, "blockHash": "0xad394ce7e84e1c904cc5dd6dabd51b1a575c0b2fb006f10f272875b7f921904e", "firstIndex": 9663675287}, +{"blockNumber": 16574051, "blockHash": "0x098e2dca2eda0d700aa3127bdfb679ce16e3ca297a305978c6ecfff044269511", "firstIndex": 9730784563}, +{"blockNumber": 16622457, "blockHash": "0x8627f1237731c1127116898724f1b6420b8e7bcd8e18a0ea568f192bbd2cd55d", "firstIndex": 9797893241}, +{"blockNumber": 16672410, "blockHash": "0x3e32b46faa670b191b89bea84a68022d5ea8b90008b388a948ff27f871d17292", "firstIndex": 9865002964}, +{"blockNumber": 16719964, "blockHash": "0x1668c3ea6cb256f306e93851cc2c00612f85df5cf7b5527ffce14c89088d7fdf", "firstIndex": 9932111708}, +{"blockNumber": 16769014, "blockHash": "0xed9a6f88e47b5bfe4d5bc92e1c1bfeb252f716f0990ffa1f907655c0f5e3a365", "firstIndex": 9999220312}, +{"blockNumber": 16818134, "blockHash": "0x93504ba617500698a1b46f3110cd26827b8efe468190b2a91d3595784e34a1b6", "firstIndex": 10066329167}, +{"blockNumber": 16869369, "blockHash": "0x94ca4b395a5c36c4e9d07cbb08271f0680455af4e4659906b647e16abebd7d4e", "firstIndex": 10133438415}, +{"blockNumber": 16921428, "blockHash": "0x66650775bff4cb95de56099c10b06f3dfe302a731b391f406d1aa551121308ba", "firstIndex": 10200547148}, +{"blockNumber": 16974677, "blockHash": "0x58d5fc4cd45047dac10d55a036e0a4448faacc1041b29c4c7fb41a6e9394c3ff", "firstIndex": 10267655623}, +{"blockNumber": 17031050, "blockHash": "0x9d15b831979e95fc4149ecae46478cac1e54470b277d57d7102d19a72f28eeba", "firstIndex": 10334764441}, +{"blockNumber": 17086163, "blockHash": "0x978ff48863fcdb50b274ea4220e7635c13c41bae91912db9e24205fb917fde85", "firstIndex": 10401873176}, +{"blockNumber": 17141086, "blockHash": "0x4084d442499893361731f17534944d260cccd8e07d41c52d7160910555770b90", "firstIndex": 10468982462}, +{"blockNumber": 17190735, "blockHash": "0x42ed86fb42def23870df6ec23204f557669ec7cd8de59c9f3ef9ab90704e4b11", "firstIndex": 10536090726}, +{"blockNumber": 17237026, "blockHash": "0x9a49d943f82a7335f8049df4d953080f25920e3b098f6a2d872520166b225961", "firstIndex": 10603199938}, +{"blockNumber": 17287007, "blockHash": "0xac3d189676d24eadc62adcf97d8730a001a7766fe3ac27b18e745ca9727f6c46", "firstIndex": 10670308186}, +{"blockNumber": 17338498, "blockHash": "0xe50e251aa0a3209fa8a0030b237cf8357b08eeb80492a432c8aa5969a820ee42", "firstIndex": 10737417623}, +{"blockNumber": 17389178, "blockHash": "0x1e18cedcfa6c81bc845c9136fc92f9f27ca8fff8847289bb139f374f0f6ba733", "firstIndex": 10804526769}, +{"blockNumber": 17442038, "blockHash": "0x07f47ebbee99938fb96beb1ffadf8266f71d0d73f393172f0e24facd40d45c97", "firstIndex": 10871635940}, +{"blockNumber": 17497589, "blockHash": "0x247a5e852e36e46f7897ad1875e93d810ff4434bb835b469afae3a3d3d4a9c9a", "firstIndex": 10938744251}, +{"blockNumber": 17554111, "blockHash": "0x420e420a1b599e4dc3bd115652b4d6f5eedc391086ef064f97a9cbffc02b62f9", "firstIndex": 11005853686}, +{"blockNumber": 17608282, "blockHash": "0xb2ba644d3593d3c832f932f14ffec532e26e9e1cdcd8fbe7477f46e238c6bffd", "firstIndex": 11072962303}, +{"blockNumber": 17664091, "blockHash": "0x6d0bfff8145652a617bfe03ab87e3af09219cdb822c9c214b42b16f20742c1eb", "firstIndex": 11140070260}, +{"blockNumber": 17714917, "blockHash": "0x61509544b5174d3ff5d48816ff54f2929c8a400ef21454116056baa136b607b6", "firstIndex": 11207179013}, +{"blockNumber": 17763850, "blockHash": "0x8da83012621f42670601406c902ce6e48e77c6285dcc6983e108e69b3a8d5d2d", "firstIndex": 11274288732}, +{"blockNumber": 17814186, "blockHash": "0xc37bda200856589c220dc206e31186d6c12761fd193f5bc7123cf331fbbed7f5", "firstIndex": 11341397343}, +{"blockNumber": 17864461, "blockHash": "0x99af43893dc9f54c55abbdbd7b000e27248145371e9b00db116dfb961b36749c", "firstIndex": 11408506122}, +{"blockNumber": 17913212, "blockHash": "0x9b642236d63d756fcd3404713bc6abe97804b46a9e665ae5f1d9aa48e693fe78", "firstIndex": 11475614347}, +{"blockNumber": 17961497, "blockHash": "0xaec9b1e851cf51646673ec7698e1d5964d44eb04729ad11385f6ae84d497070a", "firstIndex": 11542723713}, +{"blockNumber": 18010859, "blockHash": "0xb15fb7fc308516cad5f1dadfb7dd92fd3fe612308f67154cf2a241990aa15bc4", "firstIndex": 11609833204}, +{"blockNumber": 18061052, "blockHash": "0xd8ea94b8b8a7380d21dc433abb69973f3de2eeb339f4aaffdb79f85a5d86929d", "firstIndex": 11676938565}, +{"blockNumber": 18111466, "blockHash": "0x5b5cc6d182bffaf06692a7efda8cad5d4742fdbce5c1146b3be3c5be10ea046a", "firstIndex": 11744048539}, +{"blockNumber": 18166001, "blockHash": "0x901d8401c795542c75069cd917781fa587ad416da0a5298f144fae2f14305705", "firstIndex": 11811159949}, +{"blockNumber": 18218505, "blockHash": "0x5894285023f34ec6fd501cd90b605e118b9688ca7c01082289bf69a6987142bc", "firstIndex": 11878268395}, +{"blockNumber": 18271037, "blockHash": "0x3f0fce315fd993d966b6c599ebd360462466e67046d512dc430ac07f75243bae", "firstIndex": 11945377050}, +{"blockNumber": 18322778, "blockHash": "0xfa6536b3b06e7b579e53a8d01368182808c950586e39e12485be3145c3acbf43", "firstIndex": 12012486036}, +{"blockNumber": 18372240, "blockHash": "0x5531859d5b8a4093020e2fb971808d0eeaddd7d61dc58d70f34923081f97336f", "firstIndex": 12079595358}, +{"blockNumber": 18421630, "blockHash": "0xfeadff676fe8f04d0362b77ea2d7c3ea59b05d68cc6b2cc95cfe751558b1bfa7", "firstIndex": 12146703872}, +{"blockNumber": 18471472, "blockHash": "0x352b0a17e4160d334b876eb180d686fd67c7ddc40a78219b0bc4b59f5ae5912d", "firstIndex": 12213813174}, +{"blockNumber": 18522088, "blockHash": "0xf7d408309246089531b47c787049c7ffd2820c1b87b4804033a458bc193a3432", "firstIndex": 12280921181}, +{"blockNumber": 18572725, "blockHash": "0x44126d4a5b43594b09d26e0b0a5dc859c074865cb9b11ab78c80f28406fd0240", "firstIndex": 12348029615}, +{"blockNumber": 18623223, "blockHash": "0xd3aec1f98cea30e870d0f530194e9a72e225eb45eefe435e38da5967eb2fa470", "firstIndex": 12415139522}, +{"blockNumber": 18675228, "blockHash": "0x24deaef2508038bddf8e9020f95128ae34eb1786f786c3f98df0bdf95f3e97cc", "firstIndex": 12482248123}, +{"blockNumber": 18725495, "blockHash": "0x2e5a7edd7337b20e862741e116680a472089426dbc2b0eb5f1c984f8d6da935b", "firstIndex": 12549357121}, +{"blockNumber": 18778174, "blockHash": "0xe734a1084fa5eeff427bba8e8d3623181a30dd9bad128fc22de9b15dffd0e549", "firstIndex": 12616466131}, +{"blockNumber": 18834807, "blockHash": "0x0924a55ef4f9c2605261433c5b236d47645484efa44b34dc9f15b7c2e6b47a74", "firstIndex": 12683575000}, +{"blockNumber": 18883054, "blockHash": "0xef638ec491a1cf685b22bf4072e41710440d6133a53e31250762825247d3eb64", "firstIndex": 12750683712}, +{"blockNumber": 18933405, "blockHash": "0x6e9eefc2e121d13e6c04544ea370fe8543afeb36fef3a8e2ea68afb079738c8c", "firstIndex": 12817792510}, +{"blockNumber": 18988033, "blockHash": "0x4c98c0dbb6a7217609c4b277fe7ff9e2c32c8d000ee628a3f37b2b296e1ca421", "firstIndex": 12884901389}, +{"blockNumber": 19041096, "blockHash": "0x61a219a23d111cd07861ff47b89d4b2840c0ab5421d5d55a6f1fe43da3f33b98", "firstIndex": 12952010147}, +{"blockNumber": 19088914, "blockHash": "0xefb26ba58447e50e9b820e0b0d16c08da45793a824af12a2e1f38b6f46b7e36a", "firstIndex": 13019118891}, +{"blockNumber": 19140413, "blockHash": "0x91dbb13623fc4d5a4e462dfa0a03c92bf5301948dd4b2c3e5fddd2bc88856d47", "firstIndex": 13086228137}, +{"blockNumber": 19191876, "blockHash": "0xdd7f97af7ad206b2636656ed4722fe1c47c6ba9fa5531066785acbe4f8d4c4ad", "firstIndex": 13153336551}, +{"blockNumber": 19237651, "blockHash": "0xf1e875f314314abe3654b667c6443c772dd67c525c52877b06515ea8fc5a391b", "firstIndex": 13220444812}, +{"blockNumber": 19291032, "blockHash": "0x9eda269ac47957b619d2a8c408b5efc8cfb168a46bfde351f43325fa337fce47", "firstIndex": 13287553967}, +{"blockNumber": 19344225, "blockHash": "0x62859b78d45f589f679fc6a3bf266a2e0aa906fd61d787755ec51cfadc05eaf6", "firstIndex": 13354663338}, +{"blockNumber": 19394709, "blockHash": "0x7cda3d193bcfe8ddbc89b85e98c814dbbe31a54ccab52ab7738ecd003061f3c8", "firstIndex": 13421772549}, +{"blockNumber": 19442850, "blockHash": "0xd95f3f8bab5b91c3e1f848c9455936850bf92cf6a5086b817f6f81031bc0f81e", "firstIndex": 13488880704}, +{"blockNumber": 19488177, "blockHash": "0x3a1a9623ee332d6decada9359cf84eeafb81fffa997d6cf4f3efcc92e98e9310", "firstIndex": 13555989570}, +{"blockNumber": 19534365, "blockHash": "0x5c7a8ee7cbba4894ea412cc335895a9a7d3369d35779bcdc97a6c24b6a81a7ba", "firstIndex": 13623097446}, +{"blockNumber": 19579409, "blockHash": "0x4280f5f76cb0f9947f9da1cb9de6c4b6e65b37b3a3c174f5fb29d360cf389ffb", "firstIndex": 13690206790}, +{"blockNumber": 19621406, "blockHash": "0x0a98ca72e97fd295c7c73263b17185d74dbe427f41c616a5d5454faa5a1c63a0", "firstIndex": 13757316653}, +{"blockNumber": 19664868, "blockHash": "0xc8c53d89b7a849861f3daaf9503e6397a0340cecb085850c3636bee6ea2b7189", "firstIndex": 13824424608}, +{"blockNumber": 19709011, "blockHash": "0xf65babc88ef6a9eb2ffce9a6844adf895995f65a2facd1956f5df17ca68dbf94", "firstIndex": 13891534141}, +{"blockNumber": 19755170, "blockHash": "0xf903882449eeb86d69af33b6797099275e82d8787334a02a42a2168dbd739b95", "firstIndex": 13958642627}, +{"blockNumber": 19803666, "blockHash": "0xa8a975f570db082a3771790bb1fe405667dde4674ff49e31092687fa92727e36", "firstIndex": 14025752433}, +{"blockNumber": 19847730, "blockHash": "0x89d479a4b4d9faa7af3e3f61d13a7a9c80a11c4adf7cd018859eb0ddba30bdd5", "firstIndex": 14092860598}, +{"blockNumber": 19894138, "blockHash": "0x740278e6dc79cfd2b030780868a1b8e1520dbae0e8d3ee0a3ef43c84c83c6e0a", "firstIndex": 14159952129}, +{"blockNumber": 19938334, "blockHash": "0x05ae7ce9813235c19baa57658bc5c8e160130328a1fd2cc58d2baf2d9a4ae097", "firstIndex": 14227078393}, +{"blockNumber": 19985138, "blockHash": "0x511c8876a93a77fa473ae0d29e25d8163162c09b7212f506be0ea73840ea3971", "firstIndex": 14294187397}, +{"blockNumber": 20028221, "blockHash": "0x64379baedf429cfa65df076142a7d9909d7e369d6aa6b8e161bb686944d169ff", "firstIndex": 14361296446}, +{"blockNumber": 20071555, "blockHash": "0x4f43d8b1091d2bd0c6b080ce51bce08aae6be605e00889be3cdd17cb80614ff8", "firstIndex": 14428405409}, +{"blockNumber": 20113611, "blockHash": "0xebbf87409ef6732a53513414926b0aa998a7269f9e9717a3acbe96a33c2de937", "firstIndex": 14495514255}, +{"blockNumber": 20156628, "blockHash": "0x70218108c0669f42c4ac1afff80947cf4a0a2fd8e9300b31d45bbe330a5159c9", "firstIndex": 14562622257}, +{"blockNumber": 20199886, "blockHash": "0x1d6aaa4b041eb0cc9caed34347bb943e50a139caf8998d781ff4f06d4aa727dc", "firstIndex": 14629732017}, +{"blockNumber": 20244148, "blockHash": "0x37fed57b6945a815ec56c84c6261e90cf9f3883d9cbffc62b70d9722c1f0f363", "firstIndex": 14696839730}, +{"blockNumber": 20288249, "blockHash": "0x1d9fe664431e093c4ef6139ad35fea767a2e66825e605cfea9e91d1738665619", "firstIndex": 14763948803}, +{"blockNumber": 20333358, "blockHash": "0x1ac7c1362791507057b7e87d7118b4f61c2919c8ebdb498ecf7d64a59ba3d2e2", "firstIndex": 14831058814}, +{"blockNumber": 20376855, "blockHash": "0xf6a19eac8a402f591079a4aa7ff57349c350ec96d683a031e4d85c05bc0541b2", "firstIndex": 14898167076}, +{"blockNumber": 20421474, "blockHash": "0xe8767aa7516b632dc8c0fed3a366273ef14bd7b8c2212998b7105fb80b4c0d9b", "firstIndex": 14965276285}, +{"blockNumber": 20466974, "blockHash": "0xafa2d2e6c9594762c163c339069cb15aa9fd27d301268892e12f99a3f2912ade", "firstIndex": 15032384674}, +{"blockNumber": 20512183, "blockHash": "0xc1c51f9372d781d7ac0a1a7aa052503bbde3e1706bf65f225b0f6a6eed93d055", "firstIndex": 15099493832}, +{"blockNumber": 20557251, "blockHash": "0xa905087c15e2b12b19cd23f20089b471cd9815dcff73842d2043e13a276fcc55", "firstIndex": 15166601921}, +{"blockNumber": 20595654, "blockHash": "0x9e7772dd946f65e7569f34f6896d8e597ac8ff88700b871de21ff578c0c995e6", "firstIndex": 15233711474}, +{"blockNumber": 20638340, "blockHash": "0xc08b46af83db6f0e5d1acad7020bacd59d0f18df72651d32c9b60edd0351a886", "firstIndex": 15300820584}, +{"blockNumber": 20683359, "blockHash": "0x7ec93a1f1d211c24d93ce7e344634efef680a6388374cf3831d250ecd63711f9", "firstIndex": 15367929168}, +{"blockNumber": 20728401, "blockHash": "0x4ad4124c2595abcf4477cac4eb70c0c9372bbd48f2066c7ac5b8e9f94817ab48", "firstIndex": 15435038303}, +{"blockNumber": 20771272, "blockHash": "0x68fd6b12fdb4add8b2957105a21b571c7dcb114c6c2ca02f5fa40730a108f864", "firstIndex": 15502146509}, +{"blockNumber": 20814862, "blockHash": "0x4e7f640bf3c135b2442ca4c744023021af163e8d45e8d776fb7051c49d79fcff", "firstIndex": 15569256173}, +{"blockNumber": 20857627, "blockHash": "0x9b023bffed12530b5a68db263bb621a147e0cf02b30ec39a82b82a1fdeb7ca36", "firstIndex": 15636364955}, +{"blockNumber": 20896662, "blockHash": "0x8c3adf694b8520ed1edd38282b2f58cf383426176628b5be6330a56882e2a07e", "firstIndex": 15703473445}, +{"blockNumber": 20939156, "blockHash": "0x0649e2559f767da945a674c9d6c700628109e18d0757ad33496a228a827413a4", "firstIndex": 15770581979}, +{"blockNumber": 20981054, "blockHash": "0xebf0ecca671b833f77df70106cf1171d4dd3fa0f062607e4845569073997f1cd", "firstIndex": 15837691319}, +{"blockNumber": 21022980, "blockHash": "0x07479520c08a727c42e3616aca7fc4435773f4ef277d84369190263159606d71", "firstIndex": 15904799391}, +{"blockNumber": 21068139, "blockHash": "0x669cfd03b293a61815588cb9cef928a512d328c1fe9191fe6b281573f429f8b8", "firstIndex": 15971908604}, +{"blockNumber": 21112368, "blockHash": "0x98957aedaf8b0483be208e8d16a3e6d73ff217e732f3649650671c92c8fb57a5", "firstIndex": 16039007815}, +{"blockNumber": 21155756, "blockHash": "0xc301e40988fbe19b1254ab3abe2f78fe0cd35259898be02476179a041926f777", "firstIndex": 16106127130}, +{"blockNumber": 21200717, "blockHash": "0x8d35b199ee65c5ccf4ca0fc22b50282850cd88b72580679f1568eea67d3431e8", "firstIndex": 16173235593}, +{"blockNumber": 21244396, "blockHash": "0xe7ec084eb8a7125fe26c4ef64fdbb3e0a727b2d882fb1e4fdad1d7fa3dc55fd2", "firstIndex": 16240343803}, +{"blockNumber": 21290031, "blockHash": "0x8bb31b883d30875b7ed8a01c35a6f51e603b8d974c4e74af20fb9c723e5ac5df", "firstIndex": 16307452772}, +{"blockNumber": 21335893, "blockHash": "0xd0976b7c946a7556e607e47a02d2cfcccd259508e2828b9c4faa5c7105a597a8", "firstIndex": 16374562657}, +{"blockNumber": 21378611, "blockHash": "0x72d42953f013aeb7c84bb22c9e07828b92f73885b20c208af612874df8fe85ed", "firstIndex": 16441671192}, +{"blockNumber": 21421343, "blockHash": "0x663b3cddcf3f7cfa59ed29b4894e43fa784c07576cfec897ff1a3838235ecb50", "firstIndex": 16508779766}, +{"blockNumber": 21466598, "blockHash": "0xe04ebd60289dbcc09a27e053e7b240a59e928e2a76a85a6fa4044ee387c46d18", "firstIndex": 16575888836}, +{"blockNumber": 21511717, "blockHash": "0x5ea9863a0b504f381010ff383d60a56390ba9cef162d9d5a077b1884d2adac98", "firstIndex": 16642998234}, +{"blockNumber": 21550050, "blockHash": "0x2a7b3e635c80521887ca3ebda04541090ec720dd817cd06fcba61e7ae0f78a4f", "firstIndex": 16710106733}, +{"blockNumber": 21592440, "blockHash": "0xc7095268d0d53cb5a069c2ea7107cc42172bb1b817f04154938e6b52bdbb369c", "firstIndex": 16777215261}, +{"blockNumber": 21636007, "blockHash": "0x55f463b932053ebfc02728a12c2cdfb2cb4700a115cde686c9ab22bb4c336ecb", "firstIndex": 16844324844}, +{"blockNumber": 21680799, "blockHash": "0x4521d473aa7f5150f920c66fe639b20f30cb373b32e425fcb7dcce9803b506be", "firstIndex": 16911431255}, +{"blockNumber": 21725330, "blockHash": "0x793876007ce3594df603628766b139a02273ceee43d5b3ca2d0ca2f13a5fc5d9", "firstIndex": 16978541626}, +{"blockNumber": 21771310, "blockHash": "0xd0db23bb28d6e39917348b1879dd560c73354ea6d7d0bbee0b7a8fcbcede1091", "firstIndex": 17045650902}, +{"blockNumber": 21811348, "blockHash": "0x8165948c08db34aeb8cc2a3e331a00af2d09fbff9c466019fb2f2892ca3f7b71", "firstIndex": 17112760024}, +{"blockNumber": 21851615, "blockHash": "0x3083e8f4fbb4d726ef19b1a5bce20e639a0418eac384f9967f24c7697a7e2dfc", "firstIndex": 17179868870}, +{"blockNumber": 21894041, "blockHash": "0x9b0776a020374e4985f6a7e1199596bedd387ff085f65224d724ab11955c56fe", "firstIndex": 17246977342}, +{"blockNumber": 21932218, "blockHash": "0xe9c50eddc21759d3ab790e2babba56f4b8d38eb739f1b561bc337a1ba6e15393", "firstIndex": 17314085541}, +{"blockNumber": 21959188, "blockHash": "0x6b9c482cc4822af2ad62a359b923062556ab6a046d1a39a8243b764848690601", "firstIndex": 17381193886}, +{"blockNumber": 21994911, "blockHash": "0x0d99ef9dd42dd1c62fc5249eb4f618e7672ab93fa0ba7545c77360371cf972e5", "firstIndex": 17448302795}, +{"blockNumber": 22026007, "blockHash": "0xf7cdc7f6694f2c2ed31fa8a607f65cfa59d0dd7d7424ab5c017f959ae2982c71", "firstIndex": 17515413036}, +{"blockNumber": 22059890, "blockHash": "0x82b768a0dddefda2eefd3a33596ea2be075312f1dd4b01f6b0d377faca2af98b", "firstIndex": 17582521768}, +{"blockNumber": 22090784, "blockHash": "0xf97c2eaf9a550360ac24000c0ff17ffa388a2bdd6f73f2f36718e332edfa107a", "firstIndex": 17649630983}, +{"blockNumber": 22121157, "blockHash": "0xa790025235db782e899f23d8b09663ec2d74ec149e4125d62989f98829b08e2d", "firstIndex": 17716724973}, +{"blockNumber": 22148056, "blockHash": "0xbe25ac4f1bdd89a7db5782bf1157e6c4378d80220d67a029397853ef16cd1a4c", "firstIndex": 17783838366}, +{"blockNumber": 22168652, "blockHash": "0x6ae43618c915e636794e2cc2d75dde9992766881c7405fe6479c045ed4bee57e", "firstIndex": 17850956277}, +{"blockNumber": 22190975, "blockHash": "0x9437121647899a4b7b84d67fbea7cc6ff967481c2eab4328ccd86e2cefe19420", "firstIndex": 17918066140}, +{"blockNumber": 22234357, "blockHash": "0x036030830134f9224160d5a0b62da35ec7813dc8855d554bd22e9d38545243ed", "firstIndex": 17985175075}, +{"blockNumber": 22276736, "blockHash": "0x5ceb96d98aa1b4c1c2f2fa253ae9cdb1b04e0420c11bf795400e8762c0a1635c", "firstIndex": 18052284344}, +{"blockNumber": 22321282, "blockHash": "0x8a601ebf6a757020c6d43a978f0bd2c150c4acc1ffdd50c7ee88afc78b0c11f8", "firstIndex": 18119392242}, +{"blockNumber": 22349231, "blockHash": "0xb751c026a92ba5be95ad7ea4e2729a175b0d0e11a4c108f47cab232b4715d1a2", "firstIndex": 18186501218}, +{"blockNumber": 22377469, "blockHash": "0xa47916860a22f7e26761ec2d7f717410791bd3ed0237b2f6266750214c7bbf08", "firstIndex": 18253610249}, +{"blockNumber": 22422685, "blockHash": "0x8beaee39603af55fad222730f556c840c41cd76a5eef0bad367ac94d3b86c7aa", "firstIndex": 18320716377}, +{"blockNumber": 22462378, "blockHash": "0x6dba9c5d2949f5a6a072267b590e8b15e6fb157a0fc22719387f1fd6bfcd8d5d", "firstIndex": 18387828426}, +{"blockNumber": 22500185, "blockHash": "0x2484c380df0a8f7edfdf8d917570d23fab8499aea80c35b6cf4e5fe1e34106e9", "firstIndex": 18454936227}, +{"blockNumber": 22539624, "blockHash": "0xd418071906803d25afc3842a6a6468ad3b5fea27107b314ce4e2ccf08b478acf", "firstIndex": 18522044531}, +{"blockNumber": 22577021, "blockHash": "0xff222982693f3ff60d2097822171f80a6ddd979080aeb7e995bfb1b973497c84", "firstIndex": 18589154438}, +{"blockNumber": 22614525, "blockHash": "0x9868da1fea2ffca3f67e35570f02eb5707b27f6967ea4a109eb4ddbf24566efd", "firstIndex": 18656264174}, +{"blockNumber": 22652848, "blockHash": "0x060a911da11ab0f1dda307f5196e622d23901d198925749e70ab58a439477c5a", "firstIndex": 18723372617}, +{"blockNumber": 22692432, "blockHash": "0x6a937f2c283aba8c778c1f5ef340b225fd820f8a7dfa6f24f5fe541994f32f2d", "firstIndex": 18790480232}, +{"blockNumber": 22731200, "blockHash": "0x00d57a9e7a2dad252436fe9f0382c6a8860d301a9f9ffe6d7ac64c82b95300f8", "firstIndex": 18857590076}, +{"blockNumber": 22769000, "blockHash": "0xa48db20307c19c373ef2d31d85088ea14b8df0450491c31982504c87b04edbc0", "firstIndex": 18924699130}, +{"blockNumber": 22808126, "blockHash": "0x1419c64ff003edca0586f1c8ec3063da5c54c57ff826cfb34bc866cc18949653", "firstIndex": 18991807807}, +{"blockNumber": 22845231, "blockHash": "0x691f87217e61c5d7ae9ad53a44d30e1ab6b1cc3f2b689b9fbf7c38fbacacfe3e", "firstIndex": 19058917062}, +{"blockNumber": 22884189, "blockHash": "0x7f102d44c0ea7803f5b0e1a98a6abf0e8383eb99fb114d6f7b4591753ce8bba3", "firstIndex": 19126024122}, +{"blockNumber": 22920923, "blockHash": "0x04fe6179495016fc3fe56d8ef5311c360a5761a898262173849c3494fdd73d92", "firstIndex": 19193134595}, +{"blockNumber": 22958100, "blockHash": "0xe38e0ff7b0c4065ca42ea577bc32f2566ca46f2ddeedcc4bc1f8fb00e7f26329", "firstIndex": 19260242424}, +{"blockNumber": 22988600, "blockHash": "0x04ca74758b22e0ea54b8c992022ff21c16a2af9c45144c3b0f80de921a7eee82", "firstIndex": 19327351273}, +{"blockNumber": 23018392, "blockHash": "0x61cc979b00bc97b48356f986a5b9ec997d674bc904c2a2e4b0f17de08e50b3bb", "firstIndex": 19394459627}, +{"blockNumber": 23048524, "blockHash": "0x489de15d95739ede4ab15e8b5151d80d4dc85ae10e7be800b1a4723094a678df", "firstIndex": 19461570073} ] diff --git a/core/filtermaps/checkpoints_sepolia.json b/core/filtermaps/checkpoints_sepolia.json index 234af955e8..7214a4df1f 100644 --- a/core/filtermaps/checkpoints_sepolia.json +++ b/core/filtermaps/checkpoints_sepolia.json @@ -1,99 +1,99 @@ [ -{"blockNumber": 3246675, "blockId": "0x36bf7de9e1f151963088ca3efa206b6e78411d699d2f64f3bf86895294275e0b", "firstIndex": 67108765}, -{"blockNumber": 3575560, "blockId": "0x4652e3bee440f946aeaa3358c9a62b9bc4d41f663737ab00fded00c2662bfa29", "firstIndex": 134217644}, -{"blockNumber": 3694262, "blockId": "0x433573f97e563ca04fa53e0d0eb019d0ffe9dd032a05ee5b33ce7705aedfd34d", "firstIndex": 201316990}, -{"blockNumber": 3725628, "blockId": "0x1e25ebd76b9e2a2007bfdd7c82433cce7dae8beb190d249c82bf83d0cd177735", "firstIndex": 268426577}, -{"blockNumber": 3795378, "blockId": "0xca999083e01d5947212fb878d529abecdd18a6012997b0d5326afcf6908f17e5", "firstIndex": 335543438}, -{"blockNumber": 3856681, "blockId": "0x48148cd35de3de6c626f76ce5d97cfc6acd88772af69c0aef3d640c29ba82096", "firstIndex": 402641717}, -{"blockNumber": 3869367, "blockId": "0x08bafccf9ece7b72e4e069fc3c7493fbf25732881758917df2e8f9a51fc2c75f", "firstIndex": 469761225}, -{"blockNumber": 3938346, "blockId": "0xc638eb890c0e65ce1169ef0bb6b74dede2f4f6144ff8a5739c4894ff41a22fa2", "firstIndex": 536865235}, -{"blockNumber": 3984892, "blockId": "0x6a7782f0765b0f0c406b3cb107fd8bb14c6906293d9bdc5cc00e081bfd1ae59f", "firstIndex": 603972697}, -{"blockNumber": 4002660, "blockId": "0xda473de053602bc42fcbb074a32c908303f7f6db2ed8c38192665dce80276702", "firstIndex": 671087183}, -{"blockNumber": 4113149, "blockId": "0xd1782f677262226194725028858da83e672748e409fe1c21bdd77c07be0662c8", "firstIndex": 738197365}, -{"blockNumber": 4260733, "blockId": "0xc356600819e49d311963312157c481fd9be43a36ad040a0512fd17559c78d394", "firstIndex": 805305957}, -{"blockNumber": 4391073, "blockId": "0x06f903dc765b4b0ca1a40847eb74d30b0b51904a70c46f2e836dc5b3ea3dd8d3", "firstIndex": 872415067}, -{"blockNumber": 4515567, "blockId": "0xe99d3849755a1b4ec9a65d003260c111d4886e29ad6243fc750c72dbc79dbe4c", "firstIndex": 939523746}, -{"blockNumber": 4634815, "blockId": "0x9f281e749b192ef58242ff9fccd496a9075c30bd28c61ed4ce6487fc2d167e5c", "firstIndex": 1006629595}, -{"blockNumber": 4718286, "blockId": "0x23d2a40ec3ae059a45006ba651a70f308b6b282829b9cfdf2534e4311e34050f", "firstIndex": 1073731435}, -{"blockNumber": 4753427, "blockId": "0x6df7295f9ccb45a95e6cc72a8ee56bda992513b3bf4baa3d62219871a0b5a912", "firstIndex": 1140843105}, -{"blockNumber": 4786511, "blockId": "0x705c79435411c1af81385a1eca34343427eceb3f6af72e7f665552d6b0eceb6c", "firstIndex": 1207957185}, -{"blockNumber": 4811696, "blockId": "0x69a6e60d1b958b469feafb1dca97f38f97620a90c21ed92f90080f6443ee6c78", "firstIndex": 1275061401}, -{"blockNumber": 4841770, "blockId": "0x0a0e9d2d3d177601a478428297b5b9a42124166fefe42f2221c7a612c31e7ca1", "firstIndex": 1342174442}, -{"blockNumber": 4914785, "blockId": "0x5ca0e12230aeacbce9064dff15dafb9f24e20b4a737f7d04df8549008b829fa3", "firstIndex": 1409284680}, -{"blockNumber": 4992516, "blockId": "0xb9cb1700d8b65615fc3a019b2e9b2ece3b4f1e0e2fcfc01ff40bcf53e7291b10", "firstIndex": 1476387066}, -{"blockNumber": 5088617, "blockId": "0x33122f693a264fb2ab626232fac5e2714010fd65bdceb772109511f7c9497333", "firstIndex": 1543503849}, -{"blockNumber": 5155024, "blockId": "0x187716822dd6ab9c21138ae5a8e763197a5e7ec48bf0e756c5e6fa2c9f2f24bf", "firstIndex": 1610604247}, -{"blockNumber": 5204373, "blockId": "0x753b6495979ca90a1b915edad6af33c58bbabe68b45f95c7645113ec1d35d0e6", "firstIndex": 1677721200}, -{"blockNumber": 5269952, "blockId": "0xcbb0409ed824631120d6951b55537143d9a1c82ca6dd6255d2eb31c57844685c", "firstIndex": 1744829248}, -{"blockNumber": 5337623, "blockId": "0x0b460658511c4ae5205c038430789f9b31f3c08ba4acc06d273b7a13a149cf8b", "firstIndex": 1811939255}, -{"blockNumber": 5399044, "blockId": "0x9d07e37321e2b019f4837cc4f95f103da87ecdcb77e131e9d42d8643b0446524", "firstIndex": 1879041131}, -{"blockNumber": 5422692, "blockId": "0x9d9e27614635989788e32f15cd7a0abf7470dccf3336b77547baeb3423b629ca", "firstIndex": 1946154968}, -{"blockNumber": 5454259, "blockId": "0x01e694443d471d3411fe0b52720fd4a189e687b14cc8ede116f5e93f0ad96bb6", "firstIndex": 2013265148}, -{"blockNumber": 5498863, "blockId": "0xc4cb906c96d9f80b830b8add25318c38b3ff857c1bd577cd9cae0a324a068ffc", "firstIndex": 2080373677}, -{"blockNumber": 5554788, "blockId": "0x9b934b2d66723559d82e0f4cd49a2497234fe892248cb2f46c13f6a9585cfddd", "firstIndex": 2147475372}, -{"blockNumber": 5594711, "blockId": "0x840c0b7889b8c882cfa0eceb516dd73043b51c0e5a0c1be849a0e2a5dda7e842", "firstIndex": 2214592267}, -{"blockNumber": 5645179, "blockId": "0x188873a43506fee33349bc33df4a55059401e98dc7f0a5d488a0bc5c87dfd431", "firstIndex": 2281695869}, -{"blockNumber": 5687634, "blockId": "0x7ea8d7a2afccf795a2a02eb00746d89db57f13f8936cf7ab0f0a7db1b1ceb341", "firstIndex": 2348806824}, -{"blockNumber": 5727808, "blockId": "0x3f3eaf36c8e0271403737676907eac15312927c1d60ce814337654b7d7f2b2bd", "firstIndex": 2415915517}, -{"blockNumber": 5784480, "blockId": "0x10ec5adf01449c29550f5117f09ecafc49fc614d7da5886cf01619950827f850", "firstIndex": 2483023750}, -{"blockNumber": 5843906, "blockId": "0x9e6e14b3fba395a9ae7b3f1295f33a12d0f8b9f10e1aaab1649a90a56a323676", "firstIndex": 2550136398}, -{"blockNumber": 5906280, "blockId": "0x2188869a164e9c19232f6702a5233dc27a2304839159dd7230cc93ebdfc6c048", "firstIndex": 2617245012}, -{"blockNumber": 5977929, "blockId": "0xb247181ed597d9473c2c6e5762ae45a05f328d8df215fdac83d19ae4c028109a", "firstIndex": 2684347033}, -{"blockNumber": 6051479, "blockId": "0x0e8634c95dac31b7c835b44d0a14290e9212b31b9c3f621997ad1cca2d8ab0ba", "firstIndex": 2751463372}, -{"blockNumber": 6118422, "blockId": "0xa642c4457d4b63d68a5502bdba2355616e34495bf3ab2d8518a45ded54a16be4", "firstIndex": 2818563217}, -{"blockNumber": 6174274, "blockId": "0xbb9808bc56b5f14636e654bdd6d50d35b31345255926341ef3ddb54840095bcc", "firstIndex": 2885680274}, -{"blockNumber": 6276207, "blockId": "0x7ae027a18889e1b60a46ea3e5f2759035f3d9041de68eca0b154068ec533db59", "firstIndex": 2952789656}, -{"blockNumber": 6368344, "blockId": "0x8b4401f89589d9e99259e6c05c18b1780e96b65027caf9d83edf80390d317f66", "firstIndex": 3019898378}, -{"blockNumber": 6470718, "blockId": "0x0e2f8de093392c38750581c41e93c2fb1c0f6676b65c9af52792f3c56e894457", "firstIndex": 3087007589}, -{"blockNumber": 6553237, "blockId": "0xa6c4f3a1fe3b4e116c367fcbb1ae98e0f25570fdb2f1eadee2681deb39462c1e", "firstIndex": 3154116356}, -{"blockNumber": 6663750, "blockId": "0x9049748ddf655c7cc2ccbe2d36010b3a67ae011d114d32b1c812a7a53102f27f", "firstIndex": 3221225375}, -{"blockNumber": 6766894, "blockId": "0xc98188fb313b1beb7fc4f415b4a74c363903b3b5e0172437d79446bc011238b5", "firstIndex": 3288334172}, -{"blockNumber": 6886576, "blockId": "0x4ae8237a94198c82c266f01e472fdcd3ae150c84ecf714205a9be672f1e705b8", "firstIndex": 3355443040}, -{"blockNumber": 6978746, "blockId": "0xb2b69ce29c5c9c46c5cbc6e96260c34313d8b0b4a739cf4091c98f37093aa732", "firstIndex": 3422552017}, -{"blockNumber": 7098871, "blockId": "0x9ab2b681feeb38287a1b689fa64a50efd3407b6331fd6896187cad98a29ce107", "firstIndex": 3489660879}, -{"blockNumber": 7203023, "blockId": "0xdedfff5985c1c93d476120d2afbaa61967bae4edabb9711c7794a1b1cd453aa1", "firstIndex": 3556769365}, -{"blockNumber": 7256709, "blockId": "0xc078041cffc1802b6342fd2ba5a92ff8076ac0f5b72b4a77ba75cd486fe1b3c9", "firstIndex": 3623876563}, -{"blockNumber": 7307744, "blockId": "0xdf8dcdd4af9213a79587eb2f4d91f4111da74f99207ee05e4ae50bcb4f8d5435", "firstIndex": 3690986225}, -{"blockNumber": 7369268, "blockId": "0x62315823621a38a1f138e193b85259988fcc15f4a74d0cf19a412920bcdcad98", "firstIndex": 3758096030}, -{"blockNumber": 7445109, "blockId": "0x8b668ac1274a8dcc9526eb115c2f67bd56f6657b5072c7542da2252daca10fef", "firstIndex": 3825204921}, -{"blockNumber": 7511512, "blockId": "0x056ad7411652aa64849c67705a86e42221cb88f7fb6a7012ab7d3d5b9a30eb76", "firstIndex": 3892313524}, -{"blockNumber": 7557259, "blockId": "0xef660b97661f073b7c7319b69451e16d26ecaed54f9237e209999d137da2bfd4", "firstIndex": 3959415096}, -{"blockNumber": 7606260, "blockId": "0x9f5f06ad381166261780864bf17ad4d3cdd475f87ba51fa7f9ba621bf0ac0acb", "firstIndex": 4026521703}, -{"blockNumber": 7654244, "blockId": "0xefe5b537435001203741bf445ca26466baf7ad3f63c2bba94f0f658aed8c244b", "firstIndex": 4093627258}, -{"blockNumber": 7700433, "blockId": "0x89af6e8c5aa934a9139944b3de7a15232069479dba387e0fdd043ad36156c8bf", "firstIndex": 4160749200}, -{"blockNumber": 7730079, "blockId": "0x08e0d511a93e2a9c004b3456d77d687ae86247417ada1cdd1f85332a62dfed1d", "firstIndex": 4227846795}, -{"blockNumber": 7777515, "blockId": "0xeba8faed2e12bb9ed5e7fad19dd82662f15f6f4f512a0d232eedf1e676712428", "firstIndex": 4294966082}, -{"blockNumber": 7828860, "blockId": "0x62c054bd24be351dc4777460ee922a75fdcea5c9830455cbac6fa29552719c85", "firstIndex": 4362076087}, -{"blockNumber": 7869890, "blockId": "0x5664bcfa6151f798781e22bea4f9a2c796d097402350cb131e4e3c78e13e461a", "firstIndex": 4429183267}, -{"blockNumber": 7911722, "blockId": "0x9a85e48e3135c97c51fc1786f2af0596c802e021b6c53cfca65a129cafcd23ed", "firstIndex": 4496287265}, -{"blockNumber": 7960147, "blockId": "0xc9359cc76d7090e1c8a031108f0ab7a8935d971efd4325fe53612a1d99562f6f", "firstIndex": 4563402388}, -{"blockNumber": 8030418, "blockId": "0x21867e68cd8327aed2da2601399d60f7f9e41dca4a4f2f9be982e5a2b9304a88", "firstIndex": 4630511616}, -{"blockNumber": 8087701, "blockId": "0x0fa8c8d7549cc9a8d308262706fe248efe759f8b63511efb1e7f3926e9af2dcb", "firstIndex": 4697614758}, -{"blockNumber": 8149130, "blockId": "0x655ea638fd9e35cc25f4332f260d7bf98f4f6fa9a72e1bff861209f18659e94c", "firstIndex": 4764727744}, -{"blockNumber": 8208672, "blockId": "0xb5847a670dc3b6181f9e2e40e4218548048366d237a0d12e938b9879bc8cf800", "firstIndex": 4831837882}, -{"blockNumber": 8271345, "blockId": "0x96797214946f29093883b877ccb0f2a9f771a9a3db3794a642b5dcb781c4d194", "firstIndex": 4898942160}, -{"blockNumber": 8302858, "blockId": "0x6a5977b3382ca69a9e0412333f97b911c1f69f857d8f31dd0fc930980e24f2fc", "firstIndex": 4966054626}, -{"blockNumber": 8333618, "blockId": "0x2547294aa23b67c42adbdddfcf424b17a95c4ff0f352a6a2442c529cfb0c892a", "firstIndex": 5033163605}, -{"blockNumber": 8360582, "blockId": "0xf34f5dceb0ef22e0f782b56c12790472acc675997b9c45075bd4e18a9dacd03c", "firstIndex": 5100273631}, -{"blockNumber": 8387230, "blockId": "0x0fbea42e87620b5beeb76b67febc173847c54333d7dce9fa2f8f2a3fa9c8c22a", "firstIndex": 5167381673}, -{"blockNumber": 8414795, "blockId": "0x6c9c000cf5e35da3a7e9e1cd56147c8ce9b43a76d6de945675efd9dc03b628c9", "firstIndex": 5234477010}, -{"blockNumber": 8444749, "blockId": "0xba85f8c9abaddc34e2113eb49385667ba4b008168ae701f46aa7a7ce78c633a1", "firstIndex": 5301598562}, -{"blockNumber": 8474551, "blockId": "0x720866a40242f087dd25b6f0dd79224884f435b114a39e60c5669f5c942c78c1", "firstIndex": 5368707262}, -{"blockNumber": 8501310, "blockId": "0x2b6da233532c701202fb5ac67e005f7d3eb71f88a9fac10c25d24dd11ada05e5", "firstIndex": 5435803858}, -{"blockNumber": 8526970, "blockId": "0x005f9bbad0a10234129d09894d7fcf04bf1398d326510eedb4195808c282802d", "firstIndex": 5502926509}, -{"blockNumber": 8550412, "blockId": "0x37c9f3efc9f33cf62f590087c8c9ac70011883f75e648647a6fd0fec00ca627c", "firstIndex": 5570034950}, -{"blockNumber": 8573540, "blockId": "0x81cfb46a07be7c70bb8a0f76b03a4cd502f92032bea68ad7ba10e26351673000", "firstIndex": 5637137662}, -{"blockNumber": 8590416, "blockId": "0x5c223d58ef22d7b0dd8c498e8498da4787b5dc706681c2bc83849441f5d0922d", "firstIndex": 5704252906}, -{"blockNumber": 8616793, "blockId": "0x9043ce02742fb5ec43a696602867b7ce6003a95b36cd28a37eeb9785a46ad49f", "firstIndex": 5771357264}, -{"blockNumber": 8647290, "blockId": "0xd90115193764b0a33f3f2a719381b3ddbce2532607c72fb287a864eb391eeada", "firstIndex": 5838466144}, -{"blockNumber": 8673192, "blockId": "0x9bc92d340cccaf4c8c03372efc24eb92c5159106729de8d2e9e064f5568d082b", "firstIndex": 5905577457}, -{"blockNumber": 8700694, "blockId": "0xb3d656a173b962bc6825198e94a4974289db06a8998060bd0f5ee2044a7a7deb", "firstIndex": 5972679345}, -{"blockNumber": 8724533, "blockId": "0x253ffc6d77b88fe18736e4c313e9930341c444bc87b2ee22b26cfe8d9d0b178d", "firstIndex": 6039795829}, -{"blockNumber": 8743948, "blockId": "0x04eb66d0261705d31e629193148d0685058d7759ba5f95d2d38e412dbadb8256", "firstIndex": 6106901747}, -{"blockNumber": 8758378, "blockId": "0x64adf54e662d11db716610157da672c3d8b45f001dbce40a269871b86a84d026", "firstIndex": 6174011544}, -{"blockNumber": 8777722, "blockId": "0x0a7f9a956024b404c915e70b42221aa027b2dd715b0697f099dccefae0b9af97", "firstIndex": 6241124215}, -{"blockNumber": 8800154, "blockId": "0x411f90dc18f2bca31fa63615c2866c907bbac1fae8c06782cabfaf788efba665", "firstIndex": 6308233216}, -{"blockNumber": 8829725, "blockId": "0x5686f3a5eec1b070d0113c588f8f4a560d57ad96b8045cedb5c08bbadaa0273e", "firstIndex": 6375340033}, -{"blockNumber": 8858036, "blockId": "0x4f9b5d9fac9c6f6e2224f613cda12e8ab95d636774ce87489dce8a9f805ee2e5", "firstIndex": 6442450330}, -{"blockNumber": 8884811, "blockId": "0x9cf74f978872683802c065e72b5a5326fdad95f19733c34d927b575cd85fd0bd", "firstIndex": 6509559380} +{"blockNumber": 3246675, "blockHash": "0x36bf7de9e1f151963088ca3efa206b6e78411d699d2f64f3bf86895294275e0b", "firstIndex": 67108765}, +{"blockNumber": 3575560, "blockHash": "0x4652e3bee440f946aeaa3358c9a62b9bc4d41f663737ab00fded00c2662bfa29", "firstIndex": 134217644}, +{"blockNumber": 3694262, "blockHash": "0x433573f97e563ca04fa53e0d0eb019d0ffe9dd032a05ee5b33ce7705aedfd34d", "firstIndex": 201316990}, +{"blockNumber": 3725628, "blockHash": "0x1e25ebd76b9e2a2007bfdd7c82433cce7dae8beb190d249c82bf83d0cd177735", "firstIndex": 268426577}, +{"blockNumber": 3795378, "blockHash": "0xca999083e01d5947212fb878d529abecdd18a6012997b0d5326afcf6908f17e5", "firstIndex": 335543438}, +{"blockNumber": 3856681, "blockHash": "0x48148cd35de3de6c626f76ce5d97cfc6acd88772af69c0aef3d640c29ba82096", "firstIndex": 402641717}, +{"blockNumber": 3869367, "blockHash": "0x08bafccf9ece7b72e4e069fc3c7493fbf25732881758917df2e8f9a51fc2c75f", "firstIndex": 469761225}, +{"blockNumber": 3938346, "blockHash": "0xc638eb890c0e65ce1169ef0bb6b74dede2f4f6144ff8a5739c4894ff41a22fa2", "firstIndex": 536865235}, +{"blockNumber": 3984892, "blockHash": "0x6a7782f0765b0f0c406b3cb107fd8bb14c6906293d9bdc5cc00e081bfd1ae59f", "firstIndex": 603972697}, +{"blockNumber": 4002660, "blockHash": "0xda473de053602bc42fcbb074a32c908303f7f6db2ed8c38192665dce80276702", "firstIndex": 671087183}, +{"blockNumber": 4113149, "blockHash": "0xd1782f677262226194725028858da83e672748e409fe1c21bdd77c07be0662c8", "firstIndex": 738197365}, +{"blockNumber": 4260733, "blockHash": "0xc356600819e49d311963312157c481fd9be43a36ad040a0512fd17559c78d394", "firstIndex": 805305957}, +{"blockNumber": 4391073, "blockHash": "0x06f903dc765b4b0ca1a40847eb74d30b0b51904a70c46f2e836dc5b3ea3dd8d3", "firstIndex": 872415067}, +{"blockNumber": 4515567, "blockHash": "0xe99d3849755a1b4ec9a65d003260c111d4886e29ad6243fc750c72dbc79dbe4c", "firstIndex": 939523746}, +{"blockNumber": 4634815, "blockHash": "0x9f281e749b192ef58242ff9fccd496a9075c30bd28c61ed4ce6487fc2d167e5c", "firstIndex": 1006629595}, +{"blockNumber": 4718286, "blockHash": "0x23d2a40ec3ae059a45006ba651a70f308b6b282829b9cfdf2534e4311e34050f", "firstIndex": 1073731435}, +{"blockNumber": 4753427, "blockHash": "0x6df7295f9ccb45a95e6cc72a8ee56bda992513b3bf4baa3d62219871a0b5a912", "firstIndex": 1140843105}, +{"blockNumber": 4786511, "blockHash": "0x705c79435411c1af81385a1eca34343427eceb3f6af72e7f665552d6b0eceb6c", "firstIndex": 1207957185}, +{"blockNumber": 4811696, "blockHash": "0x69a6e60d1b958b469feafb1dca97f38f97620a90c21ed92f90080f6443ee6c78", "firstIndex": 1275061401}, +{"blockNumber": 4841770, "blockHash": "0x0a0e9d2d3d177601a478428297b5b9a42124166fefe42f2221c7a612c31e7ca1", "firstIndex": 1342174442}, +{"blockNumber": 4914785, "blockHash": "0x5ca0e12230aeacbce9064dff15dafb9f24e20b4a737f7d04df8549008b829fa3", "firstIndex": 1409284680}, +{"blockNumber": 4992516, "blockHash": "0xb9cb1700d8b65615fc3a019b2e9b2ece3b4f1e0e2fcfc01ff40bcf53e7291b10", "firstIndex": 1476387066}, +{"blockNumber": 5088617, "blockHash": "0x33122f693a264fb2ab626232fac5e2714010fd65bdceb772109511f7c9497333", "firstIndex": 1543503849}, +{"blockNumber": 5155024, "blockHash": "0x187716822dd6ab9c21138ae5a8e763197a5e7ec48bf0e756c5e6fa2c9f2f24bf", "firstIndex": 1610604247}, +{"blockNumber": 5204373, "blockHash": "0x753b6495979ca90a1b915edad6af33c58bbabe68b45f95c7645113ec1d35d0e6", "firstIndex": 1677721200}, +{"blockNumber": 5269952, "blockHash": "0xcbb0409ed824631120d6951b55537143d9a1c82ca6dd6255d2eb31c57844685c", "firstIndex": 1744829248}, +{"blockNumber": 5337623, "blockHash": "0x0b460658511c4ae5205c038430789f9b31f3c08ba4acc06d273b7a13a149cf8b", "firstIndex": 1811939255}, +{"blockNumber": 5399044, "blockHash": "0x9d07e37321e2b019f4837cc4f95f103da87ecdcb77e131e9d42d8643b0446524", "firstIndex": 1879041131}, +{"blockNumber": 5422692, "blockHash": "0x9d9e27614635989788e32f15cd7a0abf7470dccf3336b77547baeb3423b629ca", "firstIndex": 1946154968}, +{"blockNumber": 5454259, "blockHash": "0x01e694443d471d3411fe0b52720fd4a189e687b14cc8ede116f5e93f0ad96bb6", "firstIndex": 2013265148}, +{"blockNumber": 5498863, "blockHash": "0xc4cb906c96d9f80b830b8add25318c38b3ff857c1bd577cd9cae0a324a068ffc", "firstIndex": 2080373677}, +{"blockNumber": 5554788, "blockHash": "0x9b934b2d66723559d82e0f4cd49a2497234fe892248cb2f46c13f6a9585cfddd", "firstIndex": 2147475372}, +{"blockNumber": 5594711, "blockHash": "0x840c0b7889b8c882cfa0eceb516dd73043b51c0e5a0c1be849a0e2a5dda7e842", "firstIndex": 2214592267}, +{"blockNumber": 5645179, "blockHash": "0x188873a43506fee33349bc33df4a55059401e98dc7f0a5d488a0bc5c87dfd431", "firstIndex": 2281695869}, +{"blockNumber": 5687634, "blockHash": "0x7ea8d7a2afccf795a2a02eb00746d89db57f13f8936cf7ab0f0a7db1b1ceb341", "firstIndex": 2348806824}, +{"blockNumber": 5727808, "blockHash": "0x3f3eaf36c8e0271403737676907eac15312927c1d60ce814337654b7d7f2b2bd", "firstIndex": 2415915517}, +{"blockNumber": 5784480, "blockHash": "0x10ec5adf01449c29550f5117f09ecafc49fc614d7da5886cf01619950827f850", "firstIndex": 2483023750}, +{"blockNumber": 5843906, "blockHash": "0x9e6e14b3fba395a9ae7b3f1295f33a12d0f8b9f10e1aaab1649a90a56a323676", "firstIndex": 2550136398}, +{"blockNumber": 5906280, "blockHash": "0x2188869a164e9c19232f6702a5233dc27a2304839159dd7230cc93ebdfc6c048", "firstIndex": 2617245012}, +{"blockNumber": 5977929, "blockHash": "0xb247181ed597d9473c2c6e5762ae45a05f328d8df215fdac83d19ae4c028109a", "firstIndex": 2684347033}, +{"blockNumber": 6051479, "blockHash": "0x0e8634c95dac31b7c835b44d0a14290e9212b31b9c3f621997ad1cca2d8ab0ba", "firstIndex": 2751463372}, +{"blockNumber": 6118422, "blockHash": "0xa642c4457d4b63d68a5502bdba2355616e34495bf3ab2d8518a45ded54a16be4", "firstIndex": 2818563217}, +{"blockNumber": 6174274, "blockHash": "0xbb9808bc56b5f14636e654bdd6d50d35b31345255926341ef3ddb54840095bcc", "firstIndex": 2885680274}, +{"blockNumber": 6276207, "blockHash": "0x7ae027a18889e1b60a46ea3e5f2759035f3d9041de68eca0b154068ec533db59", "firstIndex": 2952789656}, +{"blockNumber": 6368344, "blockHash": "0x8b4401f89589d9e99259e6c05c18b1780e96b65027caf9d83edf80390d317f66", "firstIndex": 3019898378}, +{"blockNumber": 6470718, "blockHash": "0x0e2f8de093392c38750581c41e93c2fb1c0f6676b65c9af52792f3c56e894457", "firstIndex": 3087007589}, +{"blockNumber": 6553237, "blockHash": "0xa6c4f3a1fe3b4e116c367fcbb1ae98e0f25570fdb2f1eadee2681deb39462c1e", "firstIndex": 3154116356}, +{"blockNumber": 6663750, "blockHash": "0x9049748ddf655c7cc2ccbe2d36010b3a67ae011d114d32b1c812a7a53102f27f", "firstIndex": 3221225375}, +{"blockNumber": 6766894, "blockHash": "0xc98188fb313b1beb7fc4f415b4a74c363903b3b5e0172437d79446bc011238b5", "firstIndex": 3288334172}, +{"blockNumber": 6886576, "blockHash": "0x4ae8237a94198c82c266f01e472fdcd3ae150c84ecf714205a9be672f1e705b8", "firstIndex": 3355443040}, +{"blockNumber": 6978746, "blockHash": "0xb2b69ce29c5c9c46c5cbc6e96260c34313d8b0b4a739cf4091c98f37093aa732", "firstIndex": 3422552017}, +{"blockNumber": 7098871, "blockHash": "0x9ab2b681feeb38287a1b689fa64a50efd3407b6331fd6896187cad98a29ce107", "firstIndex": 3489660879}, +{"blockNumber": 7203023, "blockHash": "0xdedfff5985c1c93d476120d2afbaa61967bae4edabb9711c7794a1b1cd453aa1", "firstIndex": 3556769365}, +{"blockNumber": 7256709, "blockHash": "0xc078041cffc1802b6342fd2ba5a92ff8076ac0f5b72b4a77ba75cd486fe1b3c9", "firstIndex": 3623876563}, +{"blockNumber": 7307744, "blockHash": "0xdf8dcdd4af9213a79587eb2f4d91f4111da74f99207ee05e4ae50bcb4f8d5435", "firstIndex": 3690986225}, +{"blockNumber": 7369268, "blockHash": "0x62315823621a38a1f138e193b85259988fcc15f4a74d0cf19a412920bcdcad98", "firstIndex": 3758096030}, +{"blockNumber": 7445109, "blockHash": "0x8b668ac1274a8dcc9526eb115c2f67bd56f6657b5072c7542da2252daca10fef", "firstIndex": 3825204921}, +{"blockNumber": 7511512, "blockHash": "0x056ad7411652aa64849c67705a86e42221cb88f7fb6a7012ab7d3d5b9a30eb76", "firstIndex": 3892313524}, +{"blockNumber": 7557259, "blockHash": "0xef660b97661f073b7c7319b69451e16d26ecaed54f9237e209999d137da2bfd4", "firstIndex": 3959415096}, +{"blockNumber": 7606260, "blockHash": "0x9f5f06ad381166261780864bf17ad4d3cdd475f87ba51fa7f9ba621bf0ac0acb", "firstIndex": 4026521703}, +{"blockNumber": 7654244, "blockHash": "0xefe5b537435001203741bf445ca26466baf7ad3f63c2bba94f0f658aed8c244b", "firstIndex": 4093627258}, +{"blockNumber": 7700433, "blockHash": "0x89af6e8c5aa934a9139944b3de7a15232069479dba387e0fdd043ad36156c8bf", "firstIndex": 4160749200}, +{"blockNumber": 7730079, "blockHash": "0x08e0d511a93e2a9c004b3456d77d687ae86247417ada1cdd1f85332a62dfed1d", "firstIndex": 4227846795}, +{"blockNumber": 7777515, "blockHash": "0xeba8faed2e12bb9ed5e7fad19dd82662f15f6f4f512a0d232eedf1e676712428", "firstIndex": 4294966082}, +{"blockNumber": 7828860, "blockHash": "0x62c054bd24be351dc4777460ee922a75fdcea5c9830455cbac6fa29552719c85", "firstIndex": 4362076087}, +{"blockNumber": 7869890, "blockHash": "0x5664bcfa6151f798781e22bea4f9a2c796d097402350cb131e4e3c78e13e461a", "firstIndex": 4429183267}, +{"blockNumber": 7911722, "blockHash": "0x9a85e48e3135c97c51fc1786f2af0596c802e021b6c53cfca65a129cafcd23ed", "firstIndex": 4496287265}, +{"blockNumber": 7960147, "blockHash": "0xc9359cc76d7090e1c8a031108f0ab7a8935d971efd4325fe53612a1d99562f6f", "firstIndex": 4563402388}, +{"blockNumber": 8030418, "blockHash": "0x21867e68cd8327aed2da2601399d60f7f9e41dca4a4f2f9be982e5a2b9304a88", "firstIndex": 4630511616}, +{"blockNumber": 8087701, "blockHash": "0x0fa8c8d7549cc9a8d308262706fe248efe759f8b63511efb1e7f3926e9af2dcb", "firstIndex": 4697614758}, +{"blockNumber": 8149130, "blockHash": "0x655ea638fd9e35cc25f4332f260d7bf98f4f6fa9a72e1bff861209f18659e94c", "firstIndex": 4764727744}, +{"blockNumber": 8208672, "blockHash": "0xb5847a670dc3b6181f9e2e40e4218548048366d237a0d12e938b9879bc8cf800", "firstIndex": 4831837882}, +{"blockNumber": 8271345, "blockHash": "0x96797214946f29093883b877ccb0f2a9f771a9a3db3794a642b5dcb781c4d194", "firstIndex": 4898942160}, +{"blockNumber": 8302858, "blockHash": "0x6a5977b3382ca69a9e0412333f97b911c1f69f857d8f31dd0fc930980e24f2fc", "firstIndex": 4966054626}, +{"blockNumber": 8333618, "blockHash": "0x2547294aa23b67c42adbdddfcf424b17a95c4ff0f352a6a2442c529cfb0c892a", "firstIndex": 5033163605}, +{"blockNumber": 8360582, "blockHash": "0xf34f5dceb0ef22e0f782b56c12790472acc675997b9c45075bd4e18a9dacd03c", "firstIndex": 5100273631}, +{"blockNumber": 8387230, "blockHash": "0x0fbea42e87620b5beeb76b67febc173847c54333d7dce9fa2f8f2a3fa9c8c22a", "firstIndex": 5167381673}, +{"blockNumber": 8414795, "blockHash": "0x6c9c000cf5e35da3a7e9e1cd56147c8ce9b43a76d6de945675efd9dc03b628c9", "firstIndex": 5234477010}, +{"blockNumber": 8444749, "blockHash": "0xba85f8c9abaddc34e2113eb49385667ba4b008168ae701f46aa7a7ce78c633a1", "firstIndex": 5301598562}, +{"blockNumber": 8474551, "blockHash": "0x720866a40242f087dd25b6f0dd79224884f435b114a39e60c5669f5c942c78c1", "firstIndex": 5368707262}, +{"blockNumber": 8501310, "blockHash": "0x2b6da233532c701202fb5ac67e005f7d3eb71f88a9fac10c25d24dd11ada05e5", "firstIndex": 5435803858}, +{"blockNumber": 8526970, "blockHash": "0x005f9bbad0a10234129d09894d7fcf04bf1398d326510eedb4195808c282802d", "firstIndex": 5502926509}, +{"blockNumber": 8550412, "blockHash": "0x37c9f3efc9f33cf62f590087c8c9ac70011883f75e648647a6fd0fec00ca627c", "firstIndex": 5570034950}, +{"blockNumber": 8573540, "blockHash": "0x81cfb46a07be7c70bb8a0f76b03a4cd502f92032bea68ad7ba10e26351673000", "firstIndex": 5637137662}, +{"blockNumber": 8590416, "blockHash": "0x5c223d58ef22d7b0dd8c498e8498da4787b5dc706681c2bc83849441f5d0922d", "firstIndex": 5704252906}, +{"blockNumber": 8616793, "blockHash": "0x9043ce02742fb5ec43a696602867b7ce6003a95b36cd28a37eeb9785a46ad49f", "firstIndex": 5771357264}, +{"blockNumber": 8647290, "blockHash": "0xd90115193764b0a33f3f2a719381b3ddbce2532607c72fb287a864eb391eeada", "firstIndex": 5838466144}, +{"blockNumber": 8673192, "blockHash": "0x9bc92d340cccaf4c8c03372efc24eb92c5159106729de8d2e9e064f5568d082b", "firstIndex": 5905577457}, +{"blockNumber": 8700694, "blockHash": "0xb3d656a173b962bc6825198e94a4974289db06a8998060bd0f5ee2044a7a7deb", "firstIndex": 5972679345}, +{"blockNumber": 8724533, "blockHash": "0x253ffc6d77b88fe18736e4c313e9930341c444bc87b2ee22b26cfe8d9d0b178d", "firstIndex": 6039795829}, +{"blockNumber": 8743948, "blockHash": "0x04eb66d0261705d31e629193148d0685058d7759ba5f95d2d38e412dbadb8256", "firstIndex": 6106901747}, +{"blockNumber": 8758378, "blockHash": "0x64adf54e662d11db716610157da672c3d8b45f001dbce40a269871b86a84d026", "firstIndex": 6174011544}, +{"blockNumber": 8777722, "blockHash": "0x0a7f9a956024b404c915e70b42221aa027b2dd715b0697f099dccefae0b9af97", "firstIndex": 6241124215}, +{"blockNumber": 8800154, "blockHash": "0x411f90dc18f2bca31fa63615c2866c907bbac1fae8c06782cabfaf788efba665", "firstIndex": 6308233216}, +{"blockNumber": 8829725, "blockHash": "0x5686f3a5eec1b070d0113c588f8f4a560d57ad96b8045cedb5c08bbadaa0273e", "firstIndex": 6375340033}, +{"blockNumber": 8858036, "blockHash": "0x4f9b5d9fac9c6f6e2224f613cda12e8ab95d636774ce87489dce8a9f805ee2e5", "firstIndex": 6442450330}, +{"blockNumber": 8884811, "blockHash": "0x9cf74f978872683802c065e72b5a5326fdad95f19733c34d927b575cd85fd0bd", "firstIndex": 6509559380} ] diff --git a/core/filtermaps/filtermaps.go b/core/filtermaps/filtermaps.go deleted file mode 100644 index fede54df57..0000000000 --- a/core/filtermaps/filtermaps.go +++ /dev/null @@ -1,889 +0,0 @@ -// Copyright 2024 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package filtermaps - -import ( - "errors" - "fmt" - "os" - "slices" - "sync" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/lru" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/metrics" -) - -var ( - mapCountGauge = metrics.NewRegisteredGauge("filtermaps/maps/count", nil) // actual number of rendered maps - mapLogValueMeter = metrics.NewRegisteredMeter("filtermaps/maps/logvalues", nil) // number of log values processed - mapBlockMeter = metrics.NewRegisteredMeter("filtermaps/maps/blocks", nil) // number of block delimiters processed - mapRenderTimer = metrics.NewRegisteredTimer("filtermaps/maps/rendertime", nil) // time elapsed while rendering a single map - mapWriteTimer = metrics.NewRegisteredTimer("filtermaps/maps/writetime", nil) // time elapsed while writing a batch of finished maps to db - matchRequestTimer = metrics.NewRegisteredTimer("filtermaps/match/requesttime", nil) // processing time a matching request in a single epoch - matchEpochTimer = metrics.NewRegisteredTimer("filtermaps/match/epochtime", nil) // total processing time a matching request - matchBaseRowAccessMeter = metrics.NewRegisteredMeter("filtermaps/match/baserowaccess", nil) // number of accessed rows on layer 0 - matchBaseRowSizeMeter = metrics.NewRegisteredMeter("filtermaps/match/baserowsize", nil) // size of accessed rows on layer 0 - matchExtRowAccessMeter = metrics.NewRegisteredMeter("filtermaps/match/extrowaccess", nil) // number of accessed rows on higher layers - matchExtRowSizeMeter = metrics.NewRegisteredMeter("filtermaps/match/extrowsize", nil) // size of accessed rows on higher layers - matchLogLookup = metrics.NewRegisteredMeter("filtermaps/match/loglookup", nil) // number of log lookups based on potential matches - matchAllMeter = metrics.NewRegisteredMeter("filtermaps/match/matchall", nil) // number of requests returned with ErrMatchAll -) - -const ( - databaseVersion = 2 // reindexed if database version does not match - cachedLastBlocks = 1000 // last block of map pointers - cachedLvPointers = 1000 // first log value pointer of block pointers - cachedFilterMaps = 3 // complete filter maps (cached by map renderer) - cachedRenderSnapshots = 8 // saved map renderer data at block boundaries -) - -// FilterMaps is the in-memory representation of the log index structure that is -// responsible for building and updating the index according to the canonical -// chain. -// -// Note that FilterMaps implements the same data structure as proposed in EIP-7745 -// without the tree hashing and consensus changes: -// https://eips.ethereum.org/EIPS/eip-7745 -type FilterMaps struct { - // If disabled is set, log indexing is fully disabled. - // This is configured by the --history.logs.disable Geth flag. - // We chose to implement disabling this way because it requires less special - // case logic in eth/filters. - disabled bool - disabledCh chan struct{} // closed by indexer if disabled - - closeCh chan struct{} - closeWg sync.WaitGroup - history uint64 - hashScheme bool // use hashdb-safe delete range method - exportFileName string - Params - - db ethdb.KeyValueStore - - // fields written by the indexer and read by matcher backend. Indexer can - // read them without a lock and write them under indexLock write lock. - // Matcher backend can read them under indexLock read lock. - indexLock sync.RWMutex - indexedRange filterMapsRange - indexedView *ChainView // always consistent with the log index - hasTempRange bool - - // cleanedEpochsBefore indicates that all unindexed data before this point - // has been cleaned. - // - // This field is only accessed and modified within tryUnindexTail, so no - // explicit locking is required. - cleanedEpochsBefore uint32 - - // also accessed by indexer and matcher backend but no locking needed. - filterMapCache *lru.Cache[uint32, filterMap] - lastBlockCache *lru.Cache[uint32, lastBlockOfMap] - lvPointerCache *lru.Cache[uint64, uint64] - - // the matchers set and the fields of FilterMapsMatcherBackend instances are - // read and written both by exported functions and the indexer. - // Note that if both indexLock and matchersLock needs to be locked then - // indexLock should be locked first. - matchersLock sync.Mutex - matchers map[*FilterMapsMatcherBackend]struct{} - - // fields only accessed by the indexer (no mutex required). - renderSnapshots *lru.Cache[uint64, *renderedMap] - startedHeadIndex, startedTailIndex, startedTailUnindex bool - startedHeadIndexAt, startedTailIndexAt, startedTailUnindexAt time.Time - loggedHeadIndex, loggedTailIndex bool - lastLogHeadIndex, lastLogTailIndex time.Time - ptrHeadIndex, ptrTailIndex, ptrTailUnindexBlock uint64 - ptrTailUnindexMap uint32 - - targetView *ChainView - matcherSyncRequests []*FilterMapsMatcherBackend - historyCutoff uint64 - finalBlock, lastFinal uint64 - lastFinalEpoch uint32 - stop bool - targetCh chan targetUpdate - blockProcessingCh chan bool - blockProcessing bool - matcherSyncCh chan *FilterMapsMatcherBackend - waitIdleCh chan chan bool - tailRenderer *mapRenderer - - // test hooks - testDisableSnapshots, testSnapshotUsed bool - testProcessEventsHook func() -} - -// filterMap is a full or partial in-memory representation of a filter map where -// rows are allowed to have a nil value meaning the row is not stored in the -// structure. Note that therefore a known empty row should be represented with -// a zero-length slice. -// It can be used as a memory cache or an overlay while preparing a batch of -// changes to the structure. In either case a nil value should be interpreted -// as transparent (uncached/unchanged). -type filterMap []FilterRow - -// fastCopy returns a copy of the given filter map. Note that the row slices are -// copied but their contents are not. This permits appending to the rows further -// (which happens during map rendering) without affecting the validity of -// copies made for snapshots during rendering. -// Appending to the rows of both the original map and the fast copy, or two fast -// copies of the same map would result in data corruption, therefore a fast copy -// should always be used in a read only way. -func (fm filterMap) fastCopy() filterMap { - return slices.Clone(fm) -} - -// fullCopy returns a copy of the given filter map, also making a copy of each -// individual filter row, ensuring that a modification to either one will never -// affect the other. -func (fm filterMap) fullCopy() filterMap { - c := make(filterMap, len(fm)) - for i, row := range fm { - c[i] = slices.Clone(row) - } - return c -} - -// FilterRow encodes a single row of a filter map as a list of column indices. -// Note that the values are always stored in the same order as they were added -// and if the same column index is added twice, it is also stored twice. -// Order of column indices and potential duplications do not matter when searching -// for a value but leaving the original order makes reverting to a previous state -// simpler. -type FilterRow []uint32 - -// Equal returns true if the given filter rows are equivalent. -func (a FilterRow) Equal(b FilterRow) bool { - return slices.Equal(a, b) -} - -// filterMapsRange describes the rendered range of filter maps and the range -// of fully rendered blocks. -type filterMapsRange struct { - initialized bool - headIndexed bool - headDelimiter uint64 // zero if headIndexed is false - - // if initialized then all maps are rendered in the maps range - maps common.Range[uint32] - - // if tailPartialEpoch > 0 then maps between firstRenderedMap-mapsPerEpoch and - // firstRenderedMap-mapsPerEpoch+tailPartialEpoch-1 are rendered - tailPartialEpoch uint32 - - // if initialized then all log values in the blocks range are fully - // rendered - // blockLvPointers are available in the blocks range - blocks common.Range[uint64] -} - -// hasIndexedBlocks returns true if the range has at least one fully indexed block. -func (fmr *filterMapsRange) hasIndexedBlocks() bool { - return fmr.initialized && !fmr.blocks.IsEmpty() && !fmr.maps.IsEmpty() -} - -// lastBlockOfMap is used for caching the (number, id) pairs belonging to the -// last block of each map. -type lastBlockOfMap struct { - number uint64 - id common.Hash -} - -// Config contains the configuration options for NewFilterMaps. -type Config struct { - History uint64 // number of historical blocks to index - Disabled bool // disables indexing completely - - // This option enables the checkpoint JSON file generator. - // If set, the given file will be updated with checkpoint information. - ExportFileName string - - // expect trie nodes of hash based state scheme in the filtermaps key range; - // use safe iterator based implementation of DeleteRange that skips them - HashScheme bool -} - -// NewFilterMaps creates a new FilterMaps and starts the indexer. -func NewFilterMaps(db ethdb.KeyValueStore, initView *ChainView, historyCutoff, finalBlock uint64, params Params, config Config) (*FilterMaps, error) { - rs, initialized, err := rawdb.ReadFilterMapsRange(db) - if err != nil || (initialized && rs.Version != databaseVersion) { - rs, initialized = rawdb.FilterMapsRange{}, false - log.Warn("Invalid log index database version; resetting log index") - } - if err := params.sanitize(); err != nil { - return nil, err - } - f := &FilterMaps{ - db: db, - closeCh: make(chan struct{}), - waitIdleCh: make(chan chan bool), - targetCh: make(chan targetUpdate, 1), - blockProcessingCh: make(chan bool, 1), - history: config.History, - disabled: config.Disabled, - hashScheme: config.HashScheme, - disabledCh: make(chan struct{}), - exportFileName: config.ExportFileName, - Params: params, - targetView: initView, - indexedView: initView, - indexedRange: filterMapsRange{ - initialized: initialized, - headIndexed: rs.HeadIndexed, - headDelimiter: rs.HeadDelimiter, - blocks: common.NewRange(rs.BlocksFirst, rs.BlocksAfterLast-rs.BlocksFirst), - maps: common.NewRange(rs.MapsFirst, rs.MapsAfterLast-rs.MapsFirst), - tailPartialEpoch: rs.TailPartialEpoch, - }, - // deleting last unindexed epoch might have been interrupted by shutdown - cleanedEpochsBefore: max(rs.MapsFirst>>params.logMapsPerEpoch, 1) - 1, - historyCutoff: historyCutoff, - finalBlock: finalBlock, - matcherSyncCh: make(chan *FilterMapsMatcherBackend), - matchers: make(map[*FilterMapsMatcherBackend]struct{}), - filterMapCache: lru.NewCache[uint32, filterMap](cachedFilterMaps), - lastBlockCache: lru.NewCache[uint32, lastBlockOfMap](cachedLastBlocks), - lvPointerCache: lru.NewCache[uint64, uint64](cachedLvPointers), - renderSnapshots: lru.NewCache[uint64, *renderedMap](cachedRenderSnapshots), - } - f.checkRevertRange() // revert maps that are inconsistent with the current chain view - - if f.indexedRange.hasIndexedBlocks() { - log.Info("Initialized log indexer", - "firstblock", f.indexedRange.blocks.First(), "lastblock", f.indexedRange.blocks.Last(), - "firstmap", f.indexedRange.maps.First(), "lastmap", f.indexedRange.maps.Last(), - "headindexed", f.indexedRange.headIndexed) - } - return f, nil -} - -// Start starts the indexer. -func (f *FilterMaps) Start() { - if !f.testDisableSnapshots && f.indexedRange.hasIndexedBlocks() && f.indexedRange.headIndexed { - // previous target head rendered; load last map as snapshot - if err := f.loadHeadSnapshot(); err != nil { - log.Error("Could not load head filter map snapshot", "error", err) - } - } - f.closeWg.Add(2) - go f.removeBloomBits() - go f.indexerLoop() -} - -// Stop ensures that the indexer is fully stopped before returning. -func (f *FilterMaps) Stop() { - close(f.closeCh) - f.closeWg.Wait() -} - -// checkRevertRange checks whether the existing index is consistent with the -// current indexed view and reverts inconsistent maps if necessary. -func (f *FilterMaps) checkRevertRange() { - if f.indexedRange.maps.Count() == 0 { - return - } - lastMap := f.indexedRange.maps.Last() - lastBlockNumber, lastBlockId, err := f.getLastBlockOfMap(lastMap) - if err != nil { - log.Error("Error initializing log index database; resetting log index", "error", err) - f.reset() - return - } - for lastBlockNumber > f.indexedView.HeadNumber() || f.indexedView.BlockId(lastBlockNumber) != lastBlockId { - // revert last map - if f.indexedRange.maps.Count() == 1 { - f.reset() // reset database if no rendered maps remained - return - } - lastMap-- - newRange := f.indexedRange - newRange.maps.SetLast(lastMap) - lastBlockNumber, lastBlockId, err = f.getLastBlockOfMap(lastMap) - if err != nil { - log.Error("Error initializing log index database; resetting log index", "error", err) - f.reset() - return - } - newRange.blocks.SetAfterLast(lastBlockNumber) // lastBlockNumber is probably partially indexed - newRange.headIndexed = false - newRange.headDelimiter = 0 - // only shorten range and leave map data; next head render will overwrite it - f.setRange(f.db, f.indexedView, newRange, false) - } -} - -// reset un-initializes the FilterMaps structure and removes all related data from -// the database. -// Note that in case of leveldb database the fallback implementation of DeleteRange -// might take a long time to finish and deleting the entire database may be -// interrupted by a shutdown. Deleting the filterMapsRange entry first does -// guarantee though that the next init() will not return successfully until the -// entire database has been cleaned. -func (f *FilterMaps) reset() { - f.indexLock.Lock() - f.indexedRange = filterMapsRange{} - f.indexedView = nil - f.filterMapCache.Purge() - f.renderSnapshots.Purge() - f.lastBlockCache.Purge() - f.lvPointerCache.Purge() - f.indexLock.Unlock() - // deleting the range first ensures that resetDb will be called again at next - // startup and any leftover data will be removed even if it cannot finish now. - rawdb.DeleteFilterMapsRange(f.db) - f.safeDeleteWithLogs(rawdb.DeleteFilterMapsDb, "Resetting log index database", f.isShuttingDown) -} - -// isShuttingDown returns true if FilterMaps is shutting down. -func (f *FilterMaps) isShuttingDown() bool { - select { - case <-f.closeCh: - return true - default: - return false - } -} - -// init initializes an empty log index according to the current targetView. -func (f *FilterMaps) init() error { - // ensure that there is no remaining data in the filter maps key range - if err := f.safeDeleteWithLogs(rawdb.DeleteFilterMapsDb, "Resetting log index database", f.isShuttingDown); err != nil { - return err - } - - f.indexLock.Lock() - defer f.indexLock.Unlock() - - var bestIdx, bestLen int - for idx, checkpointList := range checkpoints { - // binary search for the last matching epoch head - min, max := 0, len(checkpointList) - for min < max { - mid := (min + max + 1) / 2 - cp := checkpointList[mid-1] - if cp.BlockNumber <= f.targetView.HeadNumber() && f.targetView.BlockId(cp.BlockNumber) == cp.BlockId { - min = mid - } else { - max = mid - 1 - } - } - if max > bestLen { - bestIdx, bestLen = idx, max - } - } - var initBlockNumber uint64 - if bestLen > 0 { - initBlockNumber = checkpoints[bestIdx][bestLen-1].BlockNumber - } - if initBlockNumber < f.historyCutoff { - return errors.New("cannot start indexing before history cutoff point") - } - batch := f.db.NewBatch() - for epoch := range bestLen { - cp := checkpoints[bestIdx][epoch] - f.storeLastBlockOfMap(batch, f.lastEpochMap(uint32(epoch)), cp.BlockNumber, cp.BlockId) - f.storeBlockLvPointer(batch, cp.BlockNumber, cp.FirstIndex) - } - fmr := filterMapsRange{ - initialized: true, - } - if bestLen > 0 { - cp := checkpoints[bestIdx][bestLen-1] - fmr.blocks = common.NewRange(cp.BlockNumber+1, 0) - fmr.maps = common.NewRange(f.firstEpochMap(uint32(bestLen)), 0) - } - f.setRange(batch, f.targetView, fmr, false) - return batch.Write() -} - -// removeBloomBits removes old bloom bits data from the database. -func (f *FilterMaps) removeBloomBits() { - f.safeDeleteWithLogs(rawdb.DeleteBloomBitsDb, "Removing old bloom bits database", f.isShuttingDown) - f.closeWg.Done() -} - -// safeDeleteWithLogs is a wrapper for a function that performs a safe range -// delete operation using rawdb.SafeDeleteRange. It emits log messages if the -// process takes long enough to call the stop callback. -func (f *FilterMaps) safeDeleteWithLogs(deleteFn func(db ethdb.KeyValueStore, hashScheme bool, stopCb func(bool) bool) error, action string, stopCb func() bool) error { - var ( - start = time.Now() - logPrinted bool - lastLogPrinted = start - ) - switch err := deleteFn(f.db, f.hashScheme, func(deleted bool) bool { - if deleted && !logPrinted || time.Since(lastLogPrinted) > time.Second*10 { - log.Info(action+" in progress...", "elapsed", common.PrettyDuration(time.Since(start))) - logPrinted, lastLogPrinted = true, time.Now() - } - return stopCb() - }); { - case err == nil: - if logPrinted { - log.Info(action+" finished", "elapsed", common.PrettyDuration(time.Since(start))) - } - return nil - case errors.Is(err, rawdb.ErrDeleteRangeInterrupted): - log.Warn(action+" interrupted", "elapsed", common.PrettyDuration(time.Since(start))) - return err - default: - log.Error(action+" failed", "error", err) - return err - } -} - -// setRange updates the indexed chain view and covered range and also adds the -// changes to the given batch. -// -// Note that this function assumes that the index write lock is being held. -func (f *FilterMaps) setRange(batch ethdb.KeyValueWriter, newView *ChainView, newRange filterMapsRange, isTempRange bool) { - f.indexedView = newView - f.indexedRange = newRange - f.hasTempRange = isTempRange - f.updateMatchersValidRange() - if newRange.initialized { - rs := rawdb.FilterMapsRange{ - Version: databaseVersion, - HeadIndexed: newRange.headIndexed, - HeadDelimiter: newRange.headDelimiter, - BlocksFirst: newRange.blocks.First(), - BlocksAfterLast: newRange.blocks.AfterLast(), - MapsFirst: newRange.maps.First(), - MapsAfterLast: newRange.maps.AfterLast(), - TailPartialEpoch: newRange.tailPartialEpoch, - } - rawdb.WriteFilterMapsRange(batch, rs) - if !isTempRange { - mapCountGauge.Update(int64(newRange.maps.Count() + newRange.tailPartialEpoch)) - } - } else { - rawdb.DeleteFilterMapsRange(batch) - mapCountGauge.Update(0) - } -} - -// getLogByLvIndex returns the log at the given log value index. If the index does -// not point to the first log value entry of a log then no log and no error are -// returned as this can happen when the log value index was a false positive. -// Note that this function assumes that the log index structure is consistent -// with the canonical chain at the point where the given log value index points. -// If this is not the case then an invalid result or an error may be returned. -// -// Note that this function assumes that the indexer read lock is being held when -// called from outside the indexerLoop goroutine. -func (f *FilterMaps) getLogByLvIndex(lvIndex uint64) (*types.Log, error) { - mapIndex := uint32(lvIndex >> f.logValuesPerMap) - if !f.indexedRange.maps.Includes(mapIndex) { - return nil, nil - } - // find possible block range based on map to block pointers - lastBlockNumber, _, err := f.getLastBlockOfMap(mapIndex) - if err != nil { - return nil, fmt.Errorf("failed to retrieve last block of map %d containing searched log value index %d: %v", mapIndex, lvIndex, err) - } - var firstBlockNumber uint64 - if mapIndex > 0 { - firstBlockNumber, _, err = f.getLastBlockOfMap(mapIndex - 1) - if err != nil { - return nil, fmt.Errorf("failed to retrieve last block of map %d before searched log value index %d: %v", mapIndex, lvIndex, err) - } - } - if firstBlockNumber < f.indexedRange.blocks.First() { - firstBlockNumber = f.indexedRange.blocks.First() - } - // find block with binary search based on block to log value index pointers - for firstBlockNumber < lastBlockNumber { - midBlockNumber := (firstBlockNumber + lastBlockNumber + 1) / 2 - midLvPointer, err := f.getBlockLvPointer(midBlockNumber) - if err != nil { - return nil, fmt.Errorf("failed to retrieve log value pointer of block %d while binary searching log value index %d: %v", midBlockNumber, lvIndex, err) - } - if lvIndex < midLvPointer { - lastBlockNumber = midBlockNumber - 1 - } else { - firstBlockNumber = midBlockNumber - } - } - // get block receipts - receipts := f.indexedView.Receipts(firstBlockNumber) - if receipts == nil { - return nil, fmt.Errorf("failed to retrieve receipts for block %d containing searched log value index %d: %v", firstBlockNumber, lvIndex, err) - } - lvPointer, err := f.getBlockLvPointer(firstBlockNumber) - if err != nil { - return nil, fmt.Errorf("failed to retrieve log value pointer of block %d containing searched log value index %d: %v", firstBlockNumber, lvIndex, err) - } - // iterate through receipts to find the exact log starting at lvIndex - for _, receipt := range receipts { - for _, log := range receipt.Logs { - l := uint64(len(log.Topics) + 1) - r := f.valuesPerMap - lvPointer%f.valuesPerMap - if l > r { - lvPointer += r // skip to map boundary - } - if lvPointer > lvIndex { - // lvIndex does not point to the first log value (address value) - // generated by a log as true matches should always do, so it - // is considered a false positive (no log and no error returned). - return nil, nil - } - if lvPointer == lvIndex { - return log, nil // potential match - } - lvPointer += l - } - } - return nil, nil -} - -// getFilterMap fetches an entire filter map from the database. -func (f *FilterMaps) getFilterMap(mapIndex uint32) (filterMap, error) { - if fm, ok := f.filterMapCache.Get(mapIndex); ok { - return fm, nil - } - fm := make(filterMap, f.mapHeight) - for rowIndex := range fm { - rows, err := f.getFilterMapRows([]uint32{mapIndex}, uint32(rowIndex), false) - if err != nil { - return nil, fmt.Errorf("failed to load filter map %d from database: %v", mapIndex, err) - } - fm[rowIndex] = rows[0] - } - f.filterMapCache.Add(mapIndex, fm) - return fm, nil -} - -// getFilterMapRows fetches a set of filter map rows at the corresponding map -// indices and a shared row index. If baseLayerOnly is true then only the first -// baseRowLength entries are returned. -func (f *FilterMaps) getFilterMapRows(mapIndices []uint32, rowIndex uint32, baseLayerOnly bool) ([]FilterRow, error) { - rows := make([]FilterRow, len(mapIndices)) - var ptr int - for len(mapIndices) > ptr { - var ( - groupIndex = f.mapGroupIndex(mapIndices[ptr]) - groupLength = 1 - ) - for ptr+groupLength < len(mapIndices) && f.mapGroupIndex(mapIndices[ptr+groupLength]) == groupIndex { - groupLength++ - } - if err := f.getFilterMapRowsOfGroup(rows[ptr:ptr+groupLength], mapIndices[ptr:ptr+groupLength], rowIndex, baseLayerOnly); err != nil { - return nil, err - } - ptr += groupLength - } - return rows, nil -} - -// getFilterMapRowsOfGroup fetches a set of filter map rows at map indices -// belonging to the same base row group. -func (f *FilterMaps) getFilterMapRowsOfGroup(target []FilterRow, mapIndices []uint32, rowIndex uint32, baseLayerOnly bool) error { - var ( - groupIndex = f.mapGroupIndex(mapIndices[0]) - mapRowIndex = f.mapRowIndex(groupIndex, rowIndex) - ) - baseRows, err := rawdb.ReadFilterMapBaseRows(f.db, mapRowIndex, f.baseRowGroupSize, f.logMapWidth) - if err != nil { - return fmt.Errorf("failed to retrieve base row group %d of row %d: %v", groupIndex, rowIndex, err) - } - for i, mapIndex := range mapIndices { - if f.mapGroupIndex(mapIndex) != groupIndex { - return fmt.Errorf("maps are not in the same base row group, index: %d, group: %d", mapIndex, groupIndex) - } - row := baseRows[f.mapGroupOffset(mapIndex)] - if !baseLayerOnly { - extRow, err := rawdb.ReadFilterMapExtRow(f.db, f.mapRowIndex(mapIndex, rowIndex), f.logMapWidth) - if err != nil { - return fmt.Errorf("failed to retrieve filter map %d extended row %d: %v", mapIndex, rowIndex, err) - } - row = append(row, extRow...) - } - target[i] = row - } - return nil -} - -// storeFilterMapRows stores a set of filter map rows at the corresponding map -// indices and a shared row index. -func (f *FilterMaps) storeFilterMapRows(batch ethdb.Batch, mapIndices []uint32, rowIndex uint32, rows []FilterRow) error { - for len(mapIndices) > 0 { - var ( - pos = 1 - groupIndex = f.mapGroupIndex(mapIndices[0]) - ) - for pos < len(mapIndices) && f.mapGroupIndex(mapIndices[pos]) == groupIndex { - pos++ - } - if err := f.storeFilterMapRowsOfGroup(batch, mapIndices[:pos], rowIndex, rows[:pos]); err != nil { - return err - } - mapIndices, rows = mapIndices[pos:], rows[pos:] - } - return nil -} - -// storeFilterMapRowsOfGroup stores a set of filter map rows at map indices -// belonging to the same base row group. -func (f *FilterMaps) storeFilterMapRowsOfGroup(batch ethdb.Batch, mapIndices []uint32, rowIndex uint32, rows []FilterRow) error { - var ( - baseRows [][]uint32 - groupIndex = f.mapGroupIndex(mapIndices[0]) - mapRowIndex = f.mapRowIndex(groupIndex, rowIndex) - ) - if uint32(len(mapIndices)) != f.baseRowGroupSize { // skip base rows read if all rows are replaced - var err error - baseRows, err = rawdb.ReadFilterMapBaseRows(f.db, mapRowIndex, f.baseRowGroupSize, f.logMapWidth) - if err != nil { - return fmt.Errorf("failed to retrieve filter map %d base rows %d for modification: %v", groupIndex, rowIndex, err) - } - } else { - baseRows = make([][]uint32, f.baseRowGroupSize) - } - for i, mapIndex := range mapIndices { - if f.mapGroupIndex(mapIndex) != groupIndex { - return fmt.Errorf("maps are not in the same base row group, index: %d, group: %d", mapIndex, groupIndex) - } - baseRow := []uint32(rows[i]) - var extRow FilterRow - if uint32(len(rows[i])) > f.baseRowLength { - extRow = baseRow[f.baseRowLength:] - baseRow = baseRow[:f.baseRowLength] - } - baseRows[f.mapGroupOffset(mapIndex)] = baseRow - rawdb.WriteFilterMapExtRow(batch, f.mapRowIndex(mapIndex, rowIndex), extRow, f.logMapWidth) - } - rawdb.WriteFilterMapBaseRows(batch, mapRowIndex, baseRows, f.logMapWidth) - return nil -} - -// mapRowIndex calculates the unified storage index where the given row of the -// given map is stored. Note that this indexing scheme is the same as the one -// proposed in EIP-7745 for tree-hashing the filter map structure and for the -// same data proximity reasons it is also suitable for database representation. -// See also: -// https://eips.ethereum.org/EIPS/eip-7745#hash-tree-structure -func (f *FilterMaps) mapRowIndex(mapIndex, rowIndex uint32) uint64 { - epochIndex, mapSubIndex := mapIndex>>f.logMapsPerEpoch, mapIndex&(f.mapsPerEpoch-1) - return (uint64(epochIndex)< 0 { - firstBlock, _, err = f.getLastBlockOfMap(firstMap - 1) - if err != nil { - return false, fmt.Errorf("failed to retrieve last block before deleted epoch %d: %v", epoch, err) - } - firstBlock++ - } - // update rendered range if necessary - var ( - fmr = f.indexedRange - firstEpoch = f.mapEpoch(f.indexedRange.maps.First()) - afterLastEpoch = f.mapEpoch(f.indexedRange.maps.AfterLast() + f.mapsPerEpoch - 1) - ) - if f.indexedRange.tailPartialEpoch != 0 && firstEpoch > 0 { - firstEpoch-- - } - switch { - case epoch < firstEpoch: - // cleanup of already unindexed epoch; range not affected - case epoch == firstEpoch && epoch+1 < afterLastEpoch: - // first fully or partially rendered epoch and there is at least one - // rendered map in the next epoch; remove from indexed range - fmr.tailPartialEpoch = 0 - fmr.maps.SetFirst(f.firstEpochMap(epoch + 1)) - fmr.blocks.SetFirst(lastBlock + 1) - f.setRange(f.db, f.indexedView, fmr, false) - default: - // cannot be cleaned or unindexed; return with error - return false, errors.New("invalid tail epoch number") - } - // remove index data - deleteFn := func(db ethdb.KeyValueStore, hashScheme bool, stopCb func(bool) bool) error { - first := f.mapRowIndex(firstMap, 0) - count := f.mapRowIndex(firstMap+f.mapsPerEpoch, 0) - first - if err := rawdb.DeleteFilterMapRows(f.db, common.NewRange(first, count), hashScheme, stopCb); err != nil { - return err - } - for mapIndex := firstMap; mapIndex < firstMap+f.mapsPerEpoch; mapIndex++ { - f.filterMapCache.Remove(mapIndex) - } - delMapRange := common.NewRange(firstMap, f.mapsPerEpoch-1) // keep last entry - if err := rawdb.DeleteFilterMapLastBlocks(f.db, delMapRange, hashScheme, stopCb); err != nil { - return err - } - for mapIndex := firstMap; mapIndex < firstMap+f.mapsPerEpoch-1; mapIndex++ { - f.lastBlockCache.Remove(mapIndex) - } - delBlockRange := common.NewRange(firstBlock, lastBlock-firstBlock) // keep last entry - if err := rawdb.DeleteBlockLvPointers(f.db, delBlockRange, hashScheme, stopCb); err != nil { - return err - } - for blockNumber := firstBlock; blockNumber < lastBlock; blockNumber++ { - f.lvPointerCache.Remove(blockNumber) - } - return nil - } - action := fmt.Sprintf("Deleting tail epoch #%d", epoch) - stopFn := func() bool { - f.processEvents() - return f.stop || !f.targetHeadIndexed() - } - if err := f.safeDeleteWithLogs(deleteFn, action, stopFn); err == nil { - // everything removed; mark as cleaned and report success - if f.cleanedEpochsBefore == epoch { - f.cleanedEpochsBefore = epoch + 1 - } - return true, nil - } else { - // more data left in epoch range; mark as dirty and report unfinished - if f.cleanedEpochsBefore > epoch { - f.cleanedEpochsBefore = epoch - } - if errors.Is(err, rawdb.ErrDeleteRangeInterrupted) { - return false, nil - } - return false, err - } -} - -// exportCheckpoints exports epoch checkpoints in the format used by checkpoints.go. -// -// Note: acquiring the indexLock read lock is unnecessary here, as this function -// is always called within the indexLoop. -func (f *FilterMaps) exportCheckpoints() { - finalLvPtr, err := f.getBlockLvPointer(f.finalBlock + 1) - if err != nil { - log.Error("Error fetching log value pointer of finalized block", "block", f.finalBlock, "error", err) - return - } - epochCount := uint32(finalLvPtr >> (f.logValuesPerMap + f.logMapsPerEpoch)) - if epochCount == f.lastFinalEpoch { - return - } - w, err := os.Create(f.exportFileName) - if err != nil { - log.Error("Error creating checkpoint export file", "name", f.exportFileName, "error", err) - return - } - defer w.Close() - - log.Info("Exporting log index checkpoints", "epochs", epochCount, "file", f.exportFileName) - w.WriteString("[\n") - comma := "," - for epoch := uint32(0); epoch < epochCount; epoch++ { - lastBlock, lastBlockId, err := f.getLastBlockOfMap(f.lastEpochMap(epoch)) - if err != nil { - log.Error("Error fetching last block of epoch", "epoch", epoch, "error", err) - return - } - lvPtr, err := f.getBlockLvPointer(lastBlock) - if err != nil { - log.Error("Error fetching log value pointer of last block", "block", lastBlock, "error", err) - return - } - if epoch == epochCount-1 { - comma = "" - } - w.WriteString(fmt.Sprintf("{\"blockNumber\": %d, \"blockId\": \"0x%064x\", \"firstIndex\": %d}%s\n", lastBlock, lastBlockId, lvPtr, comma)) - } - w.WriteString("]\n") - f.lastFinalEpoch = epochCount -} diff --git a/core/filtermaps/index_view.go b/core/filtermaps/index_view.go new file mode 100644 index 0000000000..92999028e5 --- /dev/null +++ b/core/filtermaps/index_view.go @@ -0,0 +1,274 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package filtermaps + +import ( + "errors" + "sync/atomic" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" +) + +var ( + ErrInvalidView = errors.New("index view already invalidated") +) + +type IndexView struct { + // if invalid <= 0 then the view is considered invalid and will return error + // on any read operation (storage maps before firstMemoryMap might have changed + // since its creation) + refCount, invalid int32 + storage *mapStorage + + tailEpoch uint32 //TODO apply to read functions + blockRange common.Range[uint64] //TODO apply to read functions + headBlockHash common.Hash + headLvPointer uint64 // points after head block delimiter + + firstMemoryMap uint32 + firstMemoryBlock uint64 + finishedMaps []*finishedMap + headMapIndex uint32 + headMap *memoryMap +} + +func (iv *IndexView) Release() { + iv.addRefCount(-1) +} + +func (iv *IndexView) addRefCount(add int32) bool { + return atomic.AddInt32(&iv.refCount, add) <= 0 +} + +func (iv *IndexView) checkReleased() bool { + return atomic.LoadInt32(&iv.refCount) <= 0 +} + +func (iv *IndexView) invalidate() { + atomic.StoreInt32(&iv.invalid, 1) +} + +func (iv *IndexView) checkInvalid() bool { + return atomic.LoadInt32(&iv.invalid) != 0 +} + +func (iv *IndexView) GetParams() *Params { + return iv.storage.params +} + +func (iv *IndexView) BlockRange() common.Range[uint64] { + return iv.blockRange +} + +// getBlockLvPointer returns the starting log value index where the log values +// generated by the given block are located. +// Note that the function returns a valid result (the next free log value index) +// for headNumber + 1. +func (iv *IndexView) GetBlockLvPointer(blockNumber uint64) (uint64, error) { + if iv.checkInvalid() { + return 0, ErrInvalidView + } + + if blockNumber < iv.firstMemoryBlock { + lvPtr, err := iv.storage.getBlockLvPointer(blockNumber) + if iv.checkInvalid() { + return 0, ErrInvalidView + } + return lvPtr, err + } + if blockNumber == iv.blockRange.AfterLast() { + return iv.headLvPointer, nil + } + if blockNumber > iv.blockRange.AfterLast() { + return 0, ErrOutOfRange + } + for _, fm := range iv.finishedMaps { + if blockNumber >= fm.firstBlock() && blockNumber <= fm.lastBlock.number { + return fm.blockPtrs[blockNumber-fm.firstBlock()], nil + } + } + if blockNumber >= iv.headMap.firstBlock() && blockNumber <= iv.headMap.lastBlock.number { + return iv.headMap.blockPtrs[blockNumber-iv.headMap.firstBlock()], nil + } + panic("IndexView.GetBlockLvPointer: gap in blockLvPtrs") +} + +// GetLastBlockOfMap returns the number and hash of the block that generated the +// last log value entry of the given map. +func (iv *IndexView) GetLastBlockOfMap(mapIndex uint32) (uint64, common.Hash, error) { + if iv.checkInvalid() { + return 0, common.Hash{}, ErrInvalidView + } + + if mapIndex < iv.firstMemoryMap { + lastNumber, lastHash, err := iv.storage.getLastBlockOfMap(mapIndex) + if iv.checkInvalid() { + return 0, common.Hash{}, ErrInvalidView + } + return lastNumber, lastHash, err + } + if mapIndex > iv.headMapIndex { + return 0, common.Hash{}, ErrOutOfRange + } + if mapIndex == iv.headMapIndex { + return iv.headMap.lastBlock.number, iv.headMap.lastBlock.hash, nil + } + fm := iv.finishedMaps[mapIndex-iv.firstMemoryMap] + return fm.lastBlock.number, fm.lastBlock.hash, nil +} + +// GetFilterMapRows returns a batch of filter maps rows from the same row index, +// each truncated to the length limit of the specified mapping layer. +// The function assumes that the map indices are in strictly ascending order. +func (iv *IndexView) GetFilterMapRows(mapIndices []uint32, rowIndex, layerIndex uint32) (rows []FilterRow, err error) { + if iv.checkInvalid() { + return nil, ErrInvalidView + } + + dbIndices := len(mapIndices) + for dbIndices > 0 && mapIndices[dbIndices-1] >= iv.firstMemoryMap { + dbIndices-- + } + if dbIndices > 0 { + rows, err = iv.storage.getFilterMapRows(mapIndices[:dbIndices], rowIndex, layerIndex) + if iv.checkInvalid() { + return nil, ErrInvalidView + } + if err != nil { + return nil, err + } + } + for i := dbIndices; i < len(mapIndices); i++ { + mapIndex := mapIndices[i] + var row FilterRow + if mapIndex == iv.headMapIndex { + row = iv.headMap.getRow(rowIndex, iv.storage.params.getMaxRowLength(layerIndex+1)) + } + if mapIndex < iv.headMapIndex { + row = iv.finishedMaps[mapIndex-iv.firstMemoryMap].getRow(rowIndex, iv.storage.params.getMaxRowLength(layerIndex+1)) + } + rows = append(rows, row) + } + return rows, nil +} + +type renderState struct { + params *Params + renderRange common.Range[uint32] + lvPointer uint64 + mapIndex uint32 + currentMap *memoryMap + finishedMaps []*finishedMap + nextBlock uint64 + partialBlock bool + partialBlockHash common.Hash +} + +func (rs *renderState) checkNextHash(hash common.Hash) bool { + if rs.partialBlock && rs.partialBlockHash != hash { + return false + } + rs.partialBlock = false + return true +} + +func (rs *renderState) addReceipts(receipts types.Receipts) { + if rs.partialBlock { + panic("checkNextHash has to be called before adding partially rendered block") + } + if rs.currentMap != nil { + rs.currentMap.blockPtrs = append(rs.currentMap.blockPtrs, rs.lvPointer) + rs.currentMap.lastBlock = lastBlockOfMap{number: rs.nextBlock} // hash will be set by addHeader + } + for _, receipt := range receipts { + //TODO add tx delimiter + for _, log := range receipt.Logs { + mapRemaining := rs.params.valuesPerMap - rs.lvPointer%rs.params.valuesPerMap + if mapRemaining <= uint64(len(log.Topics)) { + rs.advance(mapRemaining) + } + if rs.currentMap != nil { + rs.addValue(addressValue(log.Address)) + for _, topic := range log.Topics { + rs.addValue(topicValue(topic)) + } + } else { + rs.advance(uint64(len(log.Topics) + 1)) + } + if rs.finished() { + return + } + } + } +} + +func (rs *renderState) addHeader(header *types.Header) (uint32, []*finishedMap) { + if rs.partialBlock { + panic("checkNextHash has to be called before adding partially rendered block") + } + if rs.nextBlock != header.Number.Uint64() { + panic("wrong block number") + } + if !rs.finished() { + rs.advance(1) //TODO blockValue + } + rs.nextBlock++ + lastBlock := lastBlockOfMap{number: header.Number.Uint64(), hash: header.Hash()} + if rs.currentMap != nil { + rs.currentMap.lastBlock = lastBlock + } + for _, fm := range rs.finishedMaps { + fm.lastBlock = lastBlock + } + firstMemoryMap, finishedMaps := rs.mapIndex-uint32(len(rs.finishedMaps)), rs.finishedMaps + rs.finishedMaps = nil + return firstMemoryMap, finishedMaps +} + +// assumes currentMap != nil +func (rs *renderState) addValue(logValue common.Hash) { + if rs.renderRange.Includes(rs.mapIndex) { + for layerIndex := uint32(0); ; layerIndex++ { + rowIndex := rs.params.rowIndex(rs.mapIndex, layerIndex, logValue) + if rs.currentMap.rowLength(rowIndex) < rs.params.getMaxRowLength(layerIndex) { + rs.currentMap.addToRow(rowIndex, rs.params.columnIndex(rs.lvPointer, &logValue)) + break + } + } + } + rs.advance(1) +} + +func (rs *renderState) advance(count uint64) { + rs.lvPointer += count + if uint32(rs.lvPointer>>rs.params.logValuesPerMap) > rs.mapIndex { + if rs.currentMap != nil { + rs.finishedMaps = append(rs.finishedMaps, rs.currentMap.finished()) + } + rs.mapIndex++ + if rs.renderRange.Includes(rs.mapIndex) { + rs.currentMap = rs.params.newMemoryMap() + } else { + rs.currentMap = nil + } + } +} + +func (rs *renderState) finished() bool { + return rs.mapIndex >= rs.renderRange.AfterLast() +} diff --git a/core/filtermaps/indexer.go b/core/filtermaps/indexer.go index ca50fb466c..675ce3a6b5 100644 --- a/core/filtermaps/indexer.go +++ b/core/filtermaps/indexer.go @@ -1,4 +1,4 @@ -// Copyright 2024 The go-ethereum Authors +// Copyright 2025 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify @@ -17,442 +17,645 @@ package filtermaps import ( - "errors" + "fmt" "math" - "time" + "os" + "sync" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/lru" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" ) const ( - logFrequency = time.Second * 20 // log info frequency during long indexing/unindexing process - headLogDelay = time.Second // head indexing log info delay (do not log if finished faster) + maxCanonicalSnapshots = 4 + maxRecentSnapshots = 4 + maxIndexViewMaps = 2 ) -// indexerLoop initializes and updates the log index structure according to the -// current targetView. -func (f *FilterMaps) indexerLoop() { - defer f.closeWg.Done() +var ( + mapCountGauge = metrics.NewRegisteredGauge("filtermaps/maps/count", nil) // actual number of rendered maps + mapLogValueMeter = metrics.NewRegisteredMeter("filtermaps/maps/logvalues", nil) // number of log values processed + mapBlockMeter = metrics.NewRegisteredMeter("filtermaps/maps/blocks", nil) // number of block delimiters processed + mapRenderTimer = metrics.NewRegisteredTimer("filtermaps/maps/rendertime", nil) // time elapsed while rendering a single map + mapWriteTimer = metrics.NewRegisteredTimer("filtermaps/maps/writetime", nil) // time elapsed while writing a batch of finished maps to db +) - if f.disabled { - f.reset() - close(f.disabledCh) - return +// Indexer maintains a search data structure based on a parent blockchain that +// is intended to make log event search more efficient. Once indexed up to the +// chain head, it provides IndexView objects for recent chain heads. +// Indexer implements core.Indexer. +type Indexer struct { + config Config + storage *mapStorage + checkpoints []checkpointList + headRenderer, tailRenderer *renderState + historyCutoff, finalized uint64 + tailRenderLast, headNumber uint64 + lastCanonical uint64 + tailEpoch, targetTailEpoch, activeViewTailEpoch uint32 + canonicalHashes []common.Hash // last one belongs to lastCanonical + recentHashes []common.Hash // last one is the most recently saved + snapshotsLock sync.RWMutex + snapshots map[common.Hash]*IndexView + headMapsCache *lru.Cache[uint32, *finishedMap] + lastFinalEpoch uint32 +} + +// Config contains the configuration options for Indexer. +type Config struct { + History uint64 // number of historical blocks to index + Disabled bool // disables indexing completely + + // This option enables the checkpoint JSON file generator. + // If set, the given file will be updated with checkpoint information. + ExportFileName string + + // expect trie nodes of hash based state scheme in the filtermaps key range; + // use safe iterator based implementation of DeleteRange that skips them + HashScheme bool +} + +// NewIndexer creates a new Indexer. +func NewIndexer(db ethdb.KeyValueStore, params Params, config Config) *Indexer { + params.sanitize() + mapDb := newMapDatabase(¶ms, db, config.HashScheme) + ix := &Indexer{ + config: config, + storage: newMapStorage(¶ms, mapDb, nil), + checkpoints: checkpoints, + snapshots: make(map[common.Hash]*IndexView), + headMapsCache: lru.NewCache[uint32, *finishedMap](maxIndexViewMaps), + lastFinalEpoch: math.MaxUint32, } - log.Info("Started log indexer") + if config.Disabled { + ix.storage.deleteMaps(common.NewRange[uint32](0, math.MaxUint32)) + return ix + } + ix.headRenderer = ix.initMapBoundary(ix.storage.lastBoundaryBefore(math.MaxUint32), math.MaxUint32) + ix.updateTailEpoch() + ix.updateActiveViewTailEpoch() + ix.updateTailState() + return ix +} - for !f.stop { - // Note: acquiring the indexLock read lock is unnecessary here, - // as the `indexedRange` is accessed within the indexerLoop. - if !f.indexedRange.initialized { - if f.targetView.HeadNumber() == 0 { - // initialize when chain head is available - f.processSingleEvent(true) - continue - } - if err := f.init(); err != nil { - f.disableForError("initialization", err) - f.reset() // remove broken index from DB - return - } - } - if !f.targetHeadIndexed() { - if err := f.tryIndexHead(); err != nil && err != errChainUpdate { - f.disableForError("head rendering", err) - return +// GetIndexView returns an immutable IndexView corresponding to the given head +// block hash if available. Note that each returned IndexView has to be released +// after use with the Release funcion in order to avoid memory leakage. +func (ix *Indexer) GetIndexView(headBlockHash common.Hash) *IndexView { + if ix.config.Disabled { + return nil + } + ix.snapshotsLock.RLock() + iv := ix.snapshots[headBlockHash] + ix.snapshotsLock.RUnlock() + if iv == nil || iv.checkReleased() { + return nil + } + iv.addRefCount(1) + return iv +} + +// Status returns the current indexer status. The ready flag indicates whether +// the indexer is ready to process new block data. The needBlocks range, if not +// empty, indicates that the indexer requests past blocks in order to complete +// the index. These blocks should be delivered in strictly ascending order. +// Note that if ready is false then needBlocks might still be non-empty, in +// which case the blocks are not expected to be delivered yet but the index +// server might already start pre-fetching them. +// Status implements core.Indexer. +func (ix *Indexer) Status() (bool, common.Range[uint64]) { + if ix.config.Disabled { + return false, common.Range[uint64]{} + } + return ix.storage.isReady(), ix.needBlocks() +} + +// AddBlockData delivers block data for new heads and requested historical range. +// It returns the indexer status. Unwanted data is silently ignored. If the +// indexer is not ready to process then the received data is also ignored and +// then requested through the needBlocks response either in the current response +// or later. +// Note that this function also resumes the storage layer background process if +// it was previously suspended. +// AddBlockData implements core.Indexer. +func (ix *Indexer) AddBlockData(header *types.Header, receipts types.Receipts) (ready bool, needBlocks common.Range[uint64]) { + if ix.config.Disabled { + return false, common.Range[uint64]{} + } + ix.storage.suspendOrResume(false) + if !ix.storage.isReady() { + return false, ix.needBlocks() + } + ix.headNumber = max(ix.headNumber, header.Number.Uint64()) + number, hash := header.Number.Uint64(), header.Hash() + if number > ix.headRenderer.nextBlock { + ix.tryCheckpointInit(number, hash) + } + if number == ix.headRenderer.nextBlock { + if ix.headRenderer.checkNextHash(hash) { + ix.headRenderer.addReceipts(receipts) + firstMapIndex, finishedMaps := ix.headRenderer.addHeader(header) + ix.storeFinishedMaps(firstMapIndex, finishedMaps, true, true) + if number+maxCanonicalSnapshots > ix.headNumber { + ix.storeHeadIndexView(number, hash) } } else { - if f.finalBlock != f.lastFinal { - if f.exportFileName != "" { - f.exportCheckpoints() + ix.headRenderer = ix.initMapBoundary(max(ix.headRenderer.renderRange.First(), 1)-1, math.MaxUint32) + } + ix.updateTailEpoch() + ix.updateTailState() + } + if ix.tailRenderer != nil && number == ix.tailRenderer.nextBlock { + if ix.tailRenderer.checkNextHash(hash) { + ix.tailRenderer.addReceipts(receipts) + firstMapIndex, finishedMaps := ix.tailRenderer.addHeader(header) + ix.storeFinishedMaps(firstMapIndex, finishedMaps, false, false) + if ix.tailRenderer.finished() { + ix.tailEpoch-- + ix.tailRenderer = nil + ix.updateTailState() + } + } else { + // Note that if there is a canonical hash mismatch at the tail epoch then we need to revert the head renderer before this point. + ix.headRenderer = ix.initMapBoundary(max(ix.tailRenderer.renderRange.First(), 1)-1, math.MaxUint32) + ix.tailRenderer = nil + } + } + return ix.storage.isReady(), ix.needBlocks() +} + +// Revert resets the index head to the given block number. Note that the indexer +// might have to discard more data if a snapshot is not available for the given +// block number. In this case it will request previously delivered but discarded +// block data through the needBlocks status response. +// Note that Revert works even if the indexer is in a "not ready" status, thereby +// guaranteeing that all index data is always consistent with the canonical chain. +// Revert implements core.Indexer. +func (ix *Indexer) Revert(blockNumber uint64) { + if ix.config.Disabled { + return + } + firstCanonical := ix.lastCanonical + 1 - uint64(len(ix.canonicalHashes)) + if blockNumber >= firstCanonical && blockNumber <= ix.lastCanonical { + blockHash := ix.canonicalHashes[blockNumber-firstCanonical] + if snapshot, ok := ix.snapshots[blockHash]; ok { + ix.headRenderer = ix.initSnapshot(snapshot) + if ix.headRenderer != nil { + return + } + } + } + mapIndex := uint32(math.MaxUint32) + for mapIndex > 0 { + mapIndex = ix.storage.lastBoundaryBefore(mapIndex - 1) + if mapIndex == 0 { + break + } + lastNumber, _, err := ix.storage.getLastBlockOfMap(mapIndex - 1) + if err != nil { + log.Error("Last block of map not found, reverting database", "mapIndex", mapIndex) + mapIndex-- + continue + } + if lastNumber < blockNumber { + break + } + } + ix.revertMaps(mapIndex) + ix.headRenderer = ix.initMapBoundary(mapIndex, math.MaxUint32) + ix.headNumber = blockNumber + ix.updateTailEpoch() +} + +// SetFinalized notifies the indexer about the latest finalized block number. +// SetFinalized implements core.Indexer. +func (ix *Indexer) SetFinalized(blockNumber uint64) { + if ix.finalized == blockNumber { + return + } + ix.finalized = blockNumber + ix.exportCheckpoints() +} + +// SetHistoryCutoff notifies the indexer about the latest historical cutoff point. +// The indexer will not request block data earlier than this point. +// SetHistoryCutoff implements core.Indexer. +func (ix *Indexer) SetHistoryCutoff(blockNumber uint64) { + ix.historyCutoff = blockNumber +} + +// Suspended suspends the asynchronous storage layer background process during +// block processing. The next AddBlockData call will resume this process. +// Suspended implements core.Indexer. +func (ix *Indexer) Suspended() { + if ix.config.Disabled { + return + } + ix.storage.suspendOrResume(true) +} + +// initMapBoundary initializes a new map renderer at the last suitable map +// boundary before startMap. If this boundary is not right before startMap then +// startMap is lowered to right after the boundary. The returned renderState +// will render maps in the startMap..limitMap-1 range. +// Note that the first requested block typically still starts in the previous +// map and in case of tail renderers with an upper map limit, the last requested +// block typically ends after the upper limit. In this case the maps outside the +// rendered range are not modified, the log values outside the range are ignored. +func (ix *Indexer) initMapBoundary(startMap, limitMap uint32) *renderState { + rs := &renderState{ + params: ix.storage.params, + } + for { + startMap = ix.storage.lastBoundaryBefore(startMap) + if startMap == 0 { + break + } + lastNumber, lastHash, err := ix.storage.getLastBlockOfMap(startMap - 1) + if err != nil { + log.Error("Last block of map not found, reverting database", "mapIndex", startMap-1) + startMap = ix.storage.lastBoundaryBefore(startMap - 1) + ix.revertMaps(startMap) + continue + } + lvPointer, err := ix.storage.getBlockLvPointer(lastNumber) + if err != nil { + log.Error("Block pointer of last block of map not found, reverting database", "mapIndex", startMap-1, "blockNumber", lastNumber) + startMap = ix.storage.lastBoundaryBefore(startMap - 1) + ix.revertMaps(startMap) + continue + } + rs.lvPointer = lvPointer + rs.mapIndex = uint32(lvPointer >> ix.storage.params.logValuesPerMap) + rs.nextBlock = lastNumber + rs.partialBlock = true + rs.partialBlockHash = lastHash + break + } + rs.renderRange = common.NewRange[uint32](startMap, limitMap-startMap) + if rs.renderRange.Includes(rs.mapIndex) { + rs.currentMap = rs.params.newMemoryMap() + } + return rs +} + +// initSnapshot initializes a new map renderer based on a snapshot. Since this +// method is only used to initialize head renderers, a snapshot initialized +// renderState always has an upper render limit of MaxUint32-1. +func (ix *Indexer) initSnapshot(snapshot *IndexView) *renderState { + mapIndex := ix.storage.lastBoundaryBefore(snapshot.firstMemoryMap) + ix.revertMaps(mapIndex) + if snapshot.checkInvalid() { + log.Error("Failed to revert to invalidated snapshot", "blockNumber", snapshot.blockRange.Last()) + return nil + } + + return &renderState{ + params: ix.storage.params, + renderRange: common.NewRange[uint32](snapshot.headMapIndex, math.MaxUint32-snapshot.headMapIndex), + currentMap: snapshot.headMap.clone(), + mapIndex: snapshot.headMapIndex, + lvPointer: snapshot.headLvPointer, + } +} + +// revertMaps removes all rendered maps starting from mapIndex. It also removes +// active renderers and snapshots invalidated by the revert and purges the map +// cache. +// Note that while headRenderer generally always exists, revertMaps might be +// called while headRenderer is nil and might set headRenderer to nil. +func (ix *Indexer) revertMaps(mapIndex uint32) { + if mapIndex < ix.storage.lastBoundaryBefore(math.MaxUint32) { + for hash, iv := range ix.snapshots { + if iv.firstMemoryMap > mapIndex { + iv.invalidate() + ix.snapshotsLock.Lock() + delete(ix.snapshots, hash) + ix.snapshotsLock.Unlock() + } + } + ix.storage.deleteMaps(common.NewRange[uint32](mapIndex, math.MaxUint32-mapIndex)) + ix.headMapsCache.Purge() // invalidate all maps cached by index + } + if ix.headRenderer != nil && mapIndex <= ix.headRenderer.mapIndex { + ix.headRenderer = nil + } + if ix.tailRenderer != nil && mapIndex <= ix.tailRenderer.mapIndex { + ix.tailRenderer = nil + } +} + +// updateTailEpoch recalculates the current tailEpoch and the targetTailEpoch. +func (ix *Indexer) updateTailEpoch() { + ix.tailEpoch = ix.storage.tailEpoch() + if ix.config.History == 0 { + ix.targetTailEpoch = 0 + return + } + headEpoch := ix.storage.params.mapEpoch(ix.headRenderer.mapIndex) + for ix.targetTailEpoch < headEpoch { + nextTailNumber, err := ix.storage.tailNumberOfEpoch(ix.targetTailEpoch + 1) + if err != nil { + log.Error("Could not get tail block number of epoch", "epoch", ix.targetTailEpoch+1, "error", err) + return + } + if nextTailNumber+ix.config.History > ix.headRenderer.nextBlock { + break + } + ix.targetTailEpoch++ + } + for ix.targetTailEpoch > 0 { + prevTailEpoch := ix.storage.params.mapEpoch(ix.storage.lastBoundaryBefore(ix.storage.params.firstEpochMap(ix.targetTailEpoch - 1))) + prevTailNumber, err := ix.storage.tailNumberOfEpoch(prevTailEpoch) + if err != nil { + log.Error("Could not get tail block number of epoch", "epoch", prevTailEpoch, "error", err) + return + } + if prevTailNumber+ix.config.History <= ix.headRenderer.nextBlock { + break + } + ix.targetTailEpoch = prevTailEpoch + } +} + +// updateActiveViewTailEpoch recalculates activeViewTailEpoch which is the earliest +// tail epoch required by an active IndexView. Tail unindexing is only allowed +// if min(targetTailEpoch, activeViewTailEpoch) > tailEpoch. +func (ix *Indexer) updateActiveViewTailEpoch() { + ix.snapshotsLock.RLock() + defer ix.snapshotsLock.RUnlock() + + ix.activeViewTailEpoch = math.MaxUint32 + for _, iv := range ix.snapshots { + ix.activeViewTailEpoch = min(ix.activeViewTailEpoch, iv.tailEpoch) + } +} + +// updateTailState performs tail unindexing or initializes a new tailRenderer to +// render a new tail epoch if necessary. +func (ix *Indexer) updateTailState() { + epoch := min(ix.targetTailEpoch, ix.activeViewTailEpoch) + if epoch >= ix.tailEpoch && ix.tailRenderer != nil { + ix.tailRenderer = nil + ix.storage.deleteMaps(common.NewRange[uint32](ix.storage.params.firstEpochMap(ix.tailEpoch-1), ix.storage.params.mapsPerEpoch)) + } + if epoch > ix.tailEpoch { + ix.storage.deleteMaps(common.NewRange[uint32](ix.storage.params.firstEpochMap(ix.tailEpoch), ix.storage.params.mapsPerEpoch*(epoch-ix.tailEpoch))) + if epoch == ix.tailEpoch+1 { + log.Info("Unindexed tail epoch #%d", ix.tailEpoch) + } else { + log.Info("Unindexed tail epochs #%d to #%d", ix.tailEpoch, epoch-1) + } + ix.tailEpoch = epoch + } + if epoch < ix.tailEpoch && ix.tailRenderer == nil { + if lastBlock, _, err := ix.storage.getLastBlockOfMap(ix.storage.params.lastEpochMap(ix.tailEpoch - 1)); err == nil { + ix.tailRenderer = ix.initMapBoundary(ix.storage.lastBoundaryBefore(ix.storage.params.lastEpochMap(ix.tailEpoch-1)), ix.storage.params.firstEpochMap(ix.tailEpoch)) + ix.tailRenderLast = lastBlock + } else { + log.Error("Could not get last block of new tail epoch", "epoch", ix.tailEpoch-1, "error", err) + } + } +} + +// epochsUntilBlock returns the numer of epochs in the checkpoint list whose +// last block number is less than or equal to the specified number. +func (cpList checkpointList) epochsUntilBlock(number uint64) uint32 { + first, last := uint32(0), uint32(len(cpList)) + for first < last { + mid := (first + last) / 2 + if cpList[mid].BlockNumber > number { + last = mid + } else { + first = mid + 1 + } + } + return first +} + +func (ix *Indexer) tryCheckpointInit(number uint64, hash common.Hash) { + var ci int + for ci < len(ix.checkpoints) { + cpList := ix.checkpoints[ci] + epochs := cpList.epochsUntilBlock(number) + if epochs == 0 || cpList[epochs-1].BlockNumber != number { + // block number does not match, skip list (a relevant block might match later) + ci++ + continue + } + if cpList[epochs-1].BlockHash == hash { + // apply matching checkpoint, discard other lists + if err := ix.storage.addKnownEpochs(cpList[:epochs]); err == nil { + ix.checkpoints = []checkpointList{cpList} + ix.headRenderer = ix.initMapBoundary(ix.storage.params.firstEpochMap(epochs), math.MaxUint32) + return + } else { + log.Error("Error initializing epoch boundaries", "error", err) + } + } + // checkpoint does not match, discard list + ix.checkpoints[ci] = ix.checkpoints[len(ix.checkpoints)-1] + ix.checkpoints = ix.checkpoints[:len(ix.checkpoints)-1] + } +} + +func (ix *Indexer) needBlocks() common.Range[uint64] { + if ix.finalized > ix.headRenderer.nextBlock { + // request potential checkpoint in this range if available + for _, cpList := range ix.checkpoints { + if epochs := cpList.epochsUntilBlock(ix.headNumber); epochs > 0 { + blockNumber := cpList[epochs-1].BlockNumber + if ix.storage.lastBoundaryBefore(math.MaxUint32) >= ix.storage.params.firstEpochMap(epochs) || + blockNumber <= ix.headRenderer.nextBlock || blockNumber < ix.historyCutoff { + continue } - f.lastFinal = f.finalBlock + return common.NewRange[uint64](blockNumber, 1) } - // always attempt unindexing before indexing the tail in order to - // ensure that a potentially dirty previously unindexed epoch is - // always cleaned up before any new maps are rendered. - if done, err := f.tryUnindexTail(); err != nil { - f.disableForError("tail unindexing", err) - return - } else if !done { - continue - } - if done, err := f.tryIndexTail(); err != nil { - f.disableForError("tail rendering", err) - return - } else if !done { - continue - } - // tail indexing/unindexing is done; if head is also indexed then - // wait here until there is a new head - f.waitForNewHead() + } + } + if ix.headRenderer.nextBlock <= ix.headNumber && ix.headRenderer.nextBlock >= ix.historyCutoff { + return common.NewRange[uint64](ix.headRenderer.nextBlock, ix.headNumber+1-ix.headRenderer.nextBlock) + } + if ix.tailRenderer != nil && + ix.tailRenderer.nextBlock <= ix.tailRenderLast && ix.tailRenderer.nextBlock >= ix.historyCutoff { + return common.NewRange[uint64](ix.tailRenderer.nextBlock, ix.tailRenderLast+1-ix.tailRenderer.nextBlock) + } + return common.Range[uint64]{} +} + +func (ix *Indexer) Stop() { + ix.storage.stop() +} + +func (ix *Indexer) releaseView(hash common.Hash) { + iv := ix.snapshots[hash] + if iv == nil { + return + } + if iv.addRefCount(-1) { + iv.invalidate() + ix.snapshotsLock.Lock() + delete(ix.snapshots, hash) + ix.snapshotsLock.Unlock() + } +} + +func (ix *Indexer) storeFinishedMaps(firstMapIndex uint32, maps []*finishedMap, forceCommit, cacheHeadMaps bool) { + if len(maps) == 0 { + return + } + for i, fm := range maps { + ix.storage.addMap(firstMapIndex+uint32(i), fm, forceCommit && i == len(maps)-1) + if cacheHeadMaps { + ix.headMapsCache.Add(firstMapIndex+uint32(i), fm) } } } -// disableForError is called when the indexer encounters a database error, for example a -// missing receipt. We can't continue operating when the database is broken, so the -// indexer goes into disabled state. -// Note that the partial index is left in disk; maybe a client update can fix the -// issue without reindexing. -func (f *FilterMaps) disableForError(op string, err error) { - log.Error("Log index "+op+" failed, reverting to unindexed mode", "error", err) - f.disabled = true - close(f.disabledCh) -} - -type targetUpdate struct { - targetView *ChainView - historyCutoff, finalBlock uint64 -} - -// SetTarget sets a new target chain view for the indexer to render. -// Note that SetTargetView never blocks. -func (f *FilterMaps) SetTarget(targetView *ChainView, historyCutoff, finalBlock uint64) { - if targetView == nil { - panic("nil targetView") +func (ix *Indexer) getFilterMap(mapIndex uint32) (*finishedMap, error) { + if fm, ok := ix.headMapsCache.Get(mapIndex); ok { + return fm, nil } - for { - select { - case <-f.targetCh: - case f.targetCh <- targetUpdate{ - targetView: targetView, - historyCutoff: historyCutoff, - finalBlock: finalBlock, - }: - return + fm, err := ix.storage.getFilterMap(mapIndex) + if err != nil { + return nil, err + } + ix.headMapsCache.Add(mapIndex, fm) + return fm, nil +} + +func (ix *Indexer) checkReleasedViews() { + var deleted bool + for hash, iv := range ix.snapshots { + if iv.checkReleased() { + iv.invalidate() + ix.snapshotsLock.Lock() + delete(ix.snapshots, hash) + ix.snapshotsLock.Unlock() + deleted = true } } + if deleted { + ix.updateActiveViewTailEpoch() + ix.updateTailState() + } } -// SetBlockProcessing sets the block processing flag that temporarily suspends -// log index rendering. -// Note that SetBlockProcessing never blocks. -func (f *FilterMaps) SetBlockProcessing(blockProcessing bool) { - for { - select { - case <-f.blockProcessingCh: - case f.blockProcessingCh <- blockProcessing: - return +func (ix *Indexer) storeHeadIndexView(number uint64, hash common.Hash) { + if ix.headRenderer.currentMap == nil { + return + } + ix.checkReleasedViews() + firstMemoryMap := max(ix.headRenderer.mapIndex, maxIndexViewMaps) - maxIndexViewMaps + finishedMaps := make([]*finishedMap, 0, ix.headRenderer.mapIndex-firstMemoryMap) + for mapIndex := firstMemoryMap; mapIndex < ix.headRenderer.mapIndex; mapIndex++ { + fm, err := ix.getFilterMap(mapIndex) + if err != nil { + log.Error("Error loading recent filter map", "mapIndex", mapIndex, "error", err) + } + if fm != nil && err == nil { + finishedMaps = append(finishedMaps, fm) + } else { + finishedMaps = finishedMaps[:0] + firstMemoryMap = mapIndex + 1 } } -} - -// WaitIdle blocks until the indexer is in an idle state while synced up to the -// latest targetView. -func (f *FilterMaps) WaitIdle() { - for { - ch := make(chan bool) - select { - case f.waitIdleCh <- ch: - if <-ch { - return - } - case <-f.disabledCh: - f.closeWg.Wait() - return - } + var firstMemoryBlock uint64 + if len(finishedMaps) > 0 { + firstMemoryBlock = finishedMaps[0].firstBlock() + } else { + firstMemoryBlock = ix.headRenderer.currentMap.firstBlock() } -} - -// waitForNewHead blocks until there is a new target head to index and block -// processing has been finished. -func (f *FilterMaps) waitForNewHead() { - for !f.stop && (f.blockProcessing || f.targetHeadIndexed()) { - f.processSingleEvent(true) + tailEpoch := max(ix.tailEpoch, ix.targetTailEpoch) + tailNumber, err := ix.storage.tailNumberOfEpoch(tailEpoch) + if err != nil { + log.Error("Could not get tail block number of epoch", "epoch", tailEpoch, "error", err) + return } -} - -// processEvents processes all events, blocking only if a block processing is -// happening and indexing should be suspended. -func (f *FilterMaps) processEvents() { - if f.testProcessEventsHook != nil { - f.testProcessEventsHook() + ix.snapshotsLock.Lock() + ix.snapshots[hash] = &IndexView{ + refCount: 2, + storage: ix.storage, + tailEpoch: tailEpoch, + blockRange: common.NewRange(tailNumber, number+1-tailNumber), + headBlockHash: hash, + headLvPointer: ix.headRenderer.lvPointer, + headMap: ix.headRenderer.currentMap.clone(), + headMapIndex: ix.headRenderer.mapIndex, + firstMemoryMap: firstMemoryMap, + firstMemoryBlock: firstMemoryBlock, + finishedMaps: finishedMaps, } - for f.processSingleEvent(f.blockProcessing) { - } -} - -// processSingleEvent processes a single event either in a blocking or -// non-blocking manner. It returns true if it did process an event. -func (f *FilterMaps) processSingleEvent(blocking bool) bool { - if f.stop { - return false - } - // Note: acquiring the indexLock read lock is unnecessary here, - // as this function is always called within the indexLoop. - if !f.hasTempRange { - for _, mb := range f.matcherSyncRequests { - mb.synced() - } - f.matcherSyncRequests = nil - } - if blocking { - select { - case target := <-f.targetCh: - f.setTarget(target) - case mb := <-f.matcherSyncCh: - f.matcherSyncRequests = append(f.matcherSyncRequests, mb) - case f.blockProcessing = <-f.blockProcessingCh: - case <-f.closeCh: - f.stop = true - case ch := <-f.waitIdleCh: - select { - case target := <-f.targetCh: - f.setTarget(target) - default: - } - ch <- !f.blockProcessing && f.targetHeadIndexed() + ix.snapshotsLock.Unlock() + if number == ix.lastCanonical+1 { + if len(ix.canonicalHashes) == maxCanonicalSnapshots { + ix.releaseView(ix.canonicalHashes[0]) + copy(ix.canonicalHashes[0:maxCanonicalSnapshots-1], ix.canonicalHashes[1:maxCanonicalSnapshots]) + ix.canonicalHashes[maxCanonicalSnapshots-1] = hash + } else { + ix.canonicalHashes = append(ix.canonicalHashes, hash) } } else { - select { - case target := <-f.targetCh: - f.setTarget(target) - case mb := <-f.matcherSyncCh: - f.matcherSyncRequests = append(f.matcherSyncRequests, mb) - case f.blockProcessing = <-f.blockProcessingCh: - case <-f.closeCh: - f.stop = true - default: - return false + for _, oldHash := range ix.canonicalHashes { + ix.releaseView(oldHash) } + ix.canonicalHashes = []common.Hash{hash} } - return true + ix.lastCanonical = number + if len(ix.recentHashes) == maxRecentSnapshots { + ix.releaseView(ix.recentHashes[0]) + copy(ix.recentHashes[0:maxRecentSnapshots-1], ix.recentHashes[1:maxRecentSnapshots]) + ix.recentHashes[maxRecentSnapshots-1] = hash + } else { + ix.recentHashes = append(ix.recentHashes, hash) + } + ix.updateActiveViewTailEpoch() + ix.updateTailState() } -// setTarget updates the target chain view of the iterator. -func (f *FilterMaps) setTarget(target targetUpdate) { - f.targetView = target.targetView - f.historyCutoff = target.historyCutoff - f.finalBlock = target.finalBlock -} - -// tryIndexHead tries to render head maps according to the current targetView. -// Should be called when targetHeadIndexed returns false. If this function -// returns no error then either stop is true or head indexing is finished. -func (f *FilterMaps) tryIndexHead() error { - headRenderer, err := f.renderMapsBefore(math.MaxUint32) +// exportCheckpoints exports epoch checkpoints in the format used by checkpoints.go. +func (ix *Indexer) exportCheckpoints() { + finalLvPtr, err := ix.storage.getBlockLvPointer(ix.finalized + 1) if err != nil { - return err - } - if headRenderer == nil { - return errors.New("head indexer has nothing to do") // tryIndexHead should be called when head is not indexed - } - if !f.startedHeadIndex { - f.lastLogHeadIndex = time.Now() - f.startedHeadIndexAt = f.lastLogHeadIndex - f.startedHeadIndex = true - f.ptrHeadIndex = f.indexedRange.blocks.AfterLast() - } - if _, err := headRenderer.run(func() bool { - f.processEvents() - return f.stop - }, func() { - f.tryUnindexTail() - if f.indexedRange.hasIndexedBlocks() && f.indexedRange.blocks.AfterLast() >= f.ptrHeadIndex && - ((!f.loggedHeadIndex && time.Since(f.startedHeadIndexAt) > headLogDelay) || - time.Since(f.lastLogHeadIndex) > logFrequency) { - log.Info("Log index head rendering in progress", - "firstblock", f.indexedRange.blocks.First(), "lastblock", f.indexedRange.blocks.Last(), - "processed", f.indexedRange.blocks.AfterLast()-f.ptrHeadIndex, - "remaining", f.indexedView.HeadNumber()-f.indexedRange.blocks.Last(), - "elapsed", common.PrettyDuration(time.Since(f.startedHeadIndexAt))) - f.loggedHeadIndex = true - f.lastLogHeadIndex = time.Now() + if err != ErrOutOfRange { + log.Error("Error fetching log value pointer of finalized block", "block", ix.finalized, "error", err) } - }); err != nil { - return err + return } - if f.loggedHeadIndex && f.indexedRange.hasIndexedBlocks() { - log.Info("Log index head rendering finished", - "firstblock", f.indexedRange.blocks.First(), "lastblock", f.indexedRange.blocks.Last(), - "processed", f.indexedRange.blocks.AfterLast()-f.ptrHeadIndex, - "elapsed", common.PrettyDuration(time.Since(f.startedHeadIndexAt))) + epochCount := ix.storage.params.mapEpoch(uint32(finalLvPtr >> ix.storage.params.logValuesPerMap)) + if epochCount == ix.lastFinalEpoch { + return } - f.loggedHeadIndex, f.startedHeadIndex = false, false - return nil -} - -// tryIndexTail tries to render tail epochs until the tail target block is -// indexed and returns true if successful. -// Note that tail indexing is only started if the log index head is fully -// rendered according to targetView and is suspended as soon as the targetView -// is changed. -func (f *FilterMaps) tryIndexTail() (bool, error) { - for { - firstEpoch := f.mapEpoch(f.indexedRange.maps.First()) - if firstEpoch == 0 || !f.needTailEpoch(firstEpoch-1) { - break - } - f.processEvents() - if f.stop || !f.targetHeadIndexed() { - return false, nil - } - // resume process if tail rendering was interrupted because of head rendering - tailRenderer := f.tailRenderer - f.tailRenderer = nil - if tailRenderer != nil && tailRenderer.renderBefore != f.indexedRange.maps.First() { - tailRenderer = nil - } - if tailRenderer == nil { - var err error - tailRenderer, err = f.renderMapsBefore(f.indexedRange.maps.First()) - if err != nil { - return false, err - } - } - if tailRenderer == nil { - break - } - if !f.startedTailIndex { - f.lastLogTailIndex = time.Now() - f.startedTailIndexAt = f.lastLogTailIndex - f.startedTailIndex = true - f.ptrTailIndex = f.indexedRange.blocks.First() - f.tailPartialBlocks() - } - done, err := tailRenderer.run(func() bool { - f.processEvents() - return f.stop || !f.targetHeadIndexed() - }, func() { - tpb, ttb := f.tailPartialBlocks(), f.tailTargetBlock() - remaining := uint64(1) - if f.indexedRange.blocks.First() > ttb+tpb { - remaining = f.indexedRange.blocks.First() - ttb - tpb - } - if f.indexedRange.hasIndexedBlocks() && f.ptrTailIndex >= f.indexedRange.blocks.First() && - (!f.loggedTailIndex || time.Since(f.lastLogTailIndex) > logFrequency) { - log.Info("Log index tail rendering in progress", - "firstblock", f.indexedRange.blocks.First(), "last block", f.indexedRange.blocks.Last(), - "processed", f.ptrTailIndex-f.indexedRange.blocks.First()+tpb, - "remaining", remaining, - "next tail epoch percentage", f.indexedRange.tailPartialEpoch*100/f.mapsPerEpoch, - "elapsed", common.PrettyDuration(time.Since(f.startedTailIndexAt))) - f.loggedTailIndex = true - f.lastLogTailIndex = time.Now() - } - }) - if err != nil && !f.needTailEpoch(firstEpoch-1) { - // stop silently if cutoff point has move beyond epoch boundary while rendering - return true, nil - } - if err != nil { - return false, err - } - if !done { - f.tailRenderer = tailRenderer // only keep tail renderer if interrupted by stopCb - return false, nil - } - } - if f.loggedTailIndex && f.indexedRange.hasIndexedBlocks() { - log.Info("Log index tail rendering finished", - "firstblock", f.indexedRange.blocks.First(), "lastblock", f.indexedRange.blocks.Last(), - "processed", f.ptrTailIndex-f.indexedRange.blocks.First(), - "elapsed", common.PrettyDuration(time.Since(f.startedTailIndexAt))) - f.loggedTailIndex = false - } - return true, nil -} - -// tryUnindexTail removes entire epochs of log index data as long as the first -// fully indexed block is at least as old as the tail target. -// Note that unindexing is very quick as it only removes continuous ranges of -// data from the database and is also called while running head indexing. -func (f *FilterMaps) tryUnindexTail() (bool, error) { - firstEpoch := f.mapEpoch(f.indexedRange.maps.First()) - if f.indexedRange.tailPartialEpoch > 0 && firstEpoch > 0 { - firstEpoch-- - } - for epoch := min(firstEpoch, f.cleanedEpochsBefore); !f.needTailEpoch(epoch); epoch++ { - if !f.startedTailUnindex { - f.startedTailUnindexAt = time.Now() - f.startedTailUnindex = true - f.ptrTailUnindexMap = f.indexedRange.maps.First() - f.indexedRange.tailPartialEpoch - f.ptrTailUnindexBlock = f.indexedRange.blocks.First() - f.tailPartialBlocks() - } - if done, err := f.deleteTailEpoch(epoch); !done { - return false, err - } - f.processEvents() - if f.stop || !f.targetHeadIndexed() { - return false, nil - } - } - if f.startedTailUnindex && f.indexedRange.hasIndexedBlocks() { - log.Info("Log index tail unindexing finished", - "firstblock", f.indexedRange.blocks.First(), "lastblock", f.indexedRange.blocks.Last(), - "removedmaps", f.indexedRange.maps.First()-f.ptrTailUnindexMap, - "removedblocks", f.indexedRange.blocks.First()-f.tailPartialBlocks()-f.ptrTailUnindexBlock, - "elapsed", common.PrettyDuration(time.Since(f.startedTailUnindexAt))) - f.startedTailUnindex = false - } - return true, nil -} - -// needTailEpoch returns true if the given tail epoch needs to be kept -// according to the current tail target, false if it can be removed. -func (f *FilterMaps) needTailEpoch(epoch uint32) bool { - firstEpoch := f.mapEpoch(f.indexedRange.maps.First()) - if epoch > firstEpoch { - return true - } - if f.firstEpochMap(epoch+1) >= f.indexedRange.maps.AfterLast() { - return true - } - if epoch+1 < firstEpoch { - return false - } - var lastBlockOfPrevEpoch uint64 - if epoch > 0 { - var err error - lastBlockOfPrevEpoch, _, err = f.getLastBlockOfMap(f.lastEpochMap(epoch - 1)) - if err != nil { - log.Error("Could not get last block of previous epoch", "epoch", epoch-1, "error", err) - return epoch >= firstEpoch - } - } - if f.historyCutoff > lastBlockOfPrevEpoch { - return false - } - lastBlockOfEpoch, _, err := f.getLastBlockOfMap(f.lastEpochMap(epoch)) + w, err := os.Create(ix.config.ExportFileName) if err != nil { - log.Error("Could not get last block of epoch", "epoch", epoch, "error", err) - return epoch >= firstEpoch + log.Error("Error creating checkpoint export file", "name", ix.config.ExportFileName, "error", err) + return } - return f.tailTargetBlock() <= lastBlockOfEpoch -} + defer w.Close() -// tailTargetBlock returns the target value for the tail block number according -// to the log history parameter and the current index head. -func (f *FilterMaps) tailTargetBlock() uint64 { - if f.history == 0 || f.indexedView.HeadNumber() < f.history { - return 0 - } - return f.indexedView.HeadNumber() + 1 - f.history -} - -// tailPartialBlocks returns the number of rendered blocks in the partially -// rendered next tail epoch. -func (f *FilterMaps) tailPartialBlocks() uint64 { - if f.indexedRange.tailPartialEpoch == 0 { - return 0 - } - end, _, err := f.getLastBlockOfMap(f.indexedRange.maps.First() - f.mapsPerEpoch + f.indexedRange.tailPartialEpoch - 1) - if err != nil { - log.Error("Error fetching last block of map", "mapIndex", f.indexedRange.maps.First()-f.mapsPerEpoch+f.indexedRange.tailPartialEpoch-1, "error", err) - } - var start uint64 - if f.indexedRange.maps.First()-f.mapsPerEpoch > 0 { - start, _, err = f.getLastBlockOfMap(f.indexedRange.maps.First() - f.mapsPerEpoch - 1) + log.Info("Exporting log index checkpoints", "epochs", epochCount, "file", ix.config.ExportFileName) + w.WriteString("[\n") + comma := "," + for epoch := uint32(0); epoch < epochCount; epoch++ { + lastBlock, lastBlockId, err := ix.storage.getLastBlockOfMap(ix.storage.params.lastEpochMap(epoch)) if err != nil { - log.Error("Error fetching last block of map", "mapIndex", f.indexedRange.maps.First()-f.mapsPerEpoch-1, "error", err) + log.Error("Error fetching last block of epoch", "epoch", epoch, "error", err) + return } + lvPtr, err := ix.storage.getBlockLvPointer(lastBlock) + if err != nil { + log.Error("Error fetching log value pointer of last block", "block", lastBlock, "error", err) + return + } + if epoch == epochCount-1 { + comma = "" + } + w.WriteString(fmt.Sprintf("{\"blockNumber\": %d, \"blockId\": \"0x%064x\", \"firstIndex\": %d}%s\n", lastBlock, lastBlockId, lvPtr, comma)) } - return end - start -} - -// targetHeadIndexed returns true if the current log index is consistent with -// targetView with its head block fully rendered. -func (f *FilterMaps) targetHeadIndexed() bool { - return equalViews(f.targetView, f.indexedView) && f.indexedRange.headIndexed + w.WriteString("]\n") + ix.lastFinalEpoch = epochCount } diff --git a/core/filtermaps/indexer_test.go b/core/filtermaps/indexer_test.go deleted file mode 100644 index 35441905b7..0000000000 --- a/core/filtermaps/indexer_test.go +++ /dev/null @@ -1,637 +0,0 @@ -// Copyright 2024 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package filtermaps - -import ( - "context" - crand "crypto/rand" - "crypto/sha256" - "encoding/binary" - "math/big" - "math/rand" - "sync" - "testing" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/consensus/ethash" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/rlp" -) - -var testParams = Params{ - logMapHeight: 2, - logMapWidth: 24, - logMapsPerEpoch: 4, - logValuesPerMap: 4, - baseRowGroupSize: 4, - baseRowLengthRatio: 2, - logLayerDiff: 2, -} - -func TestIndexerRandomRange(t *testing.T) { - ts := newTestSetup(t) - defer ts.close() - - forks := make([][]common.Hash, 10) - ts.chain.addBlocks(1000, 5, 2, 4, false) - for i := range forks { - if i != 0 { - forkBlock := rand.Intn(1000) - ts.chain.setHead(forkBlock) - ts.chain.addBlocks(1000-forkBlock, 5, 2, 4, false) - } - forks[i] = ts.chain.getCanonicalChain() - } - expspos := func(block uint64) uint64 { // expected position of block start - if block == 0 { - return 0 - } - logCount := (block - 1) * 5 * 2 - mapIndex := logCount / 3 - spos := mapIndex*16 + (logCount%3)*5 - if mapIndex == 0 || logCount%3 != 0 { - spos++ - } - return spos - } - expdpos := func(block uint64) uint64 { // expected position of delimiter - if block == 0 { - return 0 - } - logCount := block * 5 * 2 - mapIndex := (logCount - 1) / 3 - return mapIndex*16 + (logCount-mapIndex*3)*5 - } - ts.setHistory(0, false) - var ( - history int - noHistory bool - fork, head = len(forks) - 1, 1000 - checkSnapshot bool - ) - ts.fm.WaitIdle() - for i := 0; i < 200; i++ { - switch rand.Intn(3) { - case 0: - // change history settings - switch rand.Intn(10) { - case 0: - history, noHistory = 0, false - case 1: - history, noHistory = 0, true - default: - history, noHistory = rand.Intn(1000)+1, false - } - ts.testDisableSnapshots = rand.Intn(2) == 0 - ts.setHistory(uint64(history), noHistory) - case 1: - // change head to random position of random fork - fork, head = rand.Intn(len(forks)), rand.Intn(1001) - ts.chain.setCanonicalChain(forks[fork][:head+1]) - case 2: - checkSnapshot = false - if head < 1000 { - checkSnapshot = !noHistory && head != 0 // no snapshot generated for block 0 - // add blocks after the current head - head += rand.Intn(1000-head) + 1 - ts.fm.testSnapshotUsed = false - ts.chain.setCanonicalChain(forks[fork][:head+1]) - } - } - ts.fm.WaitIdle() - if checkSnapshot { - if ts.fm.testSnapshotUsed == ts.fm.testDisableSnapshots { - ts.t.Fatalf("Invalid snapshot used state after head extension (used: %v, disabled: %v)", ts.fm.testSnapshotUsed, ts.fm.testDisableSnapshots) - } - checkSnapshot = false - } - if noHistory { - if ts.fm.indexedRange.initialized { - t.Fatalf("filterMapsRange initialized while indexing is disabled") - } - continue - } - if !ts.fm.indexedRange.initialized { - t.Fatalf("filterMapsRange not initialized while indexing is enabled") - } - var tailBlock uint64 - if history > 0 && history <= head { - tailBlock = uint64(head + 1 - history) - } - var tailEpoch uint32 - if tailBlock > 0 { - tailLvPtr := expspos(tailBlock) - 1 - tailEpoch = uint32(tailLvPtr >> (testParams.logValuesPerMap + testParams.logMapsPerEpoch)) - } - tailLvPtr := uint64(tailEpoch) << (testParams.logValuesPerMap + testParams.logMapsPerEpoch) // first available lv ptr - var expTailBlock uint64 - if tailEpoch > 0 { - for expspos(expTailBlock) <= tailLvPtr { - expTailBlock++ - } - } - if ts.fm.indexedRange.blocks.Last() != uint64(head) { - ts.t.Fatalf("Invalid index head (expected #%d, got #%d)", head, ts.fm.indexedRange.blocks.Last()) - } - expHeadDelimiter := expdpos(uint64(head)) - if ts.fm.indexedRange.headDelimiter != expHeadDelimiter { - ts.t.Fatalf("Invalid index head delimiter pointer (expected %d, got %d)", expHeadDelimiter, ts.fm.indexedRange.headDelimiter) - } - - if ts.fm.indexedRange.blocks.First() != expTailBlock { - ts.t.Fatalf("Invalid index tail block (expected #%d, got #%d)", expTailBlock, ts.fm.indexedRange.blocks.First()) - } - } -} - -func TestIndexerMatcherView(t *testing.T) { - testIndexerMatcherView(t, false) -} - -func TestIndexerMatcherViewWithConcurrentRead(t *testing.T) { - testIndexerMatcherView(t, true) -} - -func testIndexerMatcherView(t *testing.T, concurrentRead bool) { - ts := newTestSetup(t) - defer ts.close() - - forks := make([][]common.Hash, 20) - hashes := make([]common.Hash, 20) - ts.chain.addBlocks(100, 5, 2, 4, true) - ts.setHistory(0, false) - for i := range forks { - if i != 0 { - ts.chain.setHead(100 - i) - ts.chain.addBlocks(i, 5, 2, 4, true) - } - ts.fm.WaitIdle() - forks[i] = ts.chain.getCanonicalChain() - hashes[i] = ts.matcherViewHash() - } - fork := len(forks) - 1 - for i := 0; i < 5000; i++ { - oldFork := fork - fork = rand.Intn(len(forks)) - stopCh := make(chan chan struct{}) - if concurrentRead { - go func() { - for { - ts.matcherViewHash() - select { - case ch := <-stopCh: - close(ch) - return - default: - } - } - }() - } - ts.chain.setCanonicalChain(forks[fork]) - ts.fm.WaitIdle() - if concurrentRead { - ch := make(chan struct{}) - stopCh <- ch - <-ch - } - hash := ts.matcherViewHash() - if hash != hashes[fork] { - t.Fatalf("Matcher view hash mismatch when switching from for %d to %d", oldFork, fork) - } - } -} - -func TestLogsByIndex(t *testing.T) { - ts := newTestSetup(t) - defer func() { - ts.fm.testProcessEventsHook = nil - ts.close() - }() - - ts.chain.addBlocks(1000, 10, 3, 4, true) - ts.setHistory(0, false) - ts.fm.WaitIdle() - firstLog := make([]uint64, 1001) // first valid log position per block - lastLog := make([]uint64, 1001) // last valid log position per block - for i := uint64(0); i <= ts.fm.indexedRange.headDelimiter; i++ { - log, err := ts.fm.getLogByLvIndex(i) - if err != nil { - t.Fatalf("Error getting log by index %d: %v", i, err) - } - if log != nil { - if firstLog[log.BlockNumber] == 0 { - firstLog[log.BlockNumber] = i - } - lastLog[log.BlockNumber] = i - } - } - var failed bool - ts.fm.testProcessEventsHook = func() { - if ts.fm.indexedRange.blocks.IsEmpty() { - return - } - if lvi := firstLog[ts.fm.indexedRange.blocks.First()]; lvi != 0 { - log, err := ts.fm.getLogByLvIndex(lvi) - if log == nil || err != nil { - t.Errorf("Error getting first log of indexed block range: %v", err) - failed = true - } - } - if lvi := lastLog[ts.fm.indexedRange.blocks.Last()]; lvi != 0 { - log, err := ts.fm.getLogByLvIndex(lvi) - if log == nil || err != nil { - t.Errorf("Error getting last log of indexed block range: %v", err) - failed = true - } - } - } - chain := ts.chain.getCanonicalChain() - for i := 0; i < 1000 && !failed; i++ { - head := rand.Intn(len(chain)) - ts.chain.setCanonicalChain(chain[:head+1]) - ts.fm.WaitIdle() - } -} - -func TestIndexerCompareDb(t *testing.T) { - ts := newTestSetup(t) - defer ts.close() - - ts.chain.addBlocks(500, 10, 3, 4, true) - ts.setHistory(0, false) - ts.fm.WaitIdle() - // revert points are stored after block 500 - ts.chain.addBlocks(500, 10, 3, 4, true) - ts.fm.WaitIdle() - chain1 := ts.chain.getCanonicalChain() - ts.storeDbHash("chain 1 [0, 1000]") - - ts.chain.setHead(600) - ts.fm.WaitIdle() - ts.storeDbHash("chain 1/2 [0, 600]") - - ts.chain.addBlocks(600, 10, 3, 4, true) - ts.fm.WaitIdle() - chain2 := ts.chain.getCanonicalChain() - ts.storeDbHash("chain 2 [0, 1200]") - - ts.chain.setHead(600) - ts.fm.WaitIdle() - ts.checkDbHash("chain 1/2 [0, 600]") - - ts.setHistory(800, false) - ts.chain.setCanonicalChain(chain1) - ts.fm.WaitIdle() - ts.storeDbHash("chain 1 [201, 1000]") - - ts.setHistory(0, false) - ts.fm.WaitIdle() - ts.checkDbHash("chain 1 [0, 1000]") - - ts.setHistory(800, false) - ts.chain.setCanonicalChain(chain2) - ts.fm.WaitIdle() - ts.storeDbHash("chain 2 [401, 1200]") - - ts.setHistory(0, true) - ts.fm.WaitIdle() - ts.storeDbHash("no index") - - ts.chain.setCanonicalChain(chain2[:501]) - ts.setHistory(0, false) - ts.fm.WaitIdle() - ts.chain.setCanonicalChain(chain2) - ts.fm.WaitIdle() - ts.checkDbHash("chain 2 [0, 1200]") - - ts.chain.setCanonicalChain(chain1) - ts.fm.WaitIdle() - ts.setHistory(800, false) - ts.fm.WaitIdle() - ts.checkDbHash("chain 1 [201, 1000]") - - ts.chain.setCanonicalChain(chain2) - ts.fm.WaitIdle() - ts.checkDbHash("chain 2 [401, 1200]") - - ts.setHistory(0, true) - ts.fm.WaitIdle() - ts.checkDbHash("no index") -} - -type testSetup struct { - t *testing.T - fm *FilterMaps - db ethdb.Database - chain *testChain - params Params - dbHashes map[string]common.Hash - testDisableSnapshots bool -} - -func newTestSetup(t *testing.T) *testSetup { - params := testParams - params.deriveFields() - ts := &testSetup{ - t: t, - db: rawdb.NewMemoryDatabase(), - params: params, - dbHashes: make(map[string]common.Hash), - } - ts.chain = ts.newTestChain() - return ts -} - -func (ts *testSetup) setHistory(history uint64, noHistory bool) { - if ts.fm != nil { - ts.fm.Stop() - } - head := ts.chain.CurrentBlock() - view := NewChainView(ts.chain, head.Number.Uint64(), head.Hash()) - config := Config{ - History: history, - Disabled: noHistory, - } - ts.fm, _ = NewFilterMaps(ts.db, view, 0, 0, ts.params, config) - ts.fm.testDisableSnapshots = ts.testDisableSnapshots - ts.fm.Start() -} - -func (ts *testSetup) storeDbHash(id string) { - dbHash := ts.fmDbHash() - for otherId, otherHash := range ts.dbHashes { - if otherHash == dbHash { - ts.t.Fatalf("Unexpected equal database hashes `%s` and `%s`", id, otherId) - } - } - ts.dbHashes[id] = dbHash -} - -func (ts *testSetup) checkDbHash(id string) { - if ts.fmDbHash() != ts.dbHashes[id] { - ts.t.Fatalf("Database `%s` hash mismatch", id) - } -} - -func (ts *testSetup) fmDbHash() common.Hash { - hasher := sha256.New() - it := ts.db.NewIterator(nil, nil) - for it.Next() { - hasher.Write(it.Key()) - hasher.Write(it.Value()) - } - it.Release() - var result common.Hash - hasher.Sum(result[:0]) - return result -} - -func (ts *testSetup) matcherViewHash() common.Hash { - mb := ts.fm.NewMatcherBackend() - defer mb.Close() - - ctx := context.Background() - params := mb.GetParams() - hasher := sha256.New() - var headPtr uint64 - for b := uint64(0); ; b++ { - lvptr, err := mb.GetBlockLvPointer(ctx, b) - if err != nil || (b > 0 && lvptr == headPtr) { - break - } - var enc [8]byte - binary.LittleEndian.PutUint64(enc[:], lvptr) - hasher.Write(enc[:]) - headPtr = lvptr - } - headMap := uint32(headPtr >> params.logValuesPerMap) - var enc [12]byte - for r := uint32(0); r < params.mapHeight; r++ { - binary.LittleEndian.PutUint32(enc[:4], r) - for m := uint32(0); m <= headMap; m++ { - binary.LittleEndian.PutUint32(enc[4:8], m) - rows, _ := mb.GetFilterMapRows(ctx, []uint32{m}, r, false) - for _, row := range rows { - for _, v := range row { - binary.LittleEndian.PutUint32(enc[8:], v) - hasher.Write(enc[:]) - } - } - } - } - var hash common.Hash - hasher.Sum(hash[:0]) - for i := 0; i < 50; i++ { - hasher.Reset() - hasher.Write(hash[:]) - lvptr := binary.LittleEndian.Uint64(hash[:8]) % headPtr - if log, _ := mb.GetLogByLvIndex(ctx, lvptr); log != nil { - enc, err := rlp.EncodeToBytes(log) - if err != nil { - panic(err) - } - hasher.Write(enc) - } - hasher.Sum(hash[:0]) - } - return hash -} - -func (ts *testSetup) close() { - if ts.fm != nil { - ts.fm.Stop() - } - ts.db.Close() - ts.chain.db.Close() -} - -type testChain struct { - ts *testSetup - db ethdb.Database - lock sync.RWMutex - canonical []common.Hash - blocks map[common.Hash]*types.Block - receipts map[common.Hash]types.Receipts -} - -func (ts *testSetup) newTestChain() *testChain { - return &testChain{ - ts: ts, - blocks: make(map[common.Hash]*types.Block), - receipts: make(map[common.Hash]types.Receipts), - } -} - -func (tc *testChain) CurrentBlock() *types.Header { - tc.lock.RLock() - defer tc.lock.RUnlock() - - if len(tc.canonical) == 0 { - return nil - } - return tc.blocks[tc.canonical[len(tc.canonical)-1]].Header() -} - -func (tc *testChain) GetHeader(hash common.Hash, number uint64) *types.Header { - tc.lock.RLock() - defer tc.lock.RUnlock() - - if block := tc.blocks[hash]; block != nil { - return block.Header() - } - return nil -} - -func (tc *testChain) GetCanonicalHash(number uint64) common.Hash { - tc.lock.RLock() - defer tc.lock.RUnlock() - - if uint64(len(tc.canonical)) <= number { - return common.Hash{} - } - return tc.canonical[number] -} - -func (tc *testChain) GetReceiptsByHash(hash common.Hash) types.Receipts { - tc.lock.RLock() - defer tc.lock.RUnlock() - - return tc.receipts[hash] -} - -func (tc *testChain) GetRawReceipts(hash common.Hash, number uint64) types.Receipts { - tc.lock.RLock() - defer tc.lock.RUnlock() - - return tc.receipts[hash] -} - -func (tc *testChain) addBlocks(count, maxTxPerBlock, maxLogsPerReceipt, maxTopicsPerLog int, random bool) { - tc.lock.Lock() - blockGen := func(i int, gen *core.BlockGen) { - var txCount int - if random { - txCount = rand.Intn(maxTxPerBlock + 1) - } else { - txCount = maxTxPerBlock - } - for k := txCount; k > 0; k-- { - receipt := types.NewReceipt(nil, false, 0) - var logCount int - if random { - logCount = rand.Intn(maxLogsPerReceipt + 1) - } else { - logCount = maxLogsPerReceipt - } - receipt.Logs = make([]*types.Log, logCount) - for i := range receipt.Logs { - log := &types.Log{} - receipt.Logs[i] = log - crand.Read(log.Address[:]) - var topicCount int - if random { - topicCount = rand.Intn(maxTopicsPerLog + 1) - } else { - topicCount = maxTopicsPerLog - } - log.Topics = make([]common.Hash, topicCount) - for j := range log.Topics { - crand.Read(log.Topics[j][:]) - } - } - gen.AddUncheckedReceipt(receipt) - gen.AddUncheckedTx(types.NewTransaction(999, common.HexToAddress("0x999"), big.NewInt(999), 999, gen.BaseFee(), nil)) - } - } - - var ( - blocks []*types.Block - receipts []types.Receipts - engine = ethash.NewFaker() - ) - - if len(tc.canonical) == 0 { - gspec := &core.Genesis{ - Alloc: types.GenesisAlloc{}, - BaseFee: big.NewInt(params.InitialBaseFee), - Config: params.TestChainConfig, - } - tc.db, blocks, receipts = core.GenerateChainWithGenesis(gspec, engine, count, blockGen) - gblock := gspec.ToBlock() - ghash := gblock.Hash() - tc.canonical = []common.Hash{ghash} - tc.blocks[ghash] = gblock - tc.receipts[ghash] = types.Receipts{} - } else { - blocks, receipts = core.GenerateChain(params.TestChainConfig, tc.blocks[tc.canonical[len(tc.canonical)-1]], engine, tc.db, count, blockGen) - } - - for i, block := range blocks { - num, hash := int(block.NumberU64()), block.Hash() - if len(tc.canonical) != num { - panic("canonical chain length mismatch") - } - tc.canonical = append(tc.canonical, hash) - tc.blocks[hash] = block - if receipts[i] != nil { - tc.receipts[hash] = receipts[i] - } else { - tc.receipts[hash] = types.Receipts{} - } - } - tc.lock.Unlock() - tc.setTargetHead() -} - -func (tc *testChain) setHead(headNum int) { - tc.lock.Lock() - tc.canonical = tc.canonical[:headNum+1] - tc.lock.Unlock() - tc.setTargetHead() -} - -func (tc *testChain) setTargetHead() { - head := tc.CurrentBlock() - if tc.ts.fm != nil { - if !tc.ts.fm.disabled { - //tc.ts.fm.targetViewCh <- NewChainView(tc, head.Number.Uint64(), head.Hash()) - tc.ts.fm.SetTarget(NewChainView(tc, head.Number.Uint64(), head.Hash()), 0, 0) - } - } -} - -func (tc *testChain) getCanonicalChain() []common.Hash { - tc.lock.RLock() - defer tc.lock.RUnlock() - - cc := make([]common.Hash, len(tc.canonical)) - copy(cc, tc.canonical) - return cc -} - -// restore an earlier state of the chain -func (tc *testChain) setCanonicalChain(cc []common.Hash) { - tc.lock.Lock() - tc.canonical = make([]common.Hash, len(cc)) - copy(tc.canonical, cc) - tc.lock.Unlock() - tc.setTargetHead() -} diff --git a/core/filtermaps/map_database.go b/core/filtermaps/map_database.go new file mode 100644 index 0000000000..a16a1fda90 --- /dev/null +++ b/core/filtermaps/map_database.go @@ -0,0 +1,575 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package filtermaps + +import ( + "errors" + "fmt" + "math" + "sort" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/lru" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" +) + +const ( + databaseVersion = 4 // reindexed if database version does not match + cachedLastBlocks = 1000 // last block of map pointers + cachedLvPointers = 1000 // first log value pointer of block pointers + maxWritesPerBatch = 1000000 +) + +// mapDatabase implements the database storage layer for filter maps and the +// belonging pointers. +type mapDatabase struct { + params *Params + db ethdb.KeyValueStore + hashScheme bool + + lastBlockCache *lru.Cache[uint32, lastBlockOfMap] + lvPointerCache *lru.Cache[uint64, uint64] + + testReadRows bool +} + +// lastBlockOfMap is used for storing and caching the (number, hash) pairs +// belonging to the last block of each map. +type lastBlockOfMap struct { + number uint64 + hash common.Hash +} + +// newMapDatabase created a new mapDatabase layer. +func newMapDatabase(params *Params, db ethdb.KeyValueStore, hashScheme bool) *mapDatabase { + return &mapDatabase{ + params: params, + db: db, + hashScheme: hashScheme, + lastBlockCache: lru.NewCache[uint32, lastBlockOfMap](cachedLastBlocks), + lvPointerCache: lru.NewCache[uint64, uint64](cachedLvPointers), + } +} + +// loadMapRange loads the valid and dirty map ranges and known epoch boundaries +// from the database. +func (m *mapDatabase) loadMapRange() (valid, dirty rangeSet[uint32], knownEpochs uint32, found bool) { + fmr, ok, err := rawdb.ReadFilterMapsRange(m.db) + if !ok || err != nil || fmr.Version != databaseVersion { + return + } + return decodeRangeSet32(fmr.ValidMaps), decodeRangeSet32(fmr.DirtyMaps), fmr.KnownEpochs, true +} + +// storeMapRange stores the valid and dirty map ranges and known epoch boundaries +// into the database. +func (m *mapDatabase) storeMapRange(valid, dirty rangeSet[uint32], knownEpochs uint32) { + rawdb.WriteFilterMapsRange(m.db, rawdb.FilterMapsRange{ + Version: databaseVersion, + ValidMaps: encodeRangeSet32(valid), + DirtyMaps: encodeRangeSet32(dirty), + KnownEpochs: knownEpochs, + }) +} + +// deleteMapRange removes the map range entry from the database, signaling a +// revert to the non-initialized state. +func (m *mapDatabase) deleteMapRange() { + rawdb.DeleteFilterMapsRange(m.db) +} + +func decodeRangeSet32(enc []uint32) rangeSet[uint32] { + rs := make(rangeSet[uint32], len(enc)/2) + for i := range rs { + rs[i] = common.NewRange(enc[i*2], enc[i*2+1]) + } + return rs +} + +func encodeRangeSet32(rs rangeSet[uint32]) []uint32 { + enc := make([]uint32, len(rs)*2) + for i, r := range rs { + enc[i*2] = r.First() + enc[i*2+1] = r.Count() + } + return enc +} + +// getBlockLvPointer returns the starting log value index where the log values +// generated by the given block are located. +func (m *mapDatabase) getBlockLvPointer(blockNumber uint64) (uint64, error) { + if lvPointer, ok := m.lvPointerCache.Get(blockNumber); ok { + return lvPointer, nil + } + lvPointer, err := rawdb.ReadBlockLvPointer(m.db, blockNumber) + if err != nil { + return 0, fmt.Errorf("failed to retrieve log value pointer of block %d: %v", blockNumber, err) + } + m.lvPointerCache.Add(blockNumber, lvPointer) + return lvPointer, nil +} + +// storeBlockLvPointer stores the starting log value index where the log values +// generated by the given block are located. +func (m *mapDatabase) storeBlockLvPointer(db ethdb.KeyValueWriter, blockNumber, lvPointer uint64) { + m.lvPointerCache.Add(blockNumber, lvPointer) + rawdb.WriteBlockLvPointer(db, blockNumber, lvPointer) +} + +// getLastBlockOfMap returns the number and hash of the block that generated the +// last log value entry of the given map. +func (m *mapDatabase) getLastBlockOfMap(mapIndex uint32) (uint64, common.Hash, error) { + if lastBlock, ok := m.lastBlockCache.Get(mapIndex); ok { + return lastBlock.number, lastBlock.hash, nil + } + number, hash, err := rawdb.ReadFilterMapLastBlock(m.db, mapIndex) + if err != nil { + return 0, common.Hash{}, fmt.Errorf("failed to retrieve last block of map %d: %v", mapIndex, err) + } + m.lastBlockCache.Add(mapIndex, lastBlockOfMap{number: number, hash: hash}) + return number, hash, nil +} + +// storeLastBlockOfMap stores the number of the block that generated the last +// log value entry of the given map. +func (m *mapDatabase) storeLastBlockOfMap(db ethdb.KeyValueWriter, mapIndex uint32, number uint64, hash common.Hash) { + m.lastBlockCache.Add(mapIndex, lastBlockOfMap{number: number, hash: hash}) + rawdb.WriteFilterMapLastBlock(db, mapIndex, number, hash) +} + +// deleteEpochRows deletes the map rows from an entire epoch. +func (m *mapDatabase) deleteEpochRows(epoch uint32, stopCallback func() bool) (bool, error) { + deleteFn := func(db ethdb.KeyValueStore, hashScheme bool, stopCb func(bool) bool) error { + first := m.params.mapRowIndex(m.params.firstEpochMap(epoch), 0) + afterLast := m.params.mapRowIndex(m.params.firstEpochMap(epoch+1), 0) + return rawdb.DeleteFilterMapRows(db, common.NewRange(first, afterLast-first), hashScheme, stopCb) + } + action := fmt.Sprintf("Deleting epoch #%d", epoch) + switch err := m.safeDeleteWithLogs(deleteFn, action, stopCallback); err { + case nil: + return true, nil + case rawdb.ErrDeleteRangeInterrupted: + return false, nil + default: + return false, err + } +} + +// reset deletes the entire log index database. Note that deleteMapRange should +// be used first in order to prevent trying to use a partially deleted database +// later. +func (m *mapDatabase) reset(stopCallback func() bool) (bool, error) { + rawdb.DeleteFilterMapsRange(m.db) + if err := m.safeDeleteWithLogs(rawdb.DeleteFilterMapsDb, "Resetting log index database", stopCallback); err != rawdb.ErrDeleteRangeInterrupted { + return err == nil, err + } + return false, nil +} + +// deletePointers removes the last block of map and block log value pointers +// belonging to the specified map range. +// Note that in order to determine the relevant block range, the last block +// pointer of the map before the specified map range has to be present unless +// the range starts at map 0. The last block pointer of the last map in the range +// also has to be present unless the range ends at MaxUint32-1. +func (m *mapDatabase) deletePointers(deleteMaps common.Range[uint32], stopCallback func() bool) error { + var firstBlock, afterLastBlock uint64 + if deleteMaps.First() > 0 { + lb, _, err := m.getLastBlockOfMap(deleteMaps.First() - 1) + if err != nil { + return err + } + firstBlock = lb + 1 + } + if deleteMaps.AfterLast() < math.MaxUint32 { + lb, _, err := m.getLastBlockOfMap(deleteMaps.Last()) + if err != nil { + return err + } + afterLastBlock = lb + } else { + afterLastBlock = math.MaxUint64 + } + if afterLastBlock > firstBlock { + m.lvPointerCache.Purge() + blockRange := common.NewRange[uint64](firstBlock, afterLastBlock-firstBlock) // keep last pointer + if err := rawdb.DeleteBlockLvPointers(m.db, blockRange, m.hashScheme, func(bool) bool { return stopCallback() }); err != nil { + return err + } + } + if deleteMaps.Count() > 1 { + m.lastBlockCache.Purge() + mapRange := common.NewRange[uint32](deleteMaps.First(), deleteMaps.Count()-1) // keep last pointer + if err := rawdb.DeleteFilterMapLastBlocks(m.db, mapRange, m.hashScheme, func(bool) bool { return stopCallback() }); err != nil { + return err + } + } + return nil +} + +// writePointers writes the pointers belonging to the give maps to the database. +func (m *mapDatabase) writePointers(writeRange common.Range[uint32], maps []*finishedMap, stopCallback func() bool) (bool, error) { + batch := m.db.NewBatch() + var ( + batchCnt uint32 + writeErr error + ) + checkStopOrCommit := func() bool { + batchCnt++ + if batchCnt >= maxWritesPerBatch { + if writeErr = batch.Write(); writeErr != nil { + return true + } + if stopCallback() { + return true + } + batch = m.db.NewBatch() + batchCnt = 0 + } + return false + } + for mapIndex := range writeRange.Iter() { + fm := maps[mapIndex-writeRange.First()] + m.storeLastBlockOfMap(batch, mapIndex, fm.lastBlock.number, fm.lastBlock.hash) + if checkStopOrCommit() { + return false, writeErr + } + for i, lvPtr := range fm.blockPtrs { + m.storeBlockLvPointer(batch, fm.firstBlock()+uint64(i), lvPtr) + if checkStopOrCommit() { + return false, writeErr + } + } + } + if err := batch.Write(); err != nil { + return false, err + } + return true, nil +} + +// getFilterMapRows returns a batch of filter maps rows from the same row index, +// each truncated to the length limit of the specified mapping layer. +// The function assumes that the map indices are in strictly ascending order. +// Note that the function modifies the mapIndices slice. +func (m *mapDatabase) getFilterMapRows(mapIndices []uint32, rowIndex, layerIndex uint32) ([]FilterRow, error) { + rows := make([]FilterRow, len(mapIndices)) + readRows := make([]*FilterRow, len(mapIndices)) + for i := range rows { + readRows[i] = &rows[i] + } + for dbLayer := range min(layerIndex+1, uint32(len(m.params.rowGroupSize))) { + if err := m.getFilterMapRowsOfDbLayer(mapIndices, rowIndex, dbLayer, readRows); err != nil { + return nil, err + } + j := 0 + maxRowLength := m.params.maxRowLength[dbLayer] + for i, row := range readRows { + if uint32(len(*row)) == maxRowLength { + if j != i { + mapIndices[j] = mapIndices[i] + readRows[j] = readRows[i] + } + j++ + } + } + if j == 0 { + break + } + mapIndices = mapIndices[:j] + readRows = readRows[:j] + } + return rows, nil +} + +// getFilterMapRowsOfDbLayer loads the part of the given filter rows belonging to +// the specified database layer and appends it to a slice of already existing +// rows. If dbLayer > 0 then it is assumed that the data from all previous database +// layers have been appended to the filter rows. +func (m *mapDatabase) getFilterMapRowsOfDbLayer(mapIndices []uint32, rowIndex, dbLayer uint32, appendTo []*FilterRow) error { + var ptr int + for ptr < len(mapIndices) { + var ( + groupIndex = m.params.mapGroupIndex(mapIndices[ptr], dbLayer) + groupLength = 1 + ) + for ptr+groupLength < len(mapIndices) && m.params.mapGroupIndex(mapIndices[ptr+groupLength], dbLayer) == groupIndex { + groupLength++ + } + if err := m.getFilterMapRowsOfDbLayerGroup(mapIndices[ptr:ptr+groupLength], rowIndex, dbLayer, appendTo[ptr:ptr+groupLength]); err != nil { + return err + } + ptr += groupLength + } + return nil +} + +// getFilterMapRowsOfDbLayerGroup loads the part of the given filter rows belonging to +// the specified database layer and appends it to a slice of already existing +// rows. The map indices should be located in the same row group according to the +// group size at the given database layer. +func (m *mapDatabase) getFilterMapRowsOfDbLayerGroup(mapIndices []uint32, rowIndex, dbLayer uint32, appendTo []*FilterRow) error { + var ( + groupSize = m.params.rowGroupSize[dbLayer] + groupIndex = m.params.mapGroupIndex(mapIndices[0], dbLayer) + mapRowIndex = m.params.mapRowIndex(groupIndex, rowIndex) + ) + if groupSize == 1 { + row, err := rawdb.ReadFilterMapSingleRow(m.db, mapRowIndex, dbLayer, m.params.logMapWidth) + if err != nil { + return fmt.Errorf("failed to retrieve row %d of row %d layer %d: %v", groupIndex, rowIndex, dbLayer, err) + } + *appendTo[0] = append(*appendTo[0], FilterRow(row)...) + return nil + } + rows, err := rawdb.ReadFilterMapRowGroup(m.db, mapRowIndex, dbLayer, groupSize, m.params.logMapWidth) + if err != nil { + return fmt.Errorf("failed to retrieve row group %d of row %d layer %d: %v", groupIndex, rowIndex, dbLayer, err) + } + for i, mapIndex := range mapIndices { + if m.params.mapGroupIndex(mapIndex, dbLayer) != groupIndex { + return fmt.Errorf("maps are not in the same base row group, index: %d, group: %d", mapIndex, groupIndex) + } + *appendTo[i] = append(*appendTo[i], FilterRow(rows[m.params.mapGroupOffset(mapIndex, dbLayer)])...) + } + return nil +} + +// getFilterMap returns the filter map at the specified index. +func (m *mapDatabase) getFilterMap(mapIndex uint32) (*finishedMap, error) { + fm := new(finishedMap) + lastBlock, lbHash, err := m.getLastBlockOfMap(mapIndex) + if err != nil { + return nil, fmt.Errorf("failed to retrieve last block of map %d: %v", mapIndex, err) + } + fm.lastBlock = lastBlockOfMap{number: lastBlock, hash: lbHash} + var firstBlock uint64 + if mapIndex > 0 { + plb, _, err := m.getLastBlockOfMap(mapIndex - 1) + if err != nil { + return nil, fmt.Errorf("failed to retrieve last block of map %d: %v", mapIndex-1, err) + } + firstBlock = plb + 1 + } + fm.blockPtrs = make([]uint64, lastBlock+1-firstBlock) + for i := range fm.blockPtrs { + blockPtr, err := m.getBlockLvPointer(firstBlock + uint64(i)) + if err != nil { + return nil, fmt.Errorf("failed to retrieve block pointer %d: %v", firstBlock+uint64(i), err) + } + fm.blockPtrs[i] = blockPtr + } + for rowIndex := range m.params.mapHeight { + var mi [1]uint32 + mi[0] = mapIndex + row, err := m.getFilterMapRows(mi[:], rowIndex, uint32(len(m.params.rowGroupSize)-1)) + if err != nil { + return nil, fmt.Errorf("failed to retrieve row %d of map %d: %v", rowIndex, mapIndex, err) + } + fm.rowData = append(fm.rowData, row[0]...) + fm.rowPtrs = append(fm.rowPtrs, uint16(len(fm.rowData))) + } + return fm, nil +} + +// writeMapRows updates a section of the filter map range, writing a continuous +// range of new map data and/or removing dirty data. writeRange and deleteRange +// can and should overlap if dirty data is being overwritten with new map data. +// writeRange alone should only be specified where the database is known to be +// empty. +// keepEmptyRange is optional, it signals that a certain range is known to be +// empty in the database and should also stay so. It should not overlap with the +// other two ranges. If these three ranges together cover an entire row group +// then the entire group can be updated without reading its previous value, +// thereby speeding up the write process significantly. +// Note that writeMapRows only updates the filter map rows and does not touch +// the belonging pointers. +func (m *mapDatabase) writeMapRows(writeRange, deleteRange, keepEmptyRange common.Range[uint32], maps []*finishedMap, stopCallback func() bool) (bool, error) { + if !writeRange.Intersection(keepEmptyRange).IsEmpty() || !deleteRange.Intersection(keepEmptyRange).IsEmpty() { + panic("invalid writeMapRows map ranges") + } + writePattern := m.makeWritePattern(writeRange, deleteRange, keepEmptyRange) + batch := m.db.NewBatch() + rowsPerBatch := uint32(max(maxWritesPerBatch/len(writePattern), 1)) + var ( + batchCnt uint32 + writeErr error + ) + checkStopOrCommit := func() bool { + batchCnt++ + if batchCnt >= rowsPerBatch { + if writeErr = batch.Write(); writeErr != nil { + return true + } + if stopCallback() { + return true + } + batch = m.db.NewBatch() + batchCnt = 0 + } + return false + } + + for rowIndex := range m.params.mapHeight { + if err := m.writeRowUpdates(batch, writePattern, writeRange, maps, rowIndex); err != nil { + return false, err + } + if checkStopOrCommit() { + return false, writeErr + } + } + if err := batch.Write(); err != nil { + return false, err + } + return true, nil +} + +type writePatterItem struct { + mapIndex, dbLayer uint32 + writeRange, deleteRange, keepEmptyRange common.Range[uint32] +} + +// makeWritePattern pre-generates the order of operations that need to be repeated +// for every row. +func (m *mapDatabase) makeWritePattern(writeRange, deleteRange, keepEmptyRange common.Range[uint32]) (writePattern []writePatterItem) { + for dbLayer, groupSize := range m.params.rowGroupSize { + updateRange := deleteRange.Union(writeRange) + firstGroup, lastGroup := updateRange.First()/groupSize, updateRange.Last()/groupSize + for i := firstGroup; i <= lastGroup; i++ { + groupRange := common.NewRange[uint32](i*groupSize, groupSize) + writePattern = append(writePattern, writePatterItem{ + mapIndex: i * groupSize, + dbLayer: uint32(dbLayer), + writeRange: writeRange.Intersection(groupRange), + deleteRange: deleteRange.Intersection(groupRange), + keepEmptyRange: keepEmptyRange.Intersection(groupRange), + }) + } + } + sort.Slice(writePattern, func(i, j int) bool { + return writePattern[i].mapIndex < writePattern[j].mapIndex || + (writePattern[i].mapIndex == writePattern[j].mapIndex && writePattern[i].dbLayer < writePattern[j].dbLayer) + }) + return +} + +// writeRowUpdates performs the filter row update operation on a single row. +func (m *mapDatabase) writeRowUpdates(batch ethdb.Batch, writePattern []writePatterItem, writeRange common.Range[uint32], maps []*finishedMap, rowIndex uint32) error { + for _, w := range writePattern { + if groupSize := m.params.rowGroupSize[w.dbLayer]; groupSize == 1 { + var row FilterRow + if writeRange.Includes(w.mapIndex) { + row = maps[w.mapIndex-writeRange.First()].getRow(rowIndex, m.params.maxRowLength[w.dbLayer]) + } + var from uint32 + if w.dbLayer > 0 { + from = m.params.maxRowLength[w.dbLayer-1] + } + if uint32(len(row)) > from { + row = row[from:] + } else { + row = nil + } + if len(row) > 0 || !w.deleteRange.IsEmpty() { + rawdb.WriteFilterMapSingleRow(batch, m.params.mapRowIndex(w.mapIndex, rowIndex), w.dbLayer, row, m.params.logMapWidth) + } + } else { + rows := make([][]uint32, groupSize) + writeGroup := !w.deleteRange.IsEmpty() + if w.writeRange.Count()+w.deleteRange.Count()-w.writeRange.Intersection(w.deleteRange).Count()+w.keepEmptyRange.Count() < groupSize { + oldRows, err := rawdb.ReadFilterMapRowGroup(m.db, m.params.mapRowIndex(w.mapIndex, rowIndex), w.dbLayer, groupSize, m.params.logMapWidth) + m.testReadRows = true + if err != nil { + return err + } + for i := range groupSize { + if mapIndex := w.mapIndex + i; !w.writeRange.Includes(mapIndex) && !w.deleteRange.Includes(mapIndex) && !w.keepEmptyRange.Includes(mapIndex) { + rows[i] = oldRows[i] + } + } + } + var from uint32 + if w.dbLayer > 0 { + from = m.params.maxRowLength[w.dbLayer-1] + } + to := m.params.maxRowLength[w.dbLayer] + for i := range groupSize { + if writeRange.Includes(w.mapIndex + i) { + if row := maps[w.mapIndex+i-writeRange.First()].getRow(rowIndex, to); uint32(len(row)) > from { + rows[i] = row[from:] + writeGroup = true + } + } + } + if writeGroup { + rawdb.WriteFilterMapRowGroup(batch, m.params.mapRowIndex(w.mapIndex, rowIndex), w.dbLayer, rows, m.params.logMapWidth) + } + } + } + return nil +} + +// safeDeleteWithLogs is a wrapper for a function that performs a safe range +// delete operation using rawdb.SafeDeleteRange. It emits log messages if the +// process takes long enough to call the stop callback. +func (m *mapDatabase) safeDeleteWithLogs(deleteFn func(db ethdb.KeyValueStore, hashScheme bool, stopCb func(bool) bool) error, action string, stopCb func() bool) error { + var ( + start = time.Now() + logPrinted bool + lastLogPrinted = start + ) + switch err := deleteFn(m.db, m.hashScheme, func(deleted bool) bool { + if deleted && !logPrinted || time.Since(lastLogPrinted) > time.Second*10 { + log.Info(action+" in progress...", "elapsed", common.PrettyDuration(time.Since(start))) + logPrinted, lastLogPrinted = true, time.Now() + } + return stopCb() + }); { + case err == nil: + if logPrinted { + log.Info(action+" finished", "elapsed", common.PrettyDuration(time.Since(start))) + } + return nil + case errors.Is(err, rawdb.ErrDeleteRangeInterrupted): + log.Warn(action+" interrupted", "elapsed", common.PrettyDuration(time.Since(start))) + return err + default: + log.Error(action+" failed", "error", err) + return err + } +} + +// removeBloomBits removes old bloom bits data from the database. +func (m *mapDatabase) removeBloomBits(stopCb func() bool) { + m.safeDeleteWithLogs(rawdb.DeleteBloomBitsDb, "Removing old bloom bits database", stopCb) +} + +// storeEpochCheckpoint writes a single checkpoint (known epoch boundary) to the +// database. +func (m *mapDatabase) storeEpochCheckpoint(epoch uint32, cp epochCheckpoint) { + m.storeLastBlockOfMap(m.db, m.params.lastEpochMap(epoch), cp.BlockNumber, cp.BlockHash) + m.storeBlockLvPointer(m.db, cp.BlockNumber, cp.FirstIndex) +} + +// storeCheckpointList writes a list of checkpoints to the database. +func (m *mapDatabase) storeCheckpointList(firstEpoch uint32, cpList checkpointList) { + for i, cp := range cpList { + m.storeEpochCheckpoint(firstEpoch+uint32(i), cp) + } +} diff --git a/core/filtermaps/map_database_test.go b/core/filtermaps/map_database_test.go new file mode 100644 index 0000000000..e0ce431769 --- /dev/null +++ b/core/filtermaps/map_database_test.go @@ -0,0 +1,79 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package filtermaps + +import ( + "math" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb/memorydb" +) + +var testParams = Params{ + logMapHeight: 8, + logMapWidth: 24, + logMapsPerEpoch: 6, + logValuesPerMap: 8, + logMappingFrequency: []uint{6, 4, 2, 0}, + maxRowLength: []uint32{4, 16, 64, 256}, + rowGroupSize: []uint32{16, 4, 1, 1}, +} + +func testStop() bool { return false } + +func TestMapDatabase(t *testing.T) { + params := testParams + params.sanitize() + db := memorydb.New() + mapDb := newMapDatabase(¶ms, db, false) + reader := mapReader{ + getFilterMapRows: mapDb.getFilterMapRows, + getFilterMap: mapDb.getFilterMap, + getBlockLvPointer: mapDb.getBlockLvPointer, + getLastBlockOfMap: mapDb.getLastBlockOfMap, + } + // initialize database with checkpoints + maps := generateTestMaps(¶ms, nil, 0x200) + cpList := generateTestCheckpoints(¶ms, maps) + mapDb.storeCheckpointList(0, cpList) + // add new maps to the head + maps = generateTestMaps(¶ms, maps, 0x50) + mapDb.writeMapRows(common.NewRange[uint32](0x200, 0x50), common.Range[uint32]{}, common.NewRange[uint32](0x250, math.MaxUint32-0x250), maps[0x200:], testStop) + mapDb.writePointers(common.NewRange[uint32](0x200, 0x50), maps[0x200:], testStop) + testMapReader(t, "mapDatabase test #1", ¶ms, reader, cpList, maps[0x200:]) + // backfill previous epoch with a single write + mapDb.writeMapRows(common.NewRange[uint32](0x1c0, 0x40), common.Range[uint32]{}, common.Range[uint32]{}, maps[0x1c0:0x200], testStop) + mapDb.writePointers(common.NewRange[uint32](0x1c0, 0x40), maps[0x1c0:0x200], testStop) + testMapReader(t, "mapDatabase test #2", ¶ms, reader, cpList[:7], maps[0x1c0:]) + // backfill previous epoch in two steps + mapDb.writeMapRows(common.NewRange[uint32](0x180, 0x10), common.Range[uint32]{}, common.NewRange[uint32](0x190, 0x30), maps[0x180:0x190], testStop) + mapDb.writePointers(common.NewRange[uint32](0x180, 0x10), maps[0x180:0x190], testStop) + mapDb.writeMapRows(common.NewRange[uint32](0x190, 0x30), common.Range[uint32]{}, common.Range[uint32]{}, maps[0x190:0x1c0], testStop) + mapDb.writePointers(common.NewRange[uint32](0x190, 0x30), maps[0x190:0x1c0], testStop) + testMapReader(t, "mapDatabase test #3", ¶ms, reader, cpList[:6], maps[0x180:]) + // add new maps while reorging some existing ones + maps = generateTestMaps(¶ms, maps[:0x230], 0x30) + mapDb.writeMapRows(common.NewRange[uint32](0x230, 0x30), common.NewRange[uint32](0x230, 0x30), common.NewRange[uint32](0x260, math.MaxUint32-0x260), maps[0x230:], testStop) + mapDb.deletePointers(common.NewRange[uint32](0x230, math.MaxUint32-0x230), testStop) + mapDb.writePointers(common.NewRange[uint32](0x230, 0x30), maps[0x230:], testStop) + testMapReader(t, "mapDatabase test #4", ¶ms, reader, cpList[:6], maps[0x180:]) + // unindex tail epoch + mapDb.writeMapRows(common.Range[uint32]{}, common.NewRange[uint32](0x180, 0x40), common.Range[uint32]{}, nil, testStop) + mapDb.deletePointers(common.NewRange[uint32](0x180, 0x40), testStop) + testMapReader(t, "mapDatabase test #5", ¶ms, reader, cpList[:7], maps[0x1c0:]) +} diff --git a/core/filtermaps/map_renderer.go b/core/filtermaps/map_renderer.go deleted file mode 100644 index e1284a3829..0000000000 --- a/core/filtermaps/map_renderer.go +++ /dev/null @@ -1,815 +0,0 @@ -// Copyright 2024 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package filtermaps - -import ( - "errors" - "fmt" - "math" - "slices" - "sort" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/lru" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/log" -) - -const ( - maxMapsPerBatch = 32 // maximum number of maps rendered in memory - valuesPerCallback = 1024 // log values processed per event process callback - cachedRowMappings = 10000 // log value to row mappings cached during rendering - - // Number of rows written to db in a single batch. - // The map renderer splits up writes like this to ensure that regular - // block processing latency is not affected by large batch writes. - rowsPerBatch = 1024 -) - -var ( - errChainUpdate = errors.New("rendered section of chain updated") -) - -// mapRenderer represents a process that renders filter maps in a specified -// range according to the actual targetView. -type mapRenderer struct { - f *FilterMaps - renderBefore uint32 - currentMap *renderedMap - finishedMaps map[uint32]*renderedMap - finished common.Range[uint32] - iterator *logIterator -} - -// renderedMap represents a single filter map that is being rendered in memory. -type renderedMap struct { - filterMap filterMap - mapIndex uint32 - lastBlock uint64 - lastBlockId common.Hash - blockLvPtrs []uint64 // start pointers of blocks starting in this map; last one is lastBlock - finished bool // iterator finished; all values rendered - headDelimiter uint64 // if finished then points to the future block delimiter of the head block -} - -// firstBlock returns the first block number that starts in the given map. -func (r *renderedMap) firstBlock() uint64 { - return r.lastBlock + 1 - uint64(len(r.blockLvPtrs)) -} - -// renderMapsBefore creates a mapRenderer that renders the log index until the -// specified map index boundary, starting from the latest available starting -// point that is consistent with the current targetView. -// The renderer ensures that filterMapsRange, indexedView and the actual map -// data are always consistent with each other. If renderBefore is greater than -// the latest existing rendered map then indexedView is updated to targetView, -// otherwise it is checked that the rendered range is consistent with both -// views. -func (f *FilterMaps) renderMapsBefore(renderBefore uint32) (*mapRenderer, error) { - nextMap, startBlock, startLvPtr, err := f.lastCanonicalMapBoundaryBefore(renderBefore) - if err != nil { - return nil, err - } - if snapshot := f.lastCanonicalSnapshotOfMap(nextMap); snapshot != nil { - return f.renderMapsFromSnapshot(snapshot) - } - if nextMap >= renderBefore { - return nil, nil - } - return f.renderMapsFromMapBoundary(nextMap, renderBefore, startBlock, startLvPtr) -} - -// renderMapsFromSnapshot creates a mapRenderer that starts rendering from a -// snapshot made at a block boundary. -func (f *FilterMaps) renderMapsFromSnapshot(cp *renderedMap) (*mapRenderer, error) { - f.testSnapshotUsed = true - iter, err := f.newLogIteratorFromBlockDelimiter(cp.lastBlock, cp.headDelimiter) - if err != nil { - return nil, fmt.Errorf("failed to create log iterator from block delimiter %d: %v", cp.lastBlock, err) - } - return &mapRenderer{ - f: f, - currentMap: &renderedMap{ - filterMap: cp.filterMap.fullCopy(), - mapIndex: cp.mapIndex, - lastBlock: cp.lastBlock, - blockLvPtrs: slices.Clone(cp.blockLvPtrs), - }, - finishedMaps: make(map[uint32]*renderedMap), - finished: common.NewRange(cp.mapIndex, 0), - renderBefore: math.MaxUint32, - iterator: iter, - }, nil -} - -// renderMapsFromMapBoundary creates a mapRenderer that starts rendering at a -// map boundary. -func (f *FilterMaps) renderMapsFromMapBoundary(firstMap, renderBefore uint32, startBlock, startLvPtr uint64) (*mapRenderer, error) { - iter, err := f.newLogIteratorFromMapBoundary(firstMap, startBlock, startLvPtr) - if err != nil { - return nil, fmt.Errorf("failed to create log iterator from map boundary %d: %v", firstMap, err) - } - return &mapRenderer{ - f: f, - currentMap: &renderedMap{ - filterMap: f.emptyFilterMap(), - mapIndex: firstMap, - lastBlock: iter.blockNumber, - }, - finishedMaps: make(map[uint32]*renderedMap), - finished: common.NewRange(firstMap, 0), - renderBefore: renderBefore, - iterator: iter, - }, nil -} - -// lastCanonicalSnapshotOfMap returns the latest cached snapshot of the given map -// that is also consistent with the current targetView. -func (f *FilterMaps) lastCanonicalSnapshotOfMap(mapIndex uint32) *renderedMap { - var best *renderedMap - for _, blockNumber := range f.renderSnapshots.Keys() { - if cp, _ := f.renderSnapshots.Get(blockNumber); cp != nil && blockNumber < f.indexedRange.blocks.AfterLast() && - blockNumber <= f.indexedView.HeadNumber() && f.indexedView.BlockId(blockNumber) == cp.lastBlockId && - blockNumber <= f.targetView.HeadNumber() && f.targetView.BlockId(blockNumber) == cp.lastBlockId && - cp.mapIndex == mapIndex && (best == nil || blockNumber > best.lastBlock) { - best = cp - } - } - return best -} - -// lastCanonicalMapBoundaryBefore returns the latest map boundary before the -// specified map index that matches the current targetView. This can either -// be a checkpoint (hardcoded or left from a previously unindexed tail epoch) -// or the boundary of a currently rendered map. -// Along with the next map index where the rendering can be started, the number -// and starting log value pointer of the last block is also returned. -func (f *FilterMaps) lastCanonicalMapBoundaryBefore(renderBefore uint32) (nextMap uint32, startBlock, startLvPtr uint64, err error) { - if !f.indexedRange.initialized { - return 0, 0, 0, nil - } - mapIndex := renderBefore - for { - var ok bool - if mapIndex, ok = f.lastMapBoundaryBefore(mapIndex); !ok { - return 0, 0, 0, nil - } - lastBlock, lastBlockId, err := f.getLastBlockOfMap(mapIndex) - if err != nil { - return 0, 0, 0, fmt.Errorf("failed to retrieve last block of reverse iterated map %d: %v", mapIndex, err) - } - if (f.indexedRange.headIndexed && mapIndex >= f.indexedRange.maps.Last()) || - lastBlock >= f.targetView.HeadNumber() || lastBlockId != f.targetView.BlockId(lastBlock) { - continue // map is not full or inconsistent with targetView; roll back - } - lvPtr, err := f.getBlockLvPointer(lastBlock) - if err != nil { - return 0, 0, 0, fmt.Errorf("failed to retrieve log value pointer of last canonical boundary block %d: %v", lastBlock, err) - } - return mapIndex + 1, lastBlock, lvPtr, nil - } -} - -// lastMapBoundaryBefore returns the latest map boundary before the specified -// map index. -func (f *FilterMaps) lastMapBoundaryBefore(renderBefore uint32) (uint32, bool) { - if !f.indexedRange.initialized || f.indexedRange.maps.AfterLast() == 0 || renderBefore == 0 { - return 0, false - } - afterLastFullMap := f.indexedRange.maps.AfterLast() - if afterLastFullMap > 0 && f.indexedRange.headIndexed { - afterLastFullMap-- // last map is not full - } - firstRendered := min(renderBefore-1, afterLastFullMap) - if firstRendered == 0 { - return 0, false - } - if firstRendered >= f.indexedRange.maps.First() { - return firstRendered - 1, true - } - if firstRendered+f.mapsPerEpoch > f.indexedRange.maps.First() { - firstRendered = min(firstRendered, f.indexedRange.maps.First()-f.mapsPerEpoch+f.indexedRange.tailPartialEpoch) - } else { - firstRendered = (firstRendered >> f.logMapsPerEpoch) << f.logMapsPerEpoch - } - if firstRendered == 0 { - return 0, false - } - return firstRendered - 1, true -} - -// emptyFilterMap returns an empty filter map. -func (f *FilterMaps) emptyFilterMap() filterMap { - return make(filterMap, f.mapHeight) -} - -// loadHeadSnapshot loads the last rendered map from the database and creates -// a snapshot. -func (f *FilterMaps) loadHeadSnapshot() error { - fm, err := f.getFilterMap(f.indexedRange.maps.Last()) - if err != nil { - return fmt.Errorf("failed to load head snapshot map %d: %v", f.indexedRange.maps.Last(), err) - } - lastBlock, _, err := f.getLastBlockOfMap(f.indexedRange.maps.Last()) - if err != nil { - return fmt.Errorf("failed to retrieve last block of head snapshot map %d: %v", f.indexedRange.maps.Last(), err) - } - var firstBlock uint64 - if f.indexedRange.maps.AfterLast() > 1 { - prevLastBlock, _, err := f.getLastBlockOfMap(f.indexedRange.maps.Last() - 1) - if err != nil { - return fmt.Errorf("failed to retrieve last block of map %d before head snapshot: %v", f.indexedRange.maps.Last()-1, err) - } - firstBlock = prevLastBlock + 1 - } - lvPtrs := make([]uint64, lastBlock+1-firstBlock) - for i := range lvPtrs { - lvPtrs[i], err = f.getBlockLvPointer(firstBlock + uint64(i)) - if err != nil { - return fmt.Errorf("failed to retrieve log value pointer of head snapshot block %d: %v", firstBlock+uint64(i), err) - } - } - f.renderSnapshots.Add(f.indexedRange.blocks.Last(), &renderedMap{ - filterMap: fm.fullCopy(), - mapIndex: f.indexedRange.maps.Last(), - lastBlock: f.indexedRange.blocks.Last(), - lastBlockId: f.indexedView.BlockId(f.indexedRange.blocks.Last()), - blockLvPtrs: lvPtrs, - finished: true, - headDelimiter: f.indexedRange.headDelimiter, - }) - return nil -} - -// makeSnapshot creates a snapshot of the current state of the rendered map. -func (r *mapRenderer) makeSnapshot() { - if r.iterator.blockNumber != r.currentMap.lastBlock || r.iterator.chainView != r.f.targetView { - panic("iterator state inconsistent with current rendered map") - } - r.f.renderSnapshots.Add(r.currentMap.lastBlock, &renderedMap{ - filterMap: r.currentMap.filterMap.fastCopy(), - mapIndex: r.currentMap.mapIndex, - lastBlock: r.currentMap.lastBlock, - lastBlockId: r.iterator.chainView.BlockId(r.currentMap.lastBlock), - blockLvPtrs: r.currentMap.blockLvPtrs, - finished: true, - headDelimiter: r.iterator.lvIndex, - }) -} - -// run does the actual map rendering. It periodically calls the stopCb callback -// and if it returns true the process is interrupted an can be resumed later -// by calling run again. The writeCb callback is called after new maps have -// been written to disk and the index range has been updated accordingly. -func (r *mapRenderer) run(stopCb func() bool, writeCb func()) (bool, error) { - for { - if done, err := r.renderCurrentMap(stopCb); !done { - return done, err // stopped or failed - } - // map finished - r.finishedMaps[r.currentMap.mapIndex] = r.currentMap - r.finished.SetLast(r.finished.AfterLast()) - if len(r.finishedMaps) >= maxMapsPerBatch || r.f.mapGroupOffset(r.finished.AfterLast()) == 0 { - if err := r.writeFinishedMaps(stopCb); err != nil { - return false, err - } - writeCb() - } - if r.finished.AfterLast() == r.renderBefore || r.iterator.finished { - if err := r.writeFinishedMaps(stopCb); err != nil { - return false, err - } - writeCb() - return true, nil - } - r.currentMap = &renderedMap{ - filterMap: r.f.emptyFilterMap(), - mapIndex: r.finished.AfterLast(), - } - } -} - -// renderCurrentMap renders a single map. -func (r *mapRenderer) renderCurrentMap(stopCb func() bool) (bool, error) { - var ( - totalTime time.Duration - logValuesProcessed, blocksProcessed int64 - ) - start := time.Now() - if !r.iterator.updateChainView(r.f.targetView) { - return false, errChainUpdate - } - var waitCnt int - - if r.iterator.lvIndex == 0 { - r.currentMap.blockLvPtrs = []uint64{0} - } - type lvPos struct{ rowIndex, layerIndex uint32 } - rowMappingCache := lru.NewCache[common.Hash, lvPos](cachedRowMappings) - defer rowMappingCache.Purge() - - for r.iterator.lvIndex < uint64(r.currentMap.mapIndex+1)<= valuesPerCallback { - totalTime += time.Since(start) - if stopCb() { - return false, nil - } - start = time.Now() - if !r.iterator.updateChainView(r.f.targetView) { - return false, errChainUpdate - } - waitCnt = 0 - } - if logValue := r.iterator.getValueHash(); logValue != (common.Hash{}) { - lvp, cached := rowMappingCache.Get(logValue) - if !cached { - lvp = lvPos{rowIndex: r.f.rowIndex(r.currentMap.mapIndex, 0, logValue)} - } - for uint32(len(r.currentMap.filterMap[lvp.rowIndex])) >= r.f.maxRowLength(lvp.layerIndex) { - lvp.layerIndex++ - lvp.rowIndex = r.f.rowIndex(r.currentMap.mapIndex, lvp.layerIndex, logValue) - cached = false - } - r.currentMap.filterMap[lvp.rowIndex] = append(r.currentMap.filterMap[lvp.rowIndex], r.f.columnIndex(r.iterator.lvIndex, &logValue)) - if !cached { - rowMappingCache.Add(logValue, lvp) - } - } - if err := r.iterator.next(); err != nil { - return false, fmt.Errorf("failed to advance log iterator at %d while rendering map %d: %v", r.iterator.lvIndex, r.currentMap.mapIndex, err) - } - if !r.iterator.skipToBoundary { - logValuesProcessed++ - r.currentMap.lastBlock = r.iterator.blockNumber - if r.iterator.blockStart { - blocksProcessed++ - r.currentMap.blockLvPtrs = append(r.currentMap.blockLvPtrs, r.iterator.lvIndex) - } - if !r.f.testDisableSnapshots && r.renderBefore >= r.f.indexedRange.maps.AfterLast() && - (r.iterator.delimiter || r.iterator.finished) { - r.makeSnapshot() - } - } - } - if r.iterator.finished { - r.currentMap.finished = true - r.currentMap.headDelimiter = r.iterator.lvIndex - } - r.currentMap.lastBlockId = r.f.targetView.BlockId(r.currentMap.lastBlock) - totalTime += time.Since(start) - mapRenderTimer.Update(totalTime) - mapLogValueMeter.Mark(logValuesProcessed) - mapBlockMeter.Mark(blocksProcessed) - return true, nil -} - -// writeFinishedMaps writes rendered maps to the database and updates -// filterMapsRange and indexedView accordingly. -func (r *mapRenderer) writeFinishedMaps(pauseCb func() bool) error { - var totalTime time.Duration - start := time.Now() - if len(r.finishedMaps) == 0 { - return nil - } - r.f.indexLock.Lock() - defer r.f.indexLock.Unlock() - - oldRange := r.f.indexedRange - tempRange, err := r.getTempRange() - if err != nil { - return fmt.Errorf("failed to get temporary rendered range: %v", err) - } - newRange, err := r.getUpdatedRange() - if err != nil { - return fmt.Errorf("failed to get updated rendered range: %v", err) - } - renderedView := r.f.targetView // pauseCb callback might still change targetView while writing finished maps - - batch := r.f.db.NewBatch() - var writeCnt int - checkWriteCnt := func() { - writeCnt++ - if writeCnt == rowsPerBatch { - writeCnt = 0 - if err := batch.Write(); err != nil { - log.Crit("Error writing log index update batch", "error", err) - } - // do not exit while in partially written state but do allow processing - // events and pausing while block processing is in progress - r.f.indexLock.Unlock() - totalTime += time.Since(start) - pauseCb() - start = time.Now() - r.f.indexLock.Lock() - batch = r.f.db.NewBatch() - } - } - - if tempRange != r.f.indexedRange { - r.f.setRange(batch, r.f.indexedView, tempRange, true) - } - // add or update filter rows - for rowIndex := uint32(0); rowIndex < r.f.mapHeight; rowIndex++ { - var ( - mapIndices []uint32 - rows []FilterRow - ) - for mapIndex := range r.finished.Iter() { - row := r.finishedMaps[mapIndex].filterMap[rowIndex] - if fm, _ := r.f.filterMapCache.Get(mapIndex); fm != nil && row.Equal(fm[rowIndex]) { - continue - } - mapIndices = append(mapIndices, mapIndex) - rows = append(rows, row) - } - if newRange.maps.AfterLast() == r.finished.AfterLast() { // head updated; remove future entries - for mapIndex := r.finished.AfterLast(); mapIndex < oldRange.maps.AfterLast(); mapIndex++ { - if fm, _ := r.f.filterMapCache.Get(mapIndex); fm != nil && len(fm[rowIndex]) == 0 { - continue - } - mapIndices = append(mapIndices, mapIndex) - rows = append(rows, nil) - } - } - if err := r.f.storeFilterMapRows(batch, mapIndices, rowIndex, rows); err != nil { - return fmt.Errorf("failed to store filter maps %v row %d: %v", mapIndices, rowIndex, err) - } - checkWriteCnt() - } - // update filter map cache - if newRange.maps.AfterLast() == r.finished.AfterLast() { - // head updated; cache new head maps and remove future entries - for mapIndex := range r.finished.Iter() { - r.f.filterMapCache.Add(mapIndex, r.finishedMaps[mapIndex].filterMap) - } - for mapIndex := r.finished.AfterLast(); mapIndex < oldRange.maps.AfterLast(); mapIndex++ { - r.f.filterMapCache.Remove(mapIndex) - } - } else { - // head not updated; do not cache maps during tail rendering because we - // need head maps to be available in the cache - for mapIndex := range r.finished.Iter() { - r.f.filterMapCache.Remove(mapIndex) - } - } - var blockNumber uint64 - if r.finished.First() > 0 { - // in order to always ensure continuous block pointers, initialize - // blockNumber based on the last block of the previous map, then verify - // against the first block associated with each rendered map - lastBlock, _, err := r.f.getLastBlockOfMap(r.finished.First() - 1) - if err != nil { - return fmt.Errorf("failed to get last block of previous map %d: %v", r.finished.First()-1, err) - } - blockNumber = lastBlock + 1 - } - // add or update block pointers - for mapIndex := range r.finished.Iter() { - renderedMap := r.finishedMaps[mapIndex] - if blockNumber != renderedMap.firstBlock() { - return fmt.Errorf("non-continuous block numbers in rendered map %d (next expected: %d first rendered: %d)", mapIndex, blockNumber, renderedMap.firstBlock()) - } - r.f.storeLastBlockOfMap(batch, mapIndex, renderedMap.lastBlock, renderedMap.lastBlockId) - checkWriteCnt() - for _, lvPtr := range renderedMap.blockLvPtrs { - r.f.storeBlockLvPointer(batch, blockNumber, lvPtr) - checkWriteCnt() - blockNumber++ - } - } - if newRange.maps.AfterLast() == r.finished.AfterLast() { // head updated; remove future entries - for mapIndex := r.finished.AfterLast(); mapIndex < oldRange.maps.AfterLast(); mapIndex++ { - r.f.deleteLastBlockOfMap(batch, mapIndex) - checkWriteCnt() - } - for ; blockNumber < oldRange.blocks.AfterLast(); blockNumber++ { - r.f.deleteBlockLvPointer(batch, blockNumber) - checkWriteCnt() - } - } - r.finishedMaps = make(map[uint32]*renderedMap) - r.finished.SetFirst(r.finished.AfterLast()) - r.f.setRange(batch, renderedView, newRange, false) - if err := batch.Write(); err != nil { - log.Crit("Error writing log index update batch", "error", err) - } - totalTime += time.Since(start) - mapWriteTimer.Update(totalTime) - return nil -} - -// getTempRange returns a temporary filterMapsRange that is committed to the -// database while the newly rendered maps are partially written. Writing all -// processed maps in a single database batch would be a serious hit on db -// performance so instead safety is ensured by first reverting the valid map -// range to the unchanged region until all new map data is committed. -func (r *mapRenderer) getTempRange() (filterMapsRange, error) { - tempRange := r.f.indexedRange - if err := tempRange.addRenderedRange(r.finished.First(), r.finished.First(), r.renderBefore, r.f.mapsPerEpoch); err != nil { - return filterMapsRange{}, fmt.Errorf("failed to update temporary rendered range: %v", err) - } - if tempRange.maps.First() != r.f.indexedRange.maps.First() { - // first rendered map changed; update first indexed block - if tempRange.maps.First() > 0 { - firstBlock, _, err := r.f.getLastBlockOfMap(tempRange.maps.First() - 1) - if err != nil { - return filterMapsRange{}, fmt.Errorf("failed to retrieve last block of map %d before temporary range: %v", tempRange.maps.First()-1, err) - } - tempRange.blocks.SetFirst(firstBlock + 1) // firstBlock is probably partially rendered - } else { - tempRange.blocks.SetFirst(0) - } - } - if tempRange.maps.AfterLast() != r.f.indexedRange.maps.AfterLast() { - // last rendered map changed; update last indexed block - if !tempRange.maps.IsEmpty() { - lastBlock, _, err := r.f.getLastBlockOfMap(tempRange.maps.Last()) - if err != nil { - return filterMapsRange{}, fmt.Errorf("failed to retrieve last block of map %d at the end of temporary range: %v", tempRange.maps.Last(), err) - } - tempRange.blocks.SetAfterLast(lastBlock) // lastBlock is probably partially rendered - } else { - tempRange.blocks.SetAfterLast(0) - } - tempRange.headIndexed = false - tempRange.headDelimiter = 0 - } - return tempRange, nil -} - -// getUpdatedRange returns the updated filterMapsRange after writing the newly -// rendered maps. -func (r *mapRenderer) getUpdatedRange() (filterMapsRange, error) { - // update filterMapsRange - newRange := r.f.indexedRange - if err := newRange.addRenderedRange(r.finished.First(), r.finished.AfterLast(), r.renderBefore, r.f.mapsPerEpoch); err != nil { - return filterMapsRange{}, fmt.Errorf("failed to update rendered range: %v", err) - } - if newRange.maps.First() != r.f.indexedRange.maps.First() { - // first rendered map changed; update first indexed block - if newRange.maps.First() > 0 { - firstBlock, _, err := r.f.getLastBlockOfMap(newRange.maps.First() - 1) - if err != nil { - return filterMapsRange{}, fmt.Errorf("failed to retrieve last block of map %d before rendered range: %v", newRange.maps.First()-1, err) - } - newRange.blocks.SetFirst(firstBlock + 1) // firstBlock is probably partially rendered - } else { - newRange.blocks.SetFirst(0) - } - } - if newRange.maps.AfterLast() == r.finished.AfterLast() { - // last rendered map changed; update last indexed block and head pointers - lm := r.finishedMaps[r.finished.Last()] - newRange.headIndexed = lm.finished - if lm.finished { - newRange.blocks.SetLast(r.f.targetView.HeadNumber()) - if lm.lastBlock != r.f.targetView.HeadNumber() { - panic("map rendering finished but last block != head block") - } - newRange.headDelimiter = lm.headDelimiter - } else { - newRange.blocks.SetAfterLast(lm.lastBlock) // lastBlock is probably partially rendered - newRange.headDelimiter = 0 - } - } else { - // last rendered map not replaced; ensure that target chain view matches - // indexed chain view on the rendered section - if lastBlock := r.finishedMaps[r.finished.Last()].lastBlock; !matchViews(r.f.indexedView, r.f.targetView, lastBlock) { - return filterMapsRange{}, errChainUpdate - } - } - return newRange, nil -} - -// addRenderedRange adds the range [firstRendered, afterLastRendered) and -// removes [afterLastRendered, afterLastRemoved) from the set of rendered maps. -func (fmr *filterMapsRange) addRenderedRange(firstRendered, afterLastRendered, afterLastRemoved, mapsPerEpoch uint32) error { - if !fmr.initialized { - return errors.New("log index not initialized") - } - - // Here we create a slice of endpoints for the rendered sections. There are two endpoints - // for each section: the index of the first map, and the index after the last map in the - // section. We then iterate the endpoints -- adding d values -- to determine whether the - // sections are contiguous or whether they have a gap. - type endpoint struct { - m uint32 - d int - } - endpoints := []endpoint{{fmr.maps.First(), 1}, {fmr.maps.AfterLast(), -1}, {firstRendered, 1}, {afterLastRendered, -101}, {afterLastRemoved, 100}} - if fmr.tailPartialEpoch > 0 { - endpoints = append(endpoints, []endpoint{{fmr.maps.First() - mapsPerEpoch, 1}, {fmr.maps.First() - mapsPerEpoch + fmr.tailPartialEpoch, -1}}...) - } - sort.Slice(endpoints, func(i, j int) bool { return endpoints[i].m < endpoints[j].m }) - var ( - sum int - merged []uint32 - last bool - ) - for i, e := range endpoints { - sum += e.d - if i < len(endpoints)-1 && endpoints[i+1].m == e.m { - continue - } - if (sum > 0) != last { - merged = append(merged, e.m) - last = !last - } - } - - switch len(merged) { - case 0: - // Initialized database, but no finished maps yet. - fmr.tailPartialEpoch = 0 - fmr.maps = common.NewRange(firstRendered, 0) - - case 2: - // One rendered section (no partial tail epoch). - fmr.tailPartialEpoch = 0 - fmr.maps = common.NewRange(merged[0], merged[1]-merged[0]) - - case 4: - // Two rendered sections (with a gap). - // First section (merged[0]-merged[1]) is for the partial tail epoch, - // and it has to start exactly one epoch before the main section. - if merged[2] != merged[0]+mapsPerEpoch { - return fmt.Errorf("invalid tail partial epoch: %v", merged) - } - fmr.tailPartialEpoch = merged[1] - merged[0] - fmr.maps = common.NewRange(merged[2], merged[3]-merged[2]) - - default: - return fmt.Errorf("invalid number of rendered sections: %v", merged) - } - return nil -} - -// logIterator iterates on the linear log value index range. -type logIterator struct { - params *Params - chainView *ChainView - blockNumber uint64 - receipts types.Receipts - blockStart, delimiter, skipToBoundary, finished bool - txIndex, logIndex, topicIndex int - lvIndex uint64 -} - -var errUnindexedRange = errors.New("unindexed range") - -// newLogIteratorFromBlockDelimiter creates a logIterator starting at the -// given block's first log value entry (the block delimiter), according to the -// current targetView. -func (f *FilterMaps) newLogIteratorFromBlockDelimiter(blockNumber, lvIndex uint64) (*logIterator, error) { - if blockNumber > f.targetView.HeadNumber() { - return nil, fmt.Errorf("iterator entry point %d after target chain head block %d", blockNumber, f.targetView.HeadNumber()) - } - if !f.indexedRange.blocks.Includes(blockNumber) { - return nil, errUnindexedRange - } - finished := blockNumber == f.targetView.HeadNumber() - l := &logIterator{ - chainView: f.targetView, - params: &f.Params, - blockNumber: blockNumber, - finished: finished, - delimiter: !finished, - lvIndex: lvIndex, - } - l.enforceValidState() - return l, nil -} - -// newLogIteratorFromMapBoundary creates a logIterator starting at the given -// map boundary, according to the current targetView. -func (f *FilterMaps) newLogIteratorFromMapBoundary(mapIndex uint32, startBlock, startLvPtr uint64) (*logIterator, error) { - if startBlock > f.targetView.HeadNumber() { - return nil, fmt.Errorf("iterator entry point %d after target chain head block %d", startBlock, f.targetView.HeadNumber()) - } - // get block receipts - receipts := f.targetView.RawReceipts(startBlock) - if receipts == nil { - return nil, fmt.Errorf("receipts not found for start block %d", startBlock) - } - // initialize iterator at block start - l := &logIterator{ - chainView: f.targetView, - params: &f.Params, - blockNumber: startBlock, - receipts: receipts, - blockStart: true, - lvIndex: startLvPtr, - } - l.enforceValidState() - targetIndex := uint64(mapIndex) << f.logValuesPerMap - if l.lvIndex > targetIndex { - return nil, fmt.Errorf("log value pointer %d of last block of map is after map boundary %d", l.lvIndex, targetIndex) - } - // iterate to map boundary - for l.lvIndex < targetIndex { - if l.finished { - return nil, fmt.Errorf("iterator already finished at %d before map boundary target %d", l.lvIndex, targetIndex) - } - if err := l.next(); err != nil { - return nil, fmt.Errorf("failed to advance log iterator at %d before map boundary target %d: %v", l.lvIndex, targetIndex, err) - } - } - return l, nil -} - -// updateChainView updates the iterator's chain view if it still matches the -// previous view at the current position. Returns true if successful. -func (l *logIterator) updateChainView(cv *ChainView) bool { - if !matchViews(cv, l.chainView, l.blockNumber) { - return false - } - l.chainView = cv - return true -} - -// getValueHash returns the log value hash at the current position. -func (l *logIterator) getValueHash() common.Hash { - if l.delimiter || l.finished || l.skipToBoundary { - return common.Hash{} - } - log := l.receipts[l.txIndex].Logs[l.logIndex] - if l.topicIndex == 0 { - return addressValue(log.Address) - } - return topicValue(log.Topics[l.topicIndex-1]) -} - -// next moves the iterator to the next log value index. -func (l *logIterator) next() error { - if l.skipToBoundary { - l.lvIndex++ - if l.lvIndex%l.params.valuesPerMap == 0 { - l.skipToBoundary = false - } - return nil - } - if l.finished { - return nil - } - if l.delimiter { - l.delimiter = false - l.blockNumber++ - l.receipts = l.chainView.RawReceipts(l.blockNumber) - if l.receipts == nil { - return fmt.Errorf("receipts not found for block %d", l.blockNumber) - } - l.txIndex, l.logIndex, l.topicIndex, l.blockStart = 0, 0, 0, true - } else { - l.topicIndex++ - l.blockStart = false - } - l.lvIndex++ - l.enforceValidState() - return nil -} - -// enforceValidState updates the internal transaction, log and topic index pointers -// to the next existing log value of the given block if necessary. -// Note that enforceValidState does not advance the log value index pointer. -func (l *logIterator) enforceValidState() { - if l.delimiter || l.finished || l.skipToBoundary { - return - } - for ; l.txIndex < len(l.receipts); l.txIndex++ { - receipt := l.receipts[l.txIndex] - for ; l.logIndex < len(receipt.Logs); l.logIndex++ { - log := receipt.Logs[l.logIndex] - if l.topicIndex == 0 && uint64(len(log.Topics)+1) > l.params.valuesPerMap-l.lvIndex%l.params.valuesPerMap { - // next log would be split by map boundary; skip to boundary - l.skipToBoundary = true - return - } - if l.topicIndex <= len(log.Topics) { - return - } - l.topicIndex = 0 - } - l.logIndex = 0 - } - if l.blockNumber == l.chainView.HeadNumber() { - l.finished = true - } else { - l.delimiter = true - } -} diff --git a/core/filtermaps/map_storage.go b/core/filtermaps/map_storage.go new file mode 100644 index 0000000000..3efa90a5e4 --- /dev/null +++ b/core/filtermaps/map_storage.go @@ -0,0 +1,711 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package filtermaps + +import ( + "errors" + "fmt" + "math" + "sync" + "sync/atomic" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" +) + +var ErrOutOfRange = errors.New("pointer out of indexed range") + +// mapStorage implements a filter map storage layer over mapDatabase that ensures +// efficient database usage while also providing a low latency interface to the +// indexer. It uses a memory layer over mapDatabase allowing consistently quick +// addMap and deleteMaps calls while doing the actual database updates in the +// background asynchronously. +type mapStorage struct { + params *Params + mapDb *mapDatabase + triggerCh, closeCh chan struct{} + closeWg sync.WaitGroup + mtForceWrite, mtBusy uint32 + + lock sync.RWMutex + initialized bool + knownEpochs uint32 // epochs initialized with last map block pointer and corresponding reverse block lv pointer + knownEpochBlocks uint64 + valid, dirty rangeSet[uint32] // valid and dirty maps in database + writeInProgress, deleteInProgress rangeSet[uint32] // write cycle in progress + overlay rangeSet[uint32] // memory maps + overlayCount uint32 + overlayMaps map[uint32]*finishedMap + validBlocks, overlayBlocks rangeSet[uint64] + epochTrigger rangeSet[uint32] + suspended uint32 + + testHookCh chan bool +} + +// newMapStorage creates a new mapStorage layer over the given mapDatabase. +func newMapStorage(params *Params, mapDb *mapDatabase, testHookCh chan bool) *mapStorage { + m := &mapStorage{ + params: params, + mapDb: mapDb, + triggerCh: make(chan struct{}, 1), + closeCh: make(chan struct{}), + overlayMaps: make(map[uint32]*finishedMap), + testHookCh: testHookCh, + + mtForceWrite: params.rowGroupSize[0] * 9 / 8, + mtBusy: params.rowGroupSize[0] * 17 / 8, + } + if valid, dirty, knownEpochs, ok := m.mapDb.loadMapRange(); ok { + m.valid, m.dirty, m.knownEpochs, m.initialized = valid, dirty, knownEpochs, true + if knownEpochs > 0 { + if lastBlock, _, err := m.mapDb.getLastBlockOfMap(m.params.lastEpochMap(knownEpochs - 1)); err == nil { + m.knownEpochBlocks = lastBlock + 1 + } else { + m.resetWithError(fmt.Sprintf("could not initialize known epoch range: %v", err)) + } + } + if err := m.updateValidBlocks(); err != nil { + m.resetWithError(fmt.Sprintf("could not initialize valid block range: %v", err)) + } + } + m.closeWg.Add(1) + go m.eventLoop() + return m +} + +// stop stops mapStorage. +func (m *mapStorage) stop() { + close(m.closeCh) + m.closeWg.Wait() +} + +// isReady returns false if there are too many memory overlay maps at the moment. +// In this case the caller should suspend the indexing process until some maps +// are written to the database. +func (m *mapStorage) isReady() bool { + m.lock.Lock() + defer m.lock.Unlock() + + if m.overlayCount < m.mtBusy { + m.trigger() + return true + } + return false +} + +// tailEpoch returns the first epoch of the continuous rendered range. If it is +// larger than zero then the tail renderer can start indexing the previous epoch +// if necessary. After checkpoint initialization it points right after the last +// known epoch boundary (in this case the tail epoch can be empty). +func (m *mapStorage) tailEpoch() uint32 { + m.lock.Lock() + defer m.lock.Unlock() + + mapRange := m.valid.union(m.overlay) + if len(mapRange) > 0 && m.params.mapEpoch(mapRange[len(mapRange)-1].AfterLast()) >= m.knownEpochs { + return min(m.knownEpochs, m.params.mapEpoch(mapRange[len(mapRange)-1].First()+m.params.mapsPerEpoch-1)) + } + return m.knownEpochs +} + +// tailNumberOfEpoch returns the number of the first block that starts in the +// given epoch. +func (m *mapStorage) tailNumberOfEpoch(epoch uint32) (uint64, error) { + if epoch == 0 { + return 0, nil + } + number, _, err := m.getLastBlockOfMap(m.params.lastEpochMap(epoch - 1)) + if err != nil { + return 0, err + } + return number + 1, nil +} + +// lastBoundaryBefore returns the most recent map index that is less than +// or equal to the given mapIndex parameter and is either located after a stored +// map or after a known epoch boundary. +// The returned map index position may or may not contain a stored map but if it +// is empty then it is always suitable to start a rendering process. +func (m *mapStorage) lastBoundaryBefore(mapIndex uint32) uint32 { + m.lock.Lock() + defer m.lock.Unlock() + + if mapIndex == 0 { + return 0 + } + lastBoundary := m.params.firstEpochMap(min(m.params.mapEpoch(mapIndex), m.knownEpochs)) + if m, ok := m.valid.closestLte(mapIndex - 1); ok { + lastBoundary = max(lastBoundary, m+1) + } + if m, ok := m.overlay.closestLte(mapIndex - 1); ok { + lastBoundary = max(lastBoundary, m+1) + } + return lastBoundary +} + +// matchKnownEpochs returns true if the given list of checkpoints matches the +// checkpoints already stored in the database (always true with an empty database). +func (m *mapStorage) matchKnownEpochs(cpList checkpointList) bool { + m.lock.Lock() + defer m.lock.Unlock() + + if m.knownEpochs == 0 || len(cpList) == 0 { + return true + } + epoch := min(m.knownEpochs, uint32(len(cpList))) - 1 + number, hash, err := m.getLastBlockOfMap(m.params.lastEpochMap(epoch)) + if err != nil { + m.resetWithError(fmt.Sprintf("could not read last known epoch boundary: %v", err)) + return true + } + return number == cpList[epoch].BlockNumber && hash == cpList[epoch].BlockHash +} + +// addKnownEpochs adds the known epoch boundaries based on the given checkpoints. +func (m *mapStorage) addKnownEpochs(cpList checkpointList) error { + m.lock.Lock() + defer m.lock.Unlock() + + if uint32(len(cpList)) <= m.knownEpochs { + return errors.New("checkpoint init list has no new epochs") + } + if m.knownEpochs > 0 { + lastNumber, lastHash, err := m.mapDb.getLastBlockOfMap(m.params.lastEpochMap(m.knownEpochs - 1)) + if err != nil { + return err + } + lvPointer, err := m.mapDb.getBlockLvPointer(lastNumber) + if err != nil { + return err + } + if cp := cpList[m.knownEpochs-1]; cp.BlockNumber != lastNumber || cp.BlockHash != lastHash || cp.FirstIndex != lvPointer { + return errors.New("checkpoint init list does not match known epochs") + } + } + + m.mapDb.storeCheckpointList(m.knownEpochs, cpList[m.knownEpochs:]) + m.knownEpochs = uint32(len(cpList)) + m.knownEpochBlocks = cpList[len(cpList)-1].BlockNumber + 1 + m.mapDb.storeMapRange(m.valid, m.dirty, m.knownEpochs) + return nil +} + +// addMap adds a new map to the storage. If forceCommit is true then a write is +// always triggered. If it is false then a write is only triggered when a row +// group boundary is reached or if the total number of memory maps reaches a limit. +// addMap always returns right after adding the new map to the memory layer. +func (m *mapStorage) addMap(mapIndex uint32, fm *finishedMap, forceCommit bool) { + m.lock.Lock() + defer m.lock.Unlock() + + if fm == nil { + panic("trying to add nil map") + } + if m.valid.includes(mapIndex) || m.overlay.includes(mapIndex) { + panic("addMap to non-empty map index") + } + epoch := m.params.mapEpoch(mapIndex) + if (epoch > m.knownEpochs || mapIndex != m.params.firstEpochMap(epoch)) && + !m.valid.includes(mapIndex-1) && !m.overlay.includes(mapIndex-1) { + panic("addMap to map index with no known boundary") + } + mapRs := singleRangeSet[uint32](common.NewRange[uint32](mapIndex, 1)) + m.overlay = m.overlay.union(mapRs) + m.writeInProgress = m.writeInProgress.exclude(mapRs) + m.overlayMaps[mapIndex] = fm + m.updateOverlayBlocks() + if forceCommit || (mapIndex+1)%m.params.rowGroupSize[0] == 0 { + m.epochTrigger = m.epochTrigger.union(singleRangeSet[uint32](common.NewRange[uint32](epoch, 1))) + m.trigger() + } +} + +// deleteMaps deletes the given map range. Note that similarly to addMap, it only +// performs memory operations before returning. The deleted map range is marked +// dirty, then later the database update goroutine will actually delete or +// overwrite the dirty maps. +func (m *mapStorage) deleteMaps(maps common.Range[uint32]) { + m.lock.Lock() + defer m.lock.Unlock() + + dr := singleRangeSet[uint32](maps) + for i := range dr.intersection(m.overlay).iter() { + delete(m.overlayMaps, i) + } + m.writeInProgress = m.writeInProgress.exclude(dr) + knownEpochs := m.knownEpochs + if maps.Includes(m.params.firstEpochMap(knownEpochs)) { + knownEpochs = m.params.mapEpoch(maps.First()) + } + if err := m.updateRange(m.valid.exclude(dr), m.dirty.union(dr.intersection(m.valid)), m.overlay.exclude(dr), knownEpochs); err != nil { + m.resetWithError(fmt.Sprintf("could not revert valid block range: %v", err)) + } + m.trigger() +} + +// suspendOrResume suspends or resumes the background database update operations. +func (m *mapStorage) suspendOrResume(suspend bool) { + var suspendU32 uint32 + if suspend { + suspendU32 = 1 + } + old := atomic.SwapUint32(&m.suspended, suspendU32) + if suspendU32 < old { + m.trigger() + } +} + +// eventLoop is the main event loop of the database update operations. +func (m *mapStorage) eventLoop() { + defer m.closeWg.Done() + + var stopped bool + + selectEvent := func(blocking bool) { + if m.testHookCh != nil { + select { + case <-m.closeCh: + stopped = true + return + case m.testHookCh <- blocking: + } + } + if blocking { + select { + case <-m.closeCh: + stopped = true + return + case <-m.triggerCh: + } + } else { + select { + case <-m.closeCh: + stopped = true + return + case <-m.triggerCh: + default: + } + } + if m.testHookCh != nil { + select { + case <-m.closeCh: + stopped = true + return + case <-m.testHookCh: + } + } + } + + stopCallback := func() bool { + selectEvent(atomic.LoadUint32(&m.suspended) == 1) + return stopped + } + + for !stopped { + if !m.initialized { + if done, _ := m.mapDb.reset(stopCallback); !done { + return // node stopped before cleaning old database + } + m.lock.Lock() + m.initialized = true + m.lock.Unlock() + } + more, err := m.doWriteCycle(stopCallback) + if err != nil { + m.resetWithError(fmt.Sprintf("write cycle failed: %v", err)) + continue + } + selectEvent(!more && !stopped) // wait for next event if no changes done + } +} + +// trigger ensures that the database update cycle will restart if it was in a +// waiting state. +func (m *mapStorage) trigger() { + select { + case m.triggerCh <- struct{}{}: + default: + } +} + +// getBlockLvPointer returns the starting log value index where the log values +// generated by the given block are located. +func (m *mapStorage) getBlockLvPointer(blockNumber uint64) (uint64, error) { + m.lock.RLock() + defer m.lock.RUnlock() + + if m.overlayBlocks.includes(blockNumber) { + for _, fm := range m.overlayMaps { + if fm.blocks().Includes(blockNumber) { + return fm.blockPtrs[blockNumber-fm.firstBlock()], nil + } + } + return 0, errors.New("memory overlay block pointer not found") + } + if blockNumber < m.knownEpochBlocks || m.validBlocks.includes(blockNumber) { + return m.mapDb.getBlockLvPointer(blockNumber) + } + return 0, ErrOutOfRange +} + +// getLastBlockOfMap returns the number and hash of the block that generated the +// last log value entry of the given map. +func (m *mapStorage) getLastBlockOfMap(mapIndex uint32) (uint64, common.Hash, error) { + m.lock.RLock() + defer m.lock.RUnlock() + + if m.overlay.includes(mapIndex) { + fm := m.overlayMaps[mapIndex] + if fm == nil { + return 0, common.Hash{}, errors.New("memory overlay map not found") + } + return fm.lastBlock.number, fm.lastBlock.hash, nil + } + if mapIndex < m.params.firstEpochMap(m.knownEpochs) || m.valid.includes(mapIndex) || m.valid.includes(mapIndex+1) { + return m.mapDb.getLastBlockOfMap(mapIndex) + } + return 0, common.Hash{}, ErrOutOfRange +} + +// getFilterMapRows returns a batch of filter maps rows from the same row index, +// each truncated to the length limit of the specified mapping layer. +// The function assumes that the map indices are in strictly ascending order. +func (m *mapStorage) getFilterMapRows(mapIndices []uint32, rowIndex, layerIndex uint32) ([]FilterRow, error) { + m.lock.RLock() + defer m.lock.RUnlock() + + rows := make([]FilterRow, len(mapIndices)) + dbMaps := make([]uint32, 0, len(mapIndices)) + for i, mapIndex := range mapIndices { + if m.overlay.includes(mapIndex) { + fm := m.overlayMaps[mapIndex] + if fm == nil { + return nil, errors.New("memory overlay map not found") + } + rows[i] = fm.getRow(rowIndex, m.params.getMaxRowLength(layerIndex)) + } else { + dbMaps = append(dbMaps, mapIndex) + } + } + dbRows, err := m.mapDb.getFilterMapRows(dbMaps, rowIndex, layerIndex) + if err != nil { + return nil, err + } + var j int + for i, row := range rows { + if row == nil { // zero length row is represented as zero length slice + rows[i] = dbRows[j] + j++ + } + } + if j != len(dbMaps) { + panic("rows length mismatch") + } + return rows, nil +} + +// getFilterMap returns the filter map at the specified index. +func (m *mapStorage) getFilterMap(mapIndex uint32) (*finishedMap, error) { + m.lock.RLock() + defer m.lock.RUnlock() + + if m.overlay.includes(mapIndex) { + fm := m.overlayMaps[mapIndex] + if fm == nil { + return nil, errors.New("memory overlay map not found") + } + return fm, nil + } + if m.valid.includes(mapIndex) { + return m.mapDb.getFilterMap(mapIndex) + } + return nil, nil +} + +// extendDeletedPointerRange takes a map range where pointers need to be deleted +// and extends it so that on both ends there is a neighboring stored map, a +// known epoch boundary or one end of the map index range. This is required +// in order to determine the block range covered by the deleted map range. +func (m *mapStorage) extendDeletedPointerRange(deleteRange common.Range[uint32]) common.Range[uint32] { + epoch := m.params.mapEpoch(deleteRange.First()) + if m.params.mapEpoch(deleteRange.Last()) != epoch { + panic("deleted map range crosses epoch boundary") + } + first := m.params.firstEpochMap(min(m.knownEpochs, epoch)) + if deleteRange.First() > 0 { + if c, ok := m.valid.closestLte(deleteRange.First() - 1); ok { + first = max(first, c+1) + } + } + afterLast := uint32(math.MaxUint32) + if epoch < m.knownEpochs { + afterLast = m.params.firstEpochMap(epoch + 1) + } + if fa, ok := m.valid.closestGte(deleteRange.AfterLast()); ok { + afterLast = min(afterLast, fa) + } + return common.NewRange[uint32](first, afterLast-first) +} + +// selectEpochTriggeredWrite selects the next epoch to update if one of the epochs +// in the canSelect list has been triggered by addMap. +func (m *mapStorage) selectEpochTriggeredWrite(canSelect []uint32) (uint32, bool) { + for _, epoch := range canSelect { + if !m.epochTrigger.includes(epoch) { + continue + } + m.epochTrigger = m.epochTrigger.exclude(singleRangeSet[uint32](common.NewRange[uint32](epoch, 1))) + epochRange := common.NewRange[uint32](m.params.firstEpochMap(epoch), m.params.mapsPerEpoch) + if len(m.overlay.intersection(singleRangeSet[uint32](epochRange))) > 0 { + return epoch, true + } + } + return 0, false +} + +// selectEpochForcedWrite selects the next epoch to update if the total number +// of memory maps has reached a threshold. +func (m *mapStorage) selectEpochForcedWrite(canSelect []uint32) (uint32, bool) { + if m.overlayCount < m.mtForceWrite { + return 0, false + } + var best, count uint32 + for _, epoch := range canSelect { + epochRange := common.NewRange[uint32](m.params.firstEpochMap(epoch), m.params.mapsPerEpoch) + if c := m.overlay.intersection(singleRangeSet[uint32](epochRange)).count(); c > count { + best, count = epoch, c + } + } + if count == 0 { + return 0, false + } + return best, true +} + +// mapToEpochRange returns the set of epochs that are either fully or partially +// covered by the specified mapRange. +func (m *mapStorage) mapToEpochRange(mapRange rangeSet[uint32]) rangeSet[uint32] { + vb := make(rangeBoundaries[uint32], 0, len(mapRange)*2) + for _, r := range mapRange { + first := m.params.mapEpoch(r.First()) + last := m.params.mapEpoch(r.Last()) + vb.add(common.NewRange[uint32](first, last+1-first), 1) + } + return vb.makeSet(1) +} + +// selectEpochDeleteOnly selects an epoch to update where there are no memory +// maps to write, only dirty maps to clean up. +func (m *mapStorage) selectEpochDeleteOnly() (uint32, bool) { + epochs := m.mapToEpochRange(m.dirty).exclude(m.mapToEpochRange(m.overlay)) + if len(epochs) == 0 { + return 0, false + } + return epochs[0].First(), true +} + +// doWriteCycle selects an epoch where there are memory overlay maps and/or +// dirty maps in the underlying database, cleans up dirty data and writes the +// new maps to the database. +// If an epoch has been successfully updated then the function returns (true, nil). +// In this case a new write cycle can be attempted immediately until there is +// nothing more left to do. +// If there was nothing to do or if stopCallback aborted the operation then the +// function returns (false, nil). In case of an error it returns (false, err). +// Note that if the operation is aborted or failed then the partially updated +// maps are left as marked dirty and the database remains consistent. +func (m *mapStorage) doWriteCycle(stopCallback func() bool) (bool, error) { + m.lock.Lock() + defer m.lock.Unlock() + + // always operate on a single epoch + var canSelect []uint32 + if len(m.valid) == 0 { + canSelect = []uint32{m.knownEpochs} + } else { + lastValid := m.valid[len(m.valid)-1] + tailEpoch := m.params.mapEpoch(lastValid.First()) + headEpoch := m.params.mapEpoch(lastValid.AfterLast()) + if tailEpoch > 0 { + canSelect = []uint32{headEpoch, tailEpoch - 1} + } else { + canSelect = []uint32{headEpoch} + } + } + + epoch, ok := m.selectEpochTriggeredWrite(canSelect) + if !ok { + epoch, ok = m.selectEpochForcedWrite(canSelect) + if !ok { + epoch, ok = m.selectEpochDeleteOnly() + if !ok { + return false, nil + } + } + } + epochRange := singleRangeSet[uint32](common.NewRange[uint32](m.params.firstEpochMap(epoch), m.params.mapsPerEpoch)) + writeMaps := epochRange.intersection(m.overlay).singleRange() + validInEpoch := epochRange.intersection(m.valid).singleRange() + dirtyInEpoch := epochRange.intersection(m.dirty).singleRange() + keepEmptyFrom := max(m.params.firstEpochMap(epoch), writeMaps.AfterLast(), validInEpoch.AfterLast(), dirtyInEpoch.AfterLast()) + keepEmptyInEpoch := common.NewRange[uint32](keepEmptyFrom, m.params.firstEpochMap(epoch+1)-keepEmptyFrom) + // delete old pointers + if !dirtyInEpoch.IsEmpty() { + m.mapDb.deletePointers(m.extendDeletedPointerRange(dirtyInEpoch), stopCallback) + } + if writeMaps.IsEmpty() && validInEpoch.IsEmpty() { + // delete map rows of entire epoch if nothing to write or keep + m.lock.Unlock() + done, err := m.mapDb.deleteEpochRows(epoch, stopCallback) + m.lock.Lock() + if done { + if err := m.updateRange(m.valid, m.dirty.exclude(epochRange), m.overlay, m.knownEpochs); err != nil { + return false, err + } + } + return done, err + } + maps := make([]*finishedMap, writeMaps.Count()) + for i := range writeMaps.Iter() { + maps[i-writeMaps.First()] = m.overlayMaps[i] + } + // temporarily mark newly written maps as dirty (replaced/deleted maps are already dirty) + writeInProgress := singleRangeSet[uint32](writeMaps) + deleteInProgress := singleRangeSet[uint32](dirtyInEpoch).exclude(writeInProgress) + if err := m.updateRange(m.valid, m.dirty.union(writeInProgress), m.overlay, m.knownEpochs); err != nil { + return false, err + } + m.writeInProgress = writeInProgress + m.deleteInProgress = deleteInProgress + m.lock.Unlock() + // write/overwrite map rows and delete dirty map data, write new pointers + done, err := m.mapDb.writeMapRows(writeMaps, dirtyInEpoch, keepEmptyInEpoch, maps, stopCallback) + if done { + done, err = m.mapDb.writePointers(writeMaps, maps, stopCallback) + } + m.lock.Lock() + writeInProgress = m.writeInProgress + m.writeInProgress, m.deleteInProgress = nil, nil + if !done { + return false, err + } + for mapIndex := range writeInProgress.iter() { + delete(m.overlayMaps, mapIndex) + } + knownEpochs := m.knownEpochs + if len(writeInProgress) > 0 { + knownEpochs = max(knownEpochs, m.params.mapEpoch(writeInProgress[len(writeInProgress)-1].Last())) + } + if err := m.updateRange(m.valid.union(writeInProgress), m.dirty.exclude(writeInProgress.union(deleteInProgress)), m.overlay.exclude(writeInProgress), knownEpochs); err != nil { + return false, err + } + return true, nil +} + +// updateRange updates the stored valid, dirty and memory overlay map ranges and +// the number of known epoch boundaries. It also stores these ranges in the +// database and determines the corresponding block ranges. +func (m *mapStorage) updateRange(valid, dirty, overlay rangeSet[uint32], knownEpochs uint32) error { + if !valid.equal(m.valid) { + m.valid = valid + if err := m.updateValidBlocks(); err != nil { + return err + } + } + if knownEpochs != m.knownEpochs { + m.knownEpochs = knownEpochs + if err := m.updateKnownEpochBlocks(); err != nil { + return err + } + } + m.dirty = dirty + if !overlay.equal(m.overlay) { + m.overlay = overlay + m.updateOverlayBlocks() + } + m.mapDb.storeMapRange(valid, dirty, m.knownEpochs) + return nil +} + +// resetWithError resets an invalid log index database after an unrecoverable error. +func (m *mapStorage) resetWithError(errStr string) { + log.Error("Resetting invalid log index database", "error", errStr) + m.uninitialize() +} + +// uninitialize resets the map storage and returns to its non-initialized state. +func (m *mapStorage) uninitialize() { + m.valid, m.dirty, m.overlay, m.knownEpochs, m.initialized = nil, nil, nil, 0, false + m.mapDb.deleteMapRange() + m.trigger() +} + +// updateKnownEpochBlocks determines the block number belonging to the last known +// epoch boundary. +func (m *mapStorage) updateKnownEpochBlocks() error { + if m.knownEpochs > 0 { + if lastBlock, _, err := m.mapDb.getLastBlockOfMap(m.params.lastEpochMap(m.knownEpochs - 1)); err == nil { + m.knownEpochBlocks = lastBlock + 1 + } else { + return err + } + } else { + m.knownEpochBlocks = 0 + } + return nil +} + +// updateValidBlocks determines the the set of blocks where the starting log value +// pointer points into the valid map set. +func (m *mapStorage) updateValidBlocks() error { + if len(m.valid) > 2 || (len(m.valid) == 2 && m.valid[0].Count() >= m.params.mapsPerEpoch) { + panic("invalid mapStorage.valid") + return errors.New("invalid mapStorage.valid range set") + } + vb := make(rangeBoundaries[uint64], 0, len(m.valid)*2) + for _, vr := range m.valid { + var first uint64 + if vr.First() > 0 { + lb, _, err := m.mapDb.getLastBlockOfMap(vr.First() - 1) + if err != nil { + return err + } + first = lb + 1 + } + last, _, err := m.mapDb.getLastBlockOfMap(vr.Last()) + if err != nil { + return err + } + vb.add(common.NewRange[uint64](first, last+1-first), 1) + } + m.validBlocks = vb.makeSet(1) + return nil +} + +// updateOverlayBlocks determines the the set of blocks where the starting log +// value pointer points into the memory overlay map set. +func (m *mapStorage) updateOverlayBlocks() { + ob := make(rangeBoundaries[uint64], 0, len(m.overlay)*2) + for _, or := range m.overlay { + first := m.overlayMaps[or.First()].firstBlock() + last := m.overlayMaps[or.Last()].lastBlock.number + ob.add(common.NewRange[uint64](first, last+1-first), 1) + } + m.overlayBlocks = ob.makeSet(1) + m.overlayCount = m.overlay.count() +} diff --git a/core/filtermaps/map_storage_test.go b/core/filtermaps/map_storage_test.go new file mode 100644 index 0000000000..0d2939ef5f --- /dev/null +++ b/core/filtermaps/map_storage_test.go @@ -0,0 +1,150 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package filtermaps + +import ( + "math" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb/memorydb" +) + +func TestMapStorage(t *testing.T) { + params := testParams + params.sanitize() + db := memorydb.New() + mapDb := newMapDatabase(¶ms, db, false) + ms := newMapStorage(¶ms, mapDb, make(chan bool)) + <-ms.testHookCh + defer ms.stop() + + reader := mapReader{ + getFilterMapRows: ms.getFilterMapRows, + getFilterMap: ms.getFilterMap, + getBlockLvPointer: ms.getBlockLvPointer, + getLastBlockOfMap: ms.getLastBlockOfMap, + } + waitCycle := func() bool { + ms.testHookCh <- true + return !<-ms.testHookCh + } + expKnownEpochs := func(testCase int, exp uint32) { + if ms.knownEpochs != exp { + t.Fatalf("Invalid known epochs number in test case #%d (expected %d, got %d)", testCase, exp, ms.knownEpochs) + } + } + expTailEpoch := func(testCase int, exp uint32) { + if ms.tailEpoch() != exp { + t.Fatalf("Invalid tail epoch in test case #%d (expected %d, got %d)", testCase, exp, ms.tailEpoch()) + } + } + expReadRows := func(testCase int, exp bool) { + if mapDb.testReadRows != exp { + t.Fatalf("Invalid read rows flag in test case #%d (expected %v, got %v)", testCase, exp, mapDb.testReadRows) + } + mapDb.testReadRows = false + } + // initialize database with checkpoints + maps := generateTestMaps(¶ms, nil, 0x200) + cpList := generateTestCheckpoints(¶ms, maps) + ms.addKnownEpochs(cpList) + expTailEpoch(0, 8) + expKnownEpochs(0, 8) + // add new maps to the head + maps = generateTestMaps(¶ms, maps, 0x50) + for m := uint32(0x200); m < 0x250; m++ { + ms.addMap(m, maps[m], true) + } + for waitCycle() { + testMapReader(t, "mapStorage test #1", ¶ms, reader, cpList, maps[0x200:]) + } + expReadRows(1, false) + expTailEpoch(1, 8) + expKnownEpochs(1, 9) + // backfill previous epoch with a single write + for m := uint32(0x1c0); m < 0x200; m++ { + ms.addMap(m, maps[m], false) + } + for waitCycle() { + testMapReader(t, "mapDatabase test #2", ¶ms, reader, cpList[:7], maps[0x1c0:]) + } + expReadRows(2, false) + expTailEpoch(2, 7) + expKnownEpochs(2, 9) + // backfill previous epoch in two steps + for m := uint32(0x180); m < 0x192; m++ { + ms.addMap(m, maps[m], true) + } + for waitCycle() { + } + expReadRows(3, false) + expTailEpoch(3, 7) + for m := uint32(0x192); m < 0x1c0; m++ { + ms.addMap(m, maps[m], false) + } + for waitCycle() { + testMapReader(t, "mapDatabase test #3", ¶ms, reader, cpList[:6], maps[0x180:]) + } + expReadRows(3, true) + expTailEpoch(3, 6) + expKnownEpochs(3, 9) + // add new maps while reorging some existing ones + maps = generateTestMaps(¶ms, maps[:0x234], 0x30) + ms.deleteMaps(common.NewRange[uint32](0x234, math.MaxUint32-0x234)) + expKnownEpochs(4, 8) + for m := uint32(0x234); m < 0x264; m++ { + ms.addMap(m, maps[m], true) + } + for waitCycle() { + testMapReader(t, "mapDatabase test #4", ¶ms, reader, cpList[:6], maps[0x180:]) + } + expReadRows(4, true) + expTailEpoch(4, 6) + expKnownEpochs(4, 9) + // unindex tail epoch + ms.deleteMaps(common.NewRange[uint32](0x180, 0x40)) + expTailEpoch(5, 7) + for waitCycle() { + testMapReader(t, "mapDatabase test #5", ¶ms, reader, cpList[:7], maps[0x1c0:]) + } + expReadRows(5, false) + expTailEpoch(5, 7) + expKnownEpochs(5, 9) + // remove head maps + maps = maps[:0x253] + ms.deleteMaps(common.NewRange[uint32](0x253, math.MaxUint32-0x253)) + for waitCycle() { + testMapReader(t, "mapDatabase test #6", ¶ms, reader, cpList[:7], maps[0x1c0:]) + } + expReadRows(6, true) + expTailEpoch(6, 7) + expKnownEpochs(6, 9) + // add maps until epoch boundary and check known epochs increase + maps = generateTestMaps(¶ms, maps, 0x30) + for m := uint32(0x253); m < 0x283; m++ { + ms.addMap(m, maps[m], true) + } + expTailEpoch(7, 7) + expKnownEpochs(7, 9) + for waitCycle() { + testMapReader(t, "mapDatabase test #7", ¶ms, reader, cpList[:7], maps[0x1c0:]) + } + expReadRows(7, true) + expTailEpoch(7, 7) + expKnownEpochs(7, 10) +} diff --git a/core/filtermaps/matcher.go b/core/filtermaps/matcher.go index 238723fe1d..706b01f123 100644 --- a/core/filtermaps/matcher.go +++ b/core/filtermaps/matcher.go @@ -28,6 +28,18 @@ import ( "github.com/ethereum/go-ethereum/common/mclock" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" +) + +var ( + matchRequestTimer = metrics.NewRegisteredTimer("filtermaps/match/requesttime", nil) // processing time a matching request in a single epoch + matchEpochTimer = metrics.NewRegisteredTimer("filtermaps/match/epochtime", nil) // total processing time a matching request + matchBaseRowAccessMeter = metrics.NewRegisteredMeter("filtermaps/match/baserowaccess", nil) // number of accessed rows on layer 0 + matchBaseRowSizeMeter = metrics.NewRegisteredMeter("filtermaps/match/baserowsize", nil) // size of accessed rows on layer 0 + matchExtRowAccessMeter = metrics.NewRegisteredMeter("filtermaps/match/extrowaccess", nil) // number of accessed rows on higher layers + matchExtRowSizeMeter = metrics.NewRegisteredMeter("filtermaps/match/extrowsize", nil) // size of accessed rows on higher layers + matchLogLookup = metrics.NewRegisteredMeter("filtermaps/match/loglookup", nil) // number of log lookups based on potential matches + matchAllMeter = metrics.NewRegisteredMeter("filtermaps/match/matchall", nil) // number of requests returned with ErrMatchAll ) const doRuntimeStats = false @@ -37,32 +49,32 @@ const doRuntimeStats = false // would actually be slower than reverting to legacy filter. var ErrMatchAll = errors.New("match all patterns not supported") -// MatcherBackend defines the functions required for searching in the log index -// data structure. It is currently implemented by FilterMapsMatcherBackend but -// once EIP-7745 is implemented and active, these functions can also be trustlessly -// served by a remote prover. -type MatcherBackend interface { - GetParams() *Params - GetBlockLvPointer(ctx context.Context, blockNumber uint64) (uint64, error) - GetFilterMapRows(ctx context.Context, mapIndices []uint32, rowIndex uint32, baseLayerOnly bool) ([]FilterRow, error) - GetLogByLvIndex(ctx context.Context, lvIndex uint64) (*types.Log, error) - SyncLogIndex(ctx context.Context) (SyncRange, error) - Close() +type LogPosition struct { + BlockNumber, startPtr, logPtr, valuesPerMap uint64 } -// SyncRange is returned by MatcherBackend.SyncLogIndex. It contains the latest -// chain head, the indexed range that is currently consistent with the chain -// and the valid range that has not been changed and has been consistent with -// all states of the chain since the previous SyncLogIndex or the creation of -// the matcher backend. -type SyncRange struct { - IndexedView *ChainView - // block range where the index has not changed since the last matcher sync - // and therefore the set of matches found in this region is guaranteed to - // be valid and complete. - ValidBlocks common.Range[uint64] - // block range indexed according to the given chain head. - IndexedBlocks common.Range[uint64] +func (l *LogPosition) GetLog(receipts types.Receipts) (*types.Log, error) { + ptr := l.startPtr + for _, receipt := range receipts { + for _, log := range receipt.Logs { + step := uint64(len(log.Topics) + 1) + subPtr := ptr % l.valuesPerMap + if subPtr+step > l.valuesPerMap { + ptr += l.valuesPerMap - subPtr + } + if ptr == l.logPtr { + return log, nil + } + ptr += step + if ptr > l.logPtr { + return nil, nil // log position is in block range but there is no log there (false positive) + } + } + } + if ptr == l.logPtr { + return nil, nil // points to the block delimiter (false positive) + } + return nil, fmt.Errorf("invalid log position: block #%d start pointer %d log pointer %d", l.BlockNumber, l.startPtr, l.logPtr) } // GetPotentialMatches returns a list of logs that are potential matches for the @@ -70,14 +82,14 @@ type SyncRange struct { // missing or changed during the search process then the resulting logs belonging // to that block range might be missing or incorrect. // Also note that the returned list may contain false positives. -func GetPotentialMatches(ctx context.Context, backend MatcherBackend, firstBlock, lastBlock uint64, addresses []common.Address, topics [][]common.Hash) ([]*types.Log, error) { +func GetPotentialMatches(ctx context.Context, backend *IndexView, firstBlock, lastBlock uint64, addresses []common.Address, topics [][]common.Hash) ([]LogPosition, error) { params := backend.GetParams() // find the log value index range to search - firstIndex, err := backend.GetBlockLvPointer(ctx, firstBlock) + firstIndex, err := backend.GetBlockLvPointer(firstBlock) if err != nil { return nil, fmt.Errorf("failed to retrieve log value pointer for first block %d: %v", firstBlock, err) } - lastIndex, err := backend.GetBlockLvPointer(ctx, lastBlock+1) + lastIndex, err := backend.GetBlockLvPointer(lastBlock + 1) if err != nil { return nil, fmt.Errorf("failed to retrieve log value pointer after last block %d: %v", lastBlock, err) } @@ -112,10 +124,11 @@ func GetPotentialMatches(ctx context.Context, backend MatcherBackend, firstBlock matcher := newMatchSequence(params, matchers) m := &matcherEnv{ - ctx: ctx, backend: backend, params: params, matcher: matcher, + firstBlock: firstBlock, + lastBlock: lastBlock, firstIndex: firstIndex, lastIndex: lastIndex, firstMap: uint32(firstIndex >> params.logValuesPerMap), @@ -123,7 +136,7 @@ func GetPotentialMatches(ctx context.Context, backend MatcherBackend, firstBlock } start := time.Now() - res, err := m.process() + res, err := m.process(ctx) matchRequestTimer.Update(time.Since(start)) if doRuntimeStats { @@ -142,18 +155,18 @@ func GetPotentialMatches(ctx context.Context, backend MatcherBackend, firstBlock type matcherEnv struct { getLogStats runtimeStats // 64 bit aligned - ctx context.Context - backend MatcherBackend + backend *IndexView params *Params matcher matcher + firstBlock, lastBlock uint64 firstIndex, lastIndex uint64 firstMap, lastMap uint32 } -func (m *matcherEnv) process() ([]*types.Log, error) { +func (m *matcherEnv) process(ctx context.Context) ([]LogPosition, error) { type task struct { epochIndex uint32 - logs []*types.Log + logs []LogPosition err error done chan struct{} } @@ -182,7 +195,7 @@ func (m *matcherEnv) process() ([]*types.Log, error) { } firstEpoch, lastEpoch := m.firstMap>>m.params.logMapsPerEpoch, m.lastMap>>m.params.logMapsPerEpoch - var logs []*types.Log + var logs []LogPosition // startEpoch is the next task to send whenever a worker can accept it. // waitEpoch is the next task we are waiting for to finish in order to append // results in the correct order. @@ -214,15 +227,17 @@ func (m *matcherEnv) process() ([]*types.Log, error) { tasks[waitEpoch] = &task{epochIndex: waitEpoch, done: make(chan struct{})} } } + /*case <-ctx.Done(): //TODO + return nil, ctx.Err()*/ } } return logs, nil } // processEpoch returns the potentially matching logs from the given epoch. -func (m *matcherEnv) processEpoch(epochIndex uint32) ([]*types.Log, error) { +func (m *matcherEnv) processEpoch(epochIndex uint32) ([]LogPosition, error) { start := time.Now() - var logs []*types.Log + var logs []LogPosition // create a list of map indices to process fm, lm := epochIndex<> m.params.logValuesPerMap) + lastBlockNumber, _, err := m.backend.GetLastBlockOfMap(mapIndex) + if err != nil { + return LogPosition{}, fmt.Errorf("failed to retrieve last block of map %d containing searched log value index %d: %v", mapIndex, lvIndex, err) + } + var firstBlockNumber uint64 + if mapIndex > 0 { + firstBlockNumber, _, err = m.backend.GetLastBlockOfMap(mapIndex - 1) + if err != nil { + return LogPosition{}, fmt.Errorf("failed to retrieve last block of map %d before searched log value index %d: %v", mapIndex, lvIndex, err) + } + } + firstBlockNumber = max(firstBlockNumber, m.firstBlock) + lastBlockNumber = min(lastBlockNumber, m.lastBlock) + // find block with binary search based on block to log value index pointers + for firstBlockNumber < lastBlockNumber { + midBlockNumber := (firstBlockNumber + lastBlockNumber + 1) / 2 + midLvPointer, err := m.backend.GetBlockLvPointer(midBlockNumber) + if err != nil { + return LogPosition{}, fmt.Errorf("failed to retrieve log value pointer of block %d while binary searching log value index %d: %v", midBlockNumber, lvIndex, err) + } + if lvIndex < midLvPointer { + lastBlockNumber = midBlockNumber - 1 + } else { + firstBlockNumber = midBlockNumber + } + } + blockLvPointer, err := m.backend.GetBlockLvPointer(firstBlockNumber) + if err != nil { + return LogPosition{}, fmt.Errorf("failed to retrieve log value pointer of block %d containing searched log value index %d: %v", firstBlockNumber, lvIndex, err) + } + return LogPosition{BlockNumber: firstBlockNumber, startPtr: blockLvPointer, logPtr: lvIndex, valuesPerMap: m.params.valuesPerMap}, nil +} + // getLogsFromMatches returns the list of potentially matching logs located at // the given list of matching log indices. Matches outside the firstIndex to // lastIndex range are not returned. -func (m *matcherEnv) getLogsFromMatches(matches potentialMatches) ([]*types.Log, error) { - var logs []*types.Log +func (m *matcherEnv) getLogsFromMatches(matches potentialMatches) ([]LogPosition, error) { + var logs []LogPosition for _, match := range matches { if match < m.firstIndex || match > m.lastIndex { continue } - log, err := m.backend.GetLogByLvIndex(m.ctx, match) + log, err := m.getLogPosition(match) if err != nil { return logs, fmt.Errorf("failed to retrieve log at index %d: %v", match, err) } - if log != nil { - logs = append(logs, log) - } + logs = append(logs, log) matchLogLookup.Mark(1) } return logs, nil @@ -288,7 +337,7 @@ func (m *matcherEnv) getAllMatches(mapIndices []uint32) ([]potentialMatches, err instance := m.matcher.newInstance(mapIndices) resultsMap := make(map[uint32]potentialMatches) for layerIndex := uint32(0); len(resultsMap) < len(mapIndices); layerIndex++ { - results, err := instance.getMatchesForLayer(m.ctx, layerIndex) + results, err := instance.getMatchesForLayer(layerIndex) if err != nil { return nil, err } @@ -319,7 +368,7 @@ type matcher interface { // a result has been returned has no effect. Exactly one matcherResult is // returned per requested map index unless dropped. type matcherInstance interface { - getMatchesForLayer(ctx context.Context, layerIndex uint32) ([]matcherResult, error) + getMatchesForLayer(layerIndex uint32) ([]matcherResult, error) dropIndices(mapIndices []uint32) } @@ -332,7 +381,7 @@ type matcherResult struct { // singleMatcher implements matcher by returning matches for a single log value hash. type singleMatcher struct { - backend MatcherBackend + backend *IndexView value common.Hash stats runtimeStats } @@ -360,7 +409,7 @@ func (m *singleMatcher) newInstance(mapIndices []uint32) matcherInstance { } // getMatchesForLayer implements matcherInstance. -func (m *singleMatcherInstance) getMatchesForLayer(ctx context.Context, layerIndex uint32) (results []matcherResult, err error) { +func (m *singleMatcherInstance) getMatchesForLayer(layerIndex uint32) (results []matcherResult, err error) { var st int m.stats.setState(&st, stOther) params := m.backend.GetParams() @@ -378,7 +427,7 @@ func (m *singleMatcherInstance) getMatchesForLayer(ctx context.Context, layerInd } else { m.stats.setState(&st, stFetchMore) } - groupRows, err := m.backend.GetFilterMapRows(ctx, m.mapIndices[ptr:ptr+groupLength], rowIndex, layerIndex == 0) + groupRows, err := m.backend.GetFilterMapRows(m.mapIndices[ptr:ptr+groupLength], rowIndex, layerIndex) if err != nil { m.stats.setState(&st, stNone) return nil, fmt.Errorf("failed to retrieve filter map %d row %d: %v", m.mapIndices[ptr], rowIndex, err) @@ -400,7 +449,7 @@ func (m *singleMatcherInstance) getMatchesForLayer(ctx context.Context, layerInd } m.stats.addAmount(st, int64(len(filterRow))) filterRows = append(filterRows, filterRow) - if uint32(len(filterRow)) < params.maxRowLength(layerIndex) { + if uint32(len(filterRow)) < params.getMaxRowLength(layerIndex) { m.stats.setState(&st, stProcess) matches := params.potentialMatches(filterRows, mapIndex, m.value) m.stats.addAmount(st, int64(len(matches))) @@ -416,6 +465,12 @@ func (m *singleMatcherInstance) getMatchesForLayer(ctx context.Context, layerInd } ptr += groupLength } + if len(m.mapIndices) > 0 { + var r int + for _, res := range results { + r += len(res.matches) + } + } m.cleanMapIndices() m.stats.setState(&st, stNone) return results, nil @@ -491,7 +546,7 @@ func (m matchAny) newInstance(mapIndices []uint32) matcherInstance { } // getMatchesForLayer implements matcherInstance. -func (m *matchAnyInstance) getMatchesForLayer(ctx context.Context, layerIndex uint32) (mergedResults []matcherResult, err error) { +func (m *matchAnyInstance) getMatchesForLayer(layerIndex uint32) (mergedResults []matcherResult, err error) { if len(m.matchAny) == 0 { // return "wild card" results (potentialMatches(nil) is interpreted as a // potential match at every log value index of the map). @@ -504,7 +559,7 @@ func (m *matchAnyInstance) getMatchesForLayer(ctx context.Context, layerIndex ui return mergedResults, nil } for i, childInstance := range m.childInstances { - results, err := childInstance.getMatchesForLayer(ctx, layerIndex) + results, err := childInstance.getMatchesForLayer(layerIndex) if err != nil { return nil, fmt.Errorf("failed to evaluate child matcher on layer %d: %v", layerIndex, err) } @@ -698,19 +753,19 @@ type matchSequenceInstance struct { } // getMatchesForLayer implements matcherInstance. -func (m *matchSequenceInstance) getMatchesForLayer(ctx context.Context, layerIndex uint32) (matchedResults []matcherResult, err error) { +func (m *matchSequenceInstance) getMatchesForLayer(layerIndex uint32) (matchedResults []matcherResult, err error) { // decide whether to evaluate base or next matcher first baseFirst := m.baseFirst() if baseFirst { - if err := m.evalBase(ctx, layerIndex); err != nil { + if err := m.evalBase(layerIndex); err != nil { return nil, err } } - if err := m.evalNext(ctx, layerIndex); err != nil { + if err := m.evalNext(layerIndex); err != nil { return nil, err } if !baseFirst { - if err := m.evalBase(ctx, layerIndex); err != nil { + if err := m.evalBase(layerIndex); err != nil { return nil, err } } @@ -753,8 +808,8 @@ func (m *matchSequenceInstance) dropIndices(dropIndices []uint32) { // evalBase evaluates the base child matcher and drops map indices from the // next matcher if possible. -func (m *matchSequenceInstance) evalBase(ctx context.Context, layerIndex uint32) error { - results, err := m.baseInstance.getMatchesForLayer(ctx, layerIndex) +func (m *matchSequenceInstance) evalBase(layerIndex uint32) error { + results, err := m.baseInstance.getMatchesForLayer(layerIndex) if err != nil { return fmt.Errorf("failed to evaluate base matcher on layer %d: %v", layerIndex, err) } @@ -781,8 +836,8 @@ func (m *matchSequenceInstance) evalBase(ctx context.Context, layerIndex uint32) // evalNext evaluates the next child matcher and drops map indices from the // base matcher if possible. -func (m *matchSequenceInstance) evalNext(ctx context.Context, layerIndex uint32) error { - results, err := m.nextInstance.getMatchesForLayer(ctx, layerIndex) +func (m *matchSequenceInstance) evalNext(layerIndex uint32) error { + results, err := m.nextInstance.getMatchesForLayer(layerIndex) if err != nil { return fmt.Errorf("failed to evaluate next matcher on layer %d: %v", layerIndex, err) } diff --git a/core/filtermaps/matcher_backend.go b/core/filtermaps/matcher_backend.go deleted file mode 100644 index abd0fd9283..0000000000 --- a/core/filtermaps/matcher_backend.go +++ /dev/null @@ -1,201 +0,0 @@ -// Copyright 2024 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package filtermaps - -import ( - "context" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" -) - -// FilterMapsMatcherBackend implements MatcherBackend. -type FilterMapsMatcherBackend struct { - f *FilterMaps - - // these fields should be accessed under f.matchersLock mutex. - validBlocks common.Range[uint64] - syncCh chan SyncRange -} - -// NewMatcherBackend returns a FilterMapsMatcherBackend after registering it in -// the active matcher set. -// Note that Close should always be called when the matcher is no longer used. -func (f *FilterMaps) NewMatcherBackend() *FilterMapsMatcherBackend { - f.indexLock.RLock() - f.matchersLock.Lock() - defer func() { - f.matchersLock.Unlock() - f.indexLock.RUnlock() - }() - - fm := &FilterMapsMatcherBackend{f: f} - if f.indexedRange.initialized { - fm.validBlocks = f.indexedRange.blocks - } - f.matchers[fm] = struct{}{} - return fm -} - -// GetParams returns the filtermaps parameters. -// GetParams implements MatcherBackend. -func (fm *FilterMapsMatcherBackend) GetParams() *Params { - return &fm.f.Params -} - -// Close removes the matcher from the set of active matchers and ensures that -// any SyncLogIndex calls are cancelled. -// Close implements MatcherBackend. -func (fm *FilterMapsMatcherBackend) Close() { - fm.f.matchersLock.Lock() - defer fm.f.matchersLock.Unlock() - - delete(fm.f.matchers, fm) -} - -// GetFilterMapRows returns the given row of the given map. If the row is empty -// then a non-nil zero length row is returned. If baseLayerOnly is true then -// only the first baseRowLength entries of the row are guaranteed to be -// returned. -// Note that the returned slices should not be modified, they should be copied -// on write. -// GetFilterMapRows implements MatcherBackend. -func (fm *FilterMapsMatcherBackend) GetFilterMapRows(ctx context.Context, mapIndices []uint32, rowIndex uint32, baseLayerOnly bool) ([]FilterRow, error) { - return fm.f.getFilterMapRows(mapIndices, rowIndex, baseLayerOnly) -} - -// GetBlockLvPointer returns the starting log value index where the log values -// generated by the given block are located. If blockNumber is beyond the last -// indexed block then the pointer will point right after this block, ensuring -// that the matcher does not fail and can return a set of results where the -// valid range is correct. -// GetBlockLvPointer implements MatcherBackend. -func (fm *FilterMapsMatcherBackend) GetBlockLvPointer(ctx context.Context, blockNumber uint64) (uint64, error) { - fm.f.indexLock.RLock() - defer fm.f.indexLock.RUnlock() - - if blockNumber >= fm.f.indexedRange.blocks.AfterLast() { - if fm.f.indexedRange.headIndexed { - // return index after head block - return fm.f.indexedRange.headDelimiter + 1, nil - } - if fm.f.indexedRange.blocks.Count() > 0 { - // return index at the beginning of the last, partially indexed - // block (after the last fully indexed one) - blockNumber = fm.f.indexedRange.blocks.Last() - } - } - return fm.f.getBlockLvPointer(blockNumber) -} - -// GetLogByLvIndex returns the log at the given log value index. -// Note that this function assumes that the log index structure is consistent -// with the canonical chain at the point where the given log value index points. -// If this is not the case then an invalid result may be returned or certain -// logs might not be returned at all. -// No error is returned though because of an inconsistency between the chain and -// the log index. It is the caller's responsibility to verify this consistency -// using SyncLogIndex and re-process certain blocks if necessary. -// GetLogByLvIndex implements MatcherBackend. -func (fm *FilterMapsMatcherBackend) GetLogByLvIndex(ctx context.Context, lvIndex uint64) (*types.Log, error) { - fm.f.indexLock.RLock() - defer fm.f.indexLock.RUnlock() - - return fm.f.getLogByLvIndex(lvIndex) -} - -// synced signals to the matcher that has triggered a synchronisation that it -// has been finished and the log index is consistent with the chain head passed -// as a parameter. -// -// Note that if the log index head was far behind the chain head then it might not -// be synced up to the given head in a single step. Still, the latest chain head -// should be passed as a parameter and the existing log index should be consistent -// with that chain. -// -// Note: acquiring the indexLock read lock is unnecessary here, as this function -// is always called within the indexLoop. -func (fm *FilterMapsMatcherBackend) synced() { - fm.f.matchersLock.Lock() - defer fm.f.matchersLock.Unlock() - - indexedBlocks := fm.f.indexedRange.blocks - if !fm.f.indexedRange.headIndexed && !indexedBlocks.IsEmpty() { - indexedBlocks.SetAfterLast(indexedBlocks.Last()) // remove partially indexed last block - } - fm.syncCh <- SyncRange{ - IndexedView: fm.f.indexedView, - ValidBlocks: fm.validBlocks, - IndexedBlocks: indexedBlocks, - } - fm.validBlocks = indexedBlocks - fm.syncCh = nil -} - -// SyncLogIndex ensures that the log index is consistent with the current state -// of the chain and is synced up to the current head. It blocks until this state -// is achieved or the context is cancelled. -// If successful, it returns a SyncRange that contains the latest chain head, -// the indexed range that is currently consistent with the chain and the valid -// range that has not been changed and has been consistent with all states of the -// chain since the previous SyncLogIndex or the creation of the matcher backend. -func (fm *FilterMapsMatcherBackend) SyncLogIndex(ctx context.Context) (SyncRange, error) { - syncCh := make(chan SyncRange, 1) - fm.f.matchersLock.Lock() - fm.syncCh = syncCh - fm.f.matchersLock.Unlock() - - select { - case fm.f.matcherSyncCh <- fm: - case <-ctx.Done(): - return SyncRange{}, ctx.Err() - case <-fm.f.disabledCh: - // Note: acquiring the indexLock read lock is unnecessary here, - // as the indexer has already been terminated. - return SyncRange{IndexedView: fm.f.indexedView}, nil - } - select { - case vr := <-syncCh: - return vr, nil - case <-ctx.Done(): - return SyncRange{}, ctx.Err() - case <-fm.f.disabledCh: - // Note: acquiring the indexLock read lock is unnecessary here, - // as the indexer has already been terminated. - return SyncRange{IndexedView: fm.f.indexedView}, nil - } -} - -// updateMatchersValidRange iterates through active matchers and limits their -// valid range with the current indexed range. This function should be called -// whenever a part of the log index has been removed, before adding new blocks -// to it. -// -// Note: acquiring the indexLock read lock is unnecessary here, as this function -// is always called within the indexLoop. -func (f *FilterMaps) updateMatchersValidRange() { - f.matchersLock.Lock() - defer f.matchersLock.Unlock() - - for fm := range f.matchers { - if !f.indexedRange.initialized { - fm.validBlocks = common.Range[uint64]{} - continue - } - fm.validBlocks = fm.validBlocks.Intersection(f.indexedRange.blocks) - } -} diff --git a/core/filtermaps/matcher_test.go b/core/filtermaps/matcher_test.go index 5a49ce66ec..d8c1747e5e 100644 --- a/core/filtermaps/matcher_test.go +++ b/core/filtermaps/matcher_test.go @@ -16,6 +16,7 @@ package filtermaps +/* import ( "context" crand "crypto/rand" @@ -85,3 +86,4 @@ func TestMatcher(t *testing.T) { } } } +*/ diff --git a/core/filtermaps/math.go b/core/filtermaps/math.go index 68fd6debd6..8453510e5c 100644 --- a/core/filtermaps/math.go +++ b/core/filtermaps/math.go @@ -21,66 +21,59 @@ import ( "encoding/binary" "fmt" "hash/fnv" + "iter" "math" "slices" + "sort" "github.com/ethereum/go-ethereum/common" ) +// FilterRow encodes a single row of a filter map as a list of column indices. +// Note that the values are always stored in the same order as they were added +// and if the same column index is added twice, it is also stored twice. +// Order of column indices and potential duplications do not matter when searching +// for a value but leaving the original order makes reverting to a previous state +// simpler. +type FilterRow []uint32 + // Params defines the basic parameters of the log index structure. type Params struct { - logMapHeight uint // The number of bits required to represent the map height - logMapWidth uint // The number of bits required to represent the map width - logMapsPerEpoch uint // The number of bits required to represent the number of maps per epoch - logValuesPerMap uint // The number of bits required to represent the number of log values per map - - // baseRowLengthRatio represents the ratio of base row length - // to the average row length. - baseRowLengthRatio uint - - // logLayerDiff defines the logarithmic growth factor (base 2) of - // the maximum row length per layer. It indicates how much the maximum - // row length increases as the layer depth increases. - // - // Specifically: - // - the row length in base layer (layer == 0) is baseRowLength - // - the row length in layer x is baseRowLength << (logLayerDiff * x) - logLayerDiff uint + logMapHeight uint // The number of bits required to represent the map height + logMapWidth uint // The number of bits required to represent the map width + logMapsPerEpoch uint // The number of bits required to represent the number of maps per epoch + logValuesPerMap uint // The number of bits required to represent the number of log values per map + logMappingFrequency []uint + maxRowLength []uint32 // These fields can be derived with the information above - mapHeight uint32 // The number of rows in the filter map - mapsPerEpoch uint32 // The number of maps in an epoch - valuesPerMap uint64 // The number of log values marked on each filter map - baseRowLength uint32 // maximum number of log values per row on layer 0 + mapHeight uint32 // The number of rows in the filter map + mapsPerEpoch uint32 // The number of maps in an epoch + valuesPerMap uint64 // The number of log values marked on each filter map - // baseRowGroupSize defines the number of base row entries grouped together - // as a single database entry in the local database to optimize storage - // and retrieval efficiency. - // - // This value can be configured based on the specific implementation. - baseRowGroupSize uint32 + rowGroupSize []uint32 } // DefaultParams is the set of parameters used on mainnet. var DefaultParams = Params{ - logMapHeight: 16, - logMapWidth: 24, - logMapsPerEpoch: 10, - logValuesPerMap: 16, - baseRowGroupSize: 32, - baseRowLengthRatio: 8, - logLayerDiff: 4, + logMapHeight: 16, + logMapWidth: 24, + logMapsPerEpoch: 10, + logValuesPerMap: 16, + logMappingFrequency: []uint{10, 6, 2, 0}, + maxRowLength: []uint32{8, 168, 2728, 10920}, + rowGroupSize: []uint32{256, 16, 1, 1}, } // RangeTestParams puts one log value per epoch, ensuring block exact tail unindexing for testing var RangeTestParams = Params{ - logMapHeight: 4, - logMapWidth: 24, - logMapsPerEpoch: 0, - logValuesPerMap: 0, - baseRowGroupSize: 32, - baseRowLengthRatio: 16, // baseRowLength >= 1 - logLayerDiff: 4, + logMapHeight: 4, + logMapWidth: 24, + logMapsPerEpoch: 0, + logValuesPerMap: 0, + logMappingFrequency: []uint{10, 6, 2, 0}, + maxRowLength: []uint32{8, 168, 2728, 10920}, + rowGroupSize: []uint32{16, 4, 1, 1}, } // deriveFields calculates the derived fields of the parameter set. @@ -88,7 +81,25 @@ func (p *Params) deriveFields() { p.mapHeight = uint32(1) << p.logMapHeight p.mapsPerEpoch = uint32(1) << p.logMapsPerEpoch p.valuesPerMap = uint64(1) << p.logValuesPerMap - p.baseRowLength = uint32(p.valuesPerMap * uint64(p.baseRowLengthRatio) / uint64(p.mapHeight)) +} + +// mapRowIndex calculates the unified storage index where the given row of the +// given map is stored. Note that this indexing scheme is the same as the one +// proposed in EIP-7745 for tree-hashing the filter map structure and for the +// same data proximity reasons it is also suitable for database representation. +// See also: +// https://eips.ethereum.org/EIPS/eip-7745#hash-tree-structure +func (p *Params) mapRowIndex(mapIndex, rowIndex uint32) uint64 { + epochIndex, mapSubIndex := mapIndex>>p.logMapsPerEpoch, mapIndex&(p.mapsPerEpoch-1) + return (uint64(epochIndex)< 32 { // column index stored as uint32 + return fmt.Errorf("invalid configuration: logMapWidth (%d) should not exceed 32", p.logMapWidth) + } + if p.logMapHeight > 16 { // row index stored as uint16 in finishedMap + return fmt.Errorf("invalid configuration: logMapHeight (%d) should not exceed 32", p.logMapHeight) + } + for _, maxRowLength := range p.maxRowLength { + if maxRowLength >= 0x10000 { // index wrap-around issue in finishedMap + return fmt.Errorf("invalid configuration: maxRowLength entry (%d) should not exceed 2**16", maxRowLength) + } + } + for _, groupSize := range p.rowGroupSize { + if groupSize == 0 || (groupSize&(groupSize-1)) != 0 { + return fmt.Errorf("invalid configuration: rowGroupSize entry (%d) must be a power of 2", groupSize) + } + } + if len(p.maxRowLength) != len(p.rowGroupSize) { + return fmt.Errorf("invalid configuration: length of maxRowLength (%d) and rowGroupSize entry (%d) should be equal", len(p.maxRowLength), len(p.rowGroupSize)) } return nil } // mapGroupIndex returns the start index of the base row group that contains the // given map index. Assumes baseRowGroupSize is a power of 2. -func (p *Params) mapGroupIndex(index uint32) uint32 { - return index & ^(p.baseRowGroupSize - 1) +func (p *Params) mapGroupIndex(index, dbLayer uint32) uint32 { + return index & ^(p.rowGroupSize[dbLayer] - 1) } // mapGroupOffset returns the offset of the given map index within its base row group. -func (p *Params) mapGroupOffset(index uint32) uint32 { - return index & (p.baseRowGroupSize - 1) +func (p *Params) mapGroupOffset(index, dbLayer uint32) uint32 { + return index & (p.rowGroupSize[dbLayer] - 1) } // mapEpoch returns the epoch number that the given map index belongs to. @@ -181,38 +208,12 @@ func (p *Params) columnIndex(lvIndex uint64, logValue *common.Hash) uint32 { return uint32(lvIndex%p.valuesPerMap)<>(64-hashBits)) ^ uint32(hash)>>(32-hashBits)) } -// maxRowLength returns the maximum length filter rows are populated up to when -// using the given mapping layer. -// -// A log value can be marked on the map according to a given mapping layer if -// the row mapping on that layer points to a row that has not yet reached the -// maxRowLength belonging to that layer. -// -// This means that a row that is considered full on a given layer may still be -// extended further on a higher order layer. -// -// Each value is marked on the lowest order layer possible, assuming that marks -// are added in ascending log value index order. When searching for a log value -// one should consider all layers and process corresponding rows up until the -// first one where the row mapped to the given layer is not full. -func (p *Params) maxRowLength(layerIndex uint32) uint32 { - logLayerDiff := uint(layerIndex) * p.logLayerDiff - if logLayerDiff > p.logMapsPerEpoch { - logLayerDiff = p.logMapsPerEpoch - } - return p.baseRowLength << logLayerDiff -} - // maskedMapIndex returns the index used for row mapping calculation on the // given layer. On layer zero the mapping changes once per epoch, then the // frequency of re-mapping increases with every new layer until it reaches // the frequency where it is different for every mapIndex. func (p *Params) maskedMapIndex(mapIndex, layerIndex uint32) uint32 { - logLayerDiff := uint(layerIndex) * p.logLayerDiff - if logLayerDiff > p.logMapsPerEpoch { - logLayerDiff = p.logMapsPerEpoch - } - return mapIndex & (uint32(math.MaxUint32) << (p.logMapsPerEpoch - logLayerDiff)) + return mapIndex & (uint32(math.MaxUint32) << p.getLogMappingFrequency(layerIndex)) } // potentialMatches returns the list of log value indices potentially matching @@ -229,7 +230,7 @@ func (p *Params) potentialMatches(rows []FilterRow, mapIndex uint32, logValue co results := make(potentialMatches, 0, 8) mapFirst := uint64(mapIndex) << p.logValuesPerMap for i, row := range rows { - rowLen, maxLen := len(row), int(p.maxRowLength(uint32(i))) + rowLen, maxLen := len(row), int(p.getMaxRowLength(uint32(i))) if rowLen > maxLen { rowLen = maxLen // any additional entries are generated by another log value on a higher mapping layer } @@ -264,3 +265,177 @@ func (p *Params) potentialMatches(rows []FilterRow, mapIndex uint32, logValue co // indices in the filter map range are potential matches. If there are no // potential matches in the given map's range then an empty slice should be used. type potentialMatches []uint64 + +func (p potentialMatches) Len() int { return len(p) } +func (p potentialMatches) Less(i, j int) bool { return p[i] < p[j] } +func (p potentialMatches) Swap(i, j int) { p[i], p[j] = p[j], p[i] } + +type rangeSet[T uint32 | uint64] []common.Range[T] + +func (a rangeSet[T]) includes(v T) bool { + for _, r := range a { + if r.Includes(v) { + return true + } + } + return false +} + +func (a rangeSet[T]) closestLte(v T) (last T, found bool) { + for _, r := range a { + if r.First() > v { + return + } + if r.AfterLast() > v { + return v, true + } + last, found = r.Last(), true + } + return +} + +func (a rangeSet[T]) closestGte(v T) (last T, found bool) { + for _, r := range a { + if r.First() > v { + return r.First(), true + } + if r.AfterLast() > v { + return v, true + } + } + return +} + +type rangeBoundary[T uint32 | uint64] struct { + v T + d int +} + +type rangeBoundaries[T uint32 | uint64] []rangeBoundary[T] + +func (rb *rangeBoundaries[T]) add(r common.Range[T], d int) { + *rb = append((*rb), rangeBoundary[T]{v: r.First(), d: d}, rangeBoundary[T]{v: r.AfterLast(), d: -d}) +} + +func (rb rangeBoundaries[T]) makeSet(threshold int) rangeSet[T] { + res := make(rangeSet[T], 0, len(rb)/2) + sort.Slice(rb, func(i, j int) bool { + return rb[i].v < rb[j].v + }) + var ( + sum int + lastCmp bool + start T + ) + for i, r := range rb { + sum += r.d + cmp := sum >= threshold + if cmp != lastCmp && (i == len(rb)-1 || rb[i+1].v != r.v) { + if cmp { + start = r.v + } else { + res = append(res, common.NewRange[T](start, r.v-start)) + } + lastCmp = cmp + } + } + return res +} + +func (a rangeSet[T]) intersection(b rangeSet[T]) rangeSet[T] { + if len(a) == 0 || len(b) == 0 { + return nil + } + rb := make(rangeBoundaries[T], 0, (len(a)+len(b))*2) + for _, r := range a { + rb.add(r, 1) + } + for _, r := range b { + rb.add(r, 1) + } + return rb.makeSet(2) +} + +func (a rangeSet[T]) exclude(b rangeSet[T]) rangeSet[T] { + if len(a) == 0 { + return nil + } + if len(b) == 0 { + return a + } + rb := make(rangeBoundaries[T], 0, (len(a)+len(b))*2) + for _, r := range a { + rb.add(r, 1) + } + for _, r := range b { + rb.add(r, -1) + } + return rb.makeSet(1) +} + +func (a rangeSet[T]) union(b rangeSet[T]) rangeSet[T] { + if len(a) == 0 { + return b + } + if len(b) == 0 { + return a + } + rb := make(rangeBoundaries[T], 0, (len(a)+len(b))*2) + for _, r := range a { + rb.add(r, 1) + } + for _, r := range b { + rb.add(r, 1) + } + return rb.makeSet(1) +} + +// iter iterates all integers in the range set. +func (r rangeSet[T]) iter() iter.Seq[T] { + return func(yield func(T) bool) { + for _, rr := range r { + for i := range rr.Iter() { + if !yield(i) { + break + } + } + } + } +} + +func (a rangeSet[T]) count() T { + var count T + for _, r := range a { + count += r.Count() + } + return count +} + +func (a rangeSet[T]) singleRange() common.Range[T] { + if len(a) > 1 { + panic("singleRange called for non-continuous rangeSet") + } + if len(a) == 1 { + return a[0] + } + return common.NewRange[T](0, 0) +} + +func singleRangeSet[T uint32 | uint64](r common.Range[T]) rangeSet[T] { + if r.IsEmpty() { + return nil + } + return rangeSet[T]{r} +} + +func (a rangeSet[T]) equal(b rangeSet[T]) bool { + if len(a) != len(b) { + return false + } + for i, r := range a { + if b[i] != r { + return false + } + } + return true +} diff --git a/core/filtermaps/math_test.go b/core/filtermaps/math_test.go index a4c1609059..92d5abe97a 100644 --- a/core/filtermaps/math_test.go +++ b/core/filtermaps/math_test.go @@ -87,7 +87,7 @@ func TestPotentialMatches(t *testing.T) { // split up into a list of rows if longer than allowed var rows []FilterRow for layerIndex := uint32(0); row != nil; layerIndex++ { - maxLen := int(params.maxRowLength(layerIndex)) + maxLen := int(params.getMaxRowLength(layerIndex)) if len(row) > maxLen { rows = append(rows, row[:maxLen]) row = row[maxLen:] diff --git a/core/filtermaps/memory_map.go b/core/filtermaps/memory_map.go new file mode 100644 index 0000000000..516cdfd790 --- /dev/null +++ b/core/filtermaps/memory_map.go @@ -0,0 +1,171 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package filtermaps + +import ( + "slices" + + "github.com/ethereum/go-ethereum/common" +) + +// memoryMap is an in-memory representation of a filter map that represents rows +// as linked lists and stores entries in a single slice. +// memoryMap allows adding new elements and is used for rendering maps. Completed +// maps can be transformed to an immutable finishedMap. +type memoryMap struct { + entries []mmEntry // size = valuesPerMap + rows []mmRow // size = mapHeight + nextEntry uint32 + blockPtrs []uint64 + lastBlock lastBlockOfMap +} + +// mmEntry is a linked list entry. The field next points to the next entry in the +// entries slice. Zero singals the end of the list (entries[0] is always the first +// entry of a list). +type mmEntry struct { + value, next uint32 +} + +// mmRow stores the index of the first and last entry of a linked list in the +// entries slice. If length is zero then first and last are not initialized. +// Note that length could be determined by traversing the linked list but it is +// explicitly stored because map rendering needs quick access to row length. +type mmRow struct { + first, last, length uint32 +} + +// newMemoryMap creates an empty memoryMap. +func (p *Params) newMemoryMap() *memoryMap { + return &memoryMap{ + entries: make([]mmEntry, p.valuesPerMap), + rows: make([]mmRow, p.mapHeight), + } +} + +// clone creates a copy of the map. +func (m *memoryMap) clone() *memoryMap { + return &memoryMap{ + //TODO no need to clone entries if one of the maps will never be changed further. + entries: slices.Clone(m.entries), + rows: slices.Clone(m.rows), + nextEntry: m.nextEntry, + blockPtrs: slices.Clone(m.blockPtrs), + lastBlock: m.lastBlock, + } +} + +func (m *memoryMap) firstBlock() uint64 { + return m.lastBlock.number + 1 - uint64(len(m.blockPtrs)) +} + +func (m *memoryMap) blocks() common.Range[uint64] { + l := uint64(len(m.blockPtrs)) + return common.NewRange[uint64](m.lastBlock.number+1-l, l) +} + +// addToRow adds a new entry to the specified row. +func (m *memoryMap) addToRow(rowIndex, value uint32) { + row := &m.rows[rowIndex] + if row.length == 0 { + row.first = m.nextEntry + } else { + m.entries[row.last].next = m.nextEntry + } + row.last = m.nextEntry + row.length++ + m.entries[m.nextEntry].value = value + m.nextEntry++ +} + +// rowLength returns the length of a row. +func (m *memoryMap) rowLength(rowIndex uint32) uint32 { + return m.rows[rowIndex].length +} + +// getRow returns a row of the map, truncated if maxLen is smaller than the actual +// row length. +func (m *memoryMap) getRow(rowIndex, maxLen uint32) FilterRow { + row := m.rows[rowIndex] + length := min(row.length, maxLen) + res := make(FilterRow, length) + next := row.first + for i := range length { + entry := m.entries[next] + res[i], next = entry.value, entry.next + } + return res +} + +// finishedMap is an immutable memory representation of a single filter map. +// It is more compact and allows more efficient row lookup than memoryMap. +// Note that it assumes params.mapHeight <= 2**16 which is checked in deriveFields. +type finishedMap struct { + rowPtrs []uint16 // points to rowData index after end of row; 2**16 can wrap around to 0 + rowData []uint32 + blockPtrs []uint64 + lastBlock lastBlockOfMap +} + +// finished creates a new finishedMap from a memoryMap. +func (m *memoryMap) finished() *finishedMap { + fm := &finishedMap{ + rowPtrs: make([]uint16, len(m.rows)), + rowData: make([]uint32, m.nextEntry), + blockPtrs: slices.Clone(m.blockPtrs), + lastBlock: m.lastBlock, + } + var ptr uint16 + for i, row := range m.rows { + next := row.first + for range row.length { + entry := m.entries[next] + fm.rowData[ptr] = entry.value + next = entry.next + ptr++ + } + fm.rowPtrs[i] = ptr + } + return fm +} + +func (fm *finishedMap) firstBlock() uint64 { + return fm.lastBlock.number + 1 - uint64(len(fm.blockPtrs)) +} + +func (fm *finishedMap) blocks() common.Range[uint64] { + l := uint64(len(fm.blockPtrs)) + return common.NewRange[uint64](fm.lastBlock.number+1-l, l) +} + +// getRow returns a row of the map, truncated if maxLen is smaller than the actual +// row length. If the row has zero length then it returns a zero length slice. +func (fm *finishedMap) getRow(rowIndex, maxLen uint32) FilterRow { + var start uint16 + if rowIndex > 0 { + start = fm.rowPtrs[rowIndex-1] + } + // Note: uint16 subtraction ensures correct result if + // rowPtrs[rowIndex] has wrapped around to zero. For example if there are + // exactly 2**16 entries and last row has a length of 5 then + // rowPtr[0xFFFE] == 0xFFFB and rowPtr[0xFFFF] == 0x0000 and the uint16 + // subtraction is 0x0000-0xFFFB == 0x0005. + // Also note that it is guaranteed by the filter map design that no single + // row can have a length of 0x10000. + length := fm.rowPtrs[rowIndex] - start + return FilterRow(fm.rowData[start : uint32(start)+min(maxLen, uint32(length))]) +} diff --git a/core/filtermaps/test_helpers.go b/core/filtermaps/test_helpers.go new file mode 100644 index 0000000000..095dac8445 --- /dev/null +++ b/core/filtermaps/test_helpers.go @@ -0,0 +1,226 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package filtermaps + +import ( + "math" + "math/rand" + "reflect" + "slices" + "testing" + + "github.com/ethereum/go-ethereum/common" +) + +const ( + testFilterMapRowsCount = 1000 + testFilterMapsCount = 100 + testPointersCount = 1000 + testMaxBlocksPerMap = 4 +) + +type mapReader struct { + getFilterMapRows func(mapIndices []uint32, rowIndex, layerIndex uint32) ([]FilterRow, error) + getFilterMap func(mapIndex uint32) (*finishedMap, error) + getBlockLvPointer func(blockNumber uint64) (uint64, error) + getLastBlockOfMap func(mapIndex uint32) (uint64, common.Hash, error) +} + +func equalRows(a, b []FilterRow) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if !slices.Equal(a[i], b[i]) { + return false + } + } + return true +} + +func testMapReader(t *testing.T, testCase string, params *Params, reader mapReader, knownEpochs checkpointList, maps []*finishedMap) { + mapRange := common.NewRange[uint32](uint32(len(knownEpochs))*params.mapsPerEpoch, uint32(len(maps))) + for range testFilterMapRowsCount { + rowIndex := uint32(rand.Intn(int(params.mapHeight))) + lastDbLayer := uint32(rand.Intn(len(params.maxRowLength))) + maxLen := params.maxRowLength[lastDbLayer] + mid := mapRange.First() + uint32(rand.Intn(int(mapRange.Count()))) + count := uint32(rand.Intn(int(params.mapsPerEpoch))) + first := max(mid, count/2) - count/2 + testRange := common.NewRange[uint32](first, count) + mapIndices := make([]uint32, 0, testRange.Count()) + expResults := make([]FilterRow, 0, testRange.Count()) + for mapIndex := range testRange.Iter() { + if rand.Intn(2) == 1 { + mapIndices = append(mapIndices, mapIndex) + if mapRange.Includes(mapIndex) { + expResults = append(expResults, maps[mapIndex-mapRange.First()].getRow(rowIndex, maxLen)) + } else { + expResults = append(expResults, nil) + } + } + } + rows, err := reader.getFilterMapRows(mapIndices, rowIndex, lastDbLayer) + if err != nil { + t.Fatalf("Test case %s: error reading %d map indices in range %d <= i < %d: %v", testCase, len(mapIndices), testRange.First(), testRange.AfterLast(), err) + } else if !equalRows(rows, expResults) { + t.Fatalf("Test case %s: incorrect results when reading %d map indices in range %d <= i < %d", testCase, len(mapIndices), testRange.First(), testRange.AfterLast()) + } + } + for range testFilterMapsCount { + mapIndex := mapRange.First() + uint32(rand.Intn(int(mapRange.Count()))) + expResults := maps[mapIndex-mapRange.First()] + fm, err := reader.getFilterMap(mapIndex) + if err != nil { + t.Fatalf("Test case %s: error reading map %d: %v", testCase, mapIndex, err) + } else if !reflect.DeepEqual(fm, expResults) { + t.Fatalf("Test case %s: incorrect results when reading map %d", testCase, mapIndex) + } + } + + testPointers := func(mapIndex uint32, expLastBlock lastBlockOfMap, blockNumber, expBlockPtr uint64) { + lastBlock, lbHash, err := reader.getLastBlockOfMap(mapIndex) + if err != nil { + t.Fatalf("Test case %s: error reading last block of map %d: %v", testCase, mapIndex, err) + } else if (lastBlockOfMap{number: lastBlock, hash: lbHash}) != expLastBlock { + t.Fatalf("Test case %s: incorrect results when reading last block of map %d (expected %v, got %v)", testCase, mapIndex, expLastBlock, lastBlockOfMap{number: lastBlock, hash: lbHash}) + } + if blockNumber != math.MaxUint64 { + blockPtr, err := reader.getBlockLvPointer(blockNumber) + if err != nil { + t.Fatalf("Test case %s: error reading lv pointer of block %d: %v", testCase, blockNumber, err) + } else if blockPtr != expBlockPtr { + t.Fatalf("Test case %s: incorrect results when reading lv pointer of block %d (expected %v, got %v)", testCase, blockNumber, expBlockPtr, blockPtr) + } + } + } + testNoPointer := func(mapIndex uint32) { + if _, _, err := reader.getLastBlockOfMap(mapIndex); err == nil { + t.Fatalf("Test case %s: unexpected last block of map pointer at map %d", testCase, mapIndex) + } + } + testKnownEpoch := func(epoch uint32) { + testNoPointer(params.firstEpochMap(epoch)) + testPointers(params.lastEpochMap(epoch), lastBlockOfMap{number: knownEpochs[epoch].BlockNumber, hash: knownEpochs[epoch].BlockHash}, knownEpochs[epoch].BlockNumber, knownEpochs[epoch].FirstIndex) + } + if len(knownEpochs) > 0 { + for range testPointersCount { + epoch := uint32(rand.Intn(len(knownEpochs))) + testKnownEpoch(epoch) + } + testKnownEpoch(uint32(len(knownEpochs) - 1)) + } + testMapPointers := func(mapIndex uint32, blockPos int) { // 0: first 1: random 2: last + fm := maps[mapIndex-mapRange.First()] + if len(fm.blockPtrs) == 0 { + testPointers(mapIndex, fm.lastBlock, math.MaxUint64, 0) + } else { + var subIndex int + switch blockPos { + case 1: + subIndex = rand.Intn(len(fm.blockPtrs)) + case 2: + subIndex = len(fm.blockPtrs) - 1 + } + testPointers(mapIndex, fm.lastBlock, fm.firstBlock()+uint64(subIndex), fm.blockPtrs[subIndex]) + } + } + if !mapRange.IsEmpty() { + testMapPointers(mapRange.First(), 0) + for range testPointersCount { + testMapPointers(mapRange.First()+uint32(rand.Intn(int(mapRange.Count()))), 1) + } + testMapPointers(mapRange.Last(), 2) + } + testNoPointer(mapRange.AfterLast()) + testNoPointer(params.lastEpochMap(params.mapEpoch(mapRange.AfterLast()))) + testNoPointer(params.firstEpochMap(params.mapEpoch(mapRange.AfterLast()) + 1)) +} + +func generateTestMaps(params *Params, maps []*finishedMap, amount uint32) []*finishedMap { + var lastBlock lastBlockOfMap + genesis := len(maps) == 0 + if !genesis { + lastBlock = maps[len(maps)-1].lastBlock + } + for range amount { + blockCount := rand.Intn(testMaxBlocksPerMap + 1) + if genesis && blockCount == 0 { + blockCount = 1 + } + fm := &finishedMap{ + rowPtrs: make([]uint16, params.mapHeight), + rowData: make([]uint32, params.valuesPerMap), + blockPtrs: make([]uint64, blockCount), + } + addValues := params.valuesPerMap + for addValues > 0 { + addToRow := uint64(rand.Intn(int(min(addValues+1, params.valuesPerMap/2)))) + addValues -= addToRow + var rowIndex uint32 + for { + rowIndex = uint32(rand.Intn(int(params.mapHeight))) + if uint64(fm.rowPtrs[rowIndex])+addToRow <= params.valuesPerMap/2 { + break + } + } + fm.rowPtrs[rowIndex] += uint16(addToRow) + } + var ptr uint16 + for i, r := range fm.rowPtrs { + ptr += r + fm.rowPtrs[i] = ptr + } + for i := range fm.rowData { + fm.rowData[i] = uint32(rand.Intn(1 << params.logMapWidth)) + } + if blockCount > 0 { + startPtr := uint64(len(maps)) * params.valuesPerMap + for i := range fm.blockPtrs { + if genesis { + genesis = false + } else { + lastBlock.number++ + fm.blockPtrs[i] = startPtr + uint64(i)*params.valuesPerMap/uint64(blockCount) + uint64(rand.Intn(int(params.valuesPerMap)/blockCount/2)) + } + } + rand.Read(lastBlock.hash[:]) + } + fm.lastBlock = lastBlock + maps = append(maps, fm) + } + return maps +} + +func generateEpochCheckpoint(mapIndex uint32, maps []*finishedMap) epochCheckpoint { + for len(maps[mapIndex].blockPtrs) == 0 { + mapIndex-- + } + return epochCheckpoint{ + BlockNumber: maps[mapIndex].lastBlock.number, + BlockHash: maps[mapIndex].lastBlock.hash, + FirstIndex: maps[mapIndex].blockPtrs[len(maps[mapIndex].blockPtrs)-1], + } +} + +func generateTestCheckpoints(params *Params, maps []*finishedMap) checkpointList { + cpList := make(checkpointList, uint32(len(maps))/params.mapsPerEpoch) + for i := range cpList { + cpList[i] = generateEpochCheckpoint(params.lastEpochMap(uint32(i)), maps) + } + return cpList +} diff --git a/core/rawdb/accessors_indexes.go b/core/rawdb/accessors_indexes.go index a725f144d4..32f44e1c8d 100644 --- a/core/rawdb/accessors_indexes.go +++ b/core/rawdb/accessors_indexes.go @@ -320,12 +320,12 @@ func ReadCanonicalRawReceipt(db ethdb.Reader, blockHash common.Hash, blockNumber // same data proximity reasons it is also suitable for database representation. // See also: // https://eips.ethereum.org/EIPS/eip-7745#hash-tree-structure -func ReadFilterMapExtRow(db ethdb.KeyValueReader, mapRowIndex uint64, bitLength uint) ([]uint32, error) { +func ReadFilterMapSingleRow(db ethdb.KeyValueReader, mapRowIndex uint64, dbLayer uint32, bitLength uint) ([]uint32, error) { byteLength := int(bitLength) / 8 if int(bitLength) != byteLength*8 { panic("invalid bit length") } - key := filterMapRowKey(mapRowIndex, false) + key := filterMapRowKey(mapRowIndex, dbLayer) has, err := db.Has(key) if err != nil { return nil, err @@ -349,18 +349,19 @@ func ReadFilterMapExtRow(db ethdb.KeyValueReader, mapRowIndex uint64, bitLength return row, nil } -func ReadFilterMapBaseRows(db ethdb.KeyValueReader, mapRowIndex uint64, rowCount uint32, bitLength uint) ([][]uint32, error) { +func ReadFilterMapRowGroup(db ethdb.KeyValueReader, mapRowIndex uint64, dbLayer, rowCount uint32, bitLength uint) ([][]uint32, error) { byteLength := int(bitLength) / 8 if int(bitLength) != byteLength*8 { panic("invalid bit length") } - key := filterMapRowKey(mapRowIndex, true) + key := filterMapRowKey(mapRowIndex, dbLayer) has, err := db.Has(key) if err != nil { return nil, err } rows := make([][]uint32, rowCount) if !has { + //fmt.Println(" no entry") return rows, nil } encRows, err := db.Get(key) @@ -368,6 +369,7 @@ func ReadFilterMapBaseRows(db ethdb.KeyValueReader, mapRowIndex uint64, rowCount return nil, err } encLen := len(encRows) + //fmt.Println(" encLen", encLen) var ( entryCount, entriesInRow, rowIndex, headerLen, headerBits int headerByte byte @@ -411,7 +413,7 @@ func ReadFilterMapBaseRows(db ethdb.KeyValueReader, mapRowIndex uint64, rowCount // WriteFilterMapExtRow stores an extended filter map row at the given mapRowIndex // or deletes any existing entry if the row is empty. -func WriteFilterMapExtRow(db ethdb.KeyValueWriter, mapRowIndex uint64, row []uint32, bitLength uint) { +func WriteFilterMapSingleRow(db ethdb.KeyValueWriter, mapRowIndex uint64, dbLayer uint32, row []uint32, bitLength uint) { byteLength := int(bitLength) / 8 if int(bitLength) != byteLength*8 { panic("invalid bit length") @@ -424,16 +426,24 @@ func WriteFilterMapExtRow(db ethdb.KeyValueWriter, mapRowIndex uint64, row []uin binary.LittleEndian.PutUint32(b[:], c) copy(encRow[i*byteLength:(i+1)*byteLength], b[:byteLength]) } - err = db.Put(filterMapRowKey(mapRowIndex, false), encRow) + err = db.Put(filterMapRowKey(mapRowIndex, dbLayer), encRow) } else { - err = db.Delete(filterMapRowKey(mapRowIndex, false)) + err = db.Delete(filterMapRowKey(mapRowIndex, dbLayer)) } if err != nil { log.Crit("Failed to store extended filter map row", "err", err) } } -func WriteFilterMapBaseRows(db ethdb.KeyValueWriter, mapRowIndex uint64, rows [][]uint32, bitLength uint) { +func WriteFilterMapRowGroup(db ethdb.KeyValueWriter, mapRowIndex uint64, dbLayer uint32, rows [][]uint32, bitLength uint) { + /*var totalsize int //TODO + for _, row := range rows { + totalsize += len(row) + } + if totalsize > 0 { + fmt.Println("WriteFilterMapRowGroup", mapRowIndex, dbLayer, totalsize) + }*/ + byteLength := int(bitLength) / 8 if int(bitLength) != byteLength*8 { panic("invalid bit length") @@ -476,9 +486,10 @@ func WriteFilterMapBaseRows(db ethdb.KeyValueWriter, mapRowIndex uint64, rows [] addHeaderBit(false) zeroBits-- } - err = db.Put(filterMapRowKey(mapRowIndex, true), encRows) + //fmt.Println(" write encLen", len(encRows)) + err = db.Put(filterMapRowKey(mapRowIndex, dbLayer), encRows) } else { - err = db.Delete(filterMapRowKey(mapRowIndex, true)) + err = db.Delete(filterMapRowKey(mapRowIndex, dbLayer)) } if err != nil { log.Crit("Failed to store base filter map rows", "err", err) @@ -486,7 +497,7 @@ func WriteFilterMapBaseRows(db ethdb.KeyValueWriter, mapRowIndex uint64, rows [] } func DeleteFilterMapRows(db ethdb.KeyValueStore, mapRows common.Range[uint64], hashScheme bool, stopCallback func(bool) bool) error { - return SafeDeleteRange(db, filterMapRowKey(mapRows.First(), false), filterMapRowKey(mapRows.AfterLast(), false), hashScheme, stopCallback) + return SafeDeleteRange(db, filterMapRowKey(mapRows.First(), 0), filterMapRowKey(mapRows.AfterLast(), 0), hashScheme, stopCallback) } // ReadFilterMapLastBlock retrieves the number of the block that generated the @@ -565,12 +576,9 @@ func DeleteBlockLvPointers(db ethdb.KeyValueStore, blocks common.Range[uint64], // FilterMapsRange is a storage representation of the block range covered by the // filter maps structure and the corresponting log value index range. type FilterMapsRange struct { - Version uint32 - HeadIndexed bool - HeadDelimiter uint64 - BlocksFirst, BlocksAfterLast uint64 - MapsFirst, MapsAfterLast uint32 - TailPartialEpoch uint32 + Version uint32 + ValidMaps, DirtyMaps []uint32 + KnownEpochs uint32 } // ReadFilterMapsRange retrieves the filter maps range data. Note that if the diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index d9140c5fd6..b01e800f28 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -152,7 +152,7 @@ var ( SyncCommitteeKey = []byte("committee-") // bigEndian64(syncPeriod) -> serialized committee // new log index - filterMapsPrefix = "fm-" + filterMapsPrefix = "fm*" //TODO fm- filterMapsRangeKey = []byte(filterMapsPrefix + "R") filterMapRowPrefix = []byte(filterMapsPrefix + "r") // filterMapRowPrefix + mapRowIndex (uint64 big endian) -> filter row filterMapLastBlockPrefix = []byte(filterMapsPrefix + "b") // filterMapLastBlockPrefix + mapIndex (uint32 big endian) -> block number (uint64 big endian) @@ -353,15 +353,12 @@ func IsStorageTrieNode(key []byte) bool { } // filterMapRowKey = filterMapRowPrefix + mapRowIndex (uint64 big endian) -func filterMapRowKey(mapRowIndex uint64, base bool) []byte { - extLen := 8 - if base { - extLen = 9 - } +func filterMapRowKey(mapRowIndex uint64, dbLayer uint32) []byte { l := len(filterMapRowPrefix) - key := make([]byte, l+extLen) + key := make([]byte, l+9) copy(key[:l], filterMapRowPrefix) binary.BigEndian.PutUint64(key[l:l+8], mapRowIndex) + key[l+8] = byte(dbLayer) return key } diff --git a/eth/api_backend.go b/eth/api_backend.go index 766a99fc1e..50af177dbe 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -459,7 +459,7 @@ func (b *EthAPIBackend) RPCTxFeeCap() float64 { return b.eth.config.RPCTxFeeCap } -func (b *EthAPIBackend) CurrentView() *filtermaps.ChainView { +func (b *EthAPIBackend) CurrentChainView() *filtermaps.ChainView { head := b.eth.blockchain.CurrentBlock() if head == nil { return nil @@ -467,8 +467,8 @@ func (b *EthAPIBackend) CurrentView() *filtermaps.ChainView { return filtermaps.NewChainView(b.eth.blockchain, head.Number.Uint64(), head.Hash()) } -func (b *EthAPIBackend) NewMatcherBackend() filtermaps.MatcherBackend { - return b.eth.filterMaps.NewMatcherBackend() +func (b *EthAPIBackend) GetIndexView(headBlockHash common.Hash) *filtermaps.IndexView { + return b.eth.logIndexer.GetIndexView(headBlockHash) } func (b *EthAPIBackend) Engine() consensus.Engine { diff --git a/eth/backend.go b/eth/backend.go index 8509561822..ee5bb7330f 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -108,8 +108,7 @@ type Ethereum struct { engine consensus.Engine accountManager *accounts.Manager - filterMaps *filtermaps.FilterMaps - closeFilterMaps chan chan struct{} + logIndexer *filtermaps.Indexer APIBackend *EthAPIBackend @@ -285,18 +284,8 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { ExportFileName: config.LogExportCheckpoints, HashScheme: scheme == rawdb.HashScheme, } - chainView := eth.newChainView(eth.blockchain.CurrentBlock()) - historyCutoff, _ := eth.blockchain.HistoryPruningCutoff() - var finalBlock uint64 - if fb := eth.blockchain.CurrentFinalBlock(); fb != nil { - finalBlock = fb.Number.Uint64() - } - filterMaps, err := filtermaps.NewFilterMaps(chainDb, chainView, historyCutoff, finalBlock, filtermaps.DefaultParams, fmConfig) - if err != nil { - return nil, err - } - eth.filterMaps = filterMaps - eth.closeFilterMaps = make(chan chan struct{}) + eth.logIndexer = filtermaps.NewIndexer(chainDb, filtermaps.DefaultParams, fmConfig) + eth.blockchain.RegisterIndexer(eth.logIndexer, "log index") // TxPool if config.TxPool.Journal != "" { @@ -452,71 +441,9 @@ func (s *Ethereum) Start() error { // Start the connection manager s.dropper.Start(s.p2pServer, func() bool { return !s.Synced() }) - - // start log indexer - s.filterMaps.Start() - go s.updateFilterMapsHeads() return nil } -func (s *Ethereum) newChainView(head *types.Header) *filtermaps.ChainView { - if head == nil { - return nil - } - return filtermaps.NewChainView(s.blockchain, head.Number.Uint64(), head.Hash()) -} - -func (s *Ethereum) updateFilterMapsHeads() { - headEventCh := make(chan core.ChainEvent, 10) - blockProcCh := make(chan bool, 10) - sub := s.blockchain.SubscribeChainEvent(headEventCh) - sub2 := s.blockchain.SubscribeBlockProcessingEvent(blockProcCh) - defer func() { - sub.Unsubscribe() - sub2.Unsubscribe() - for { - select { - case <-headEventCh: - case <-blockProcCh: - default: - return - } - } - }() - - var head *types.Header - setHead := func(newHead *types.Header) { - if newHead == nil { - return - } - if head == nil || newHead.Hash() != head.Hash() { - head = newHead - chainView := s.newChainView(head) - historyCutoff, _ := s.blockchain.HistoryPruningCutoff() - var finalBlock uint64 - if fb := s.blockchain.CurrentFinalBlock(); fb != nil { - finalBlock = fb.Number.Uint64() - } - s.filterMaps.SetTarget(chainView, historyCutoff, finalBlock) - } - } - setHead(s.blockchain.CurrentBlock()) - - for { - select { - case ev := <-headEventCh: - setHead(ev.Header) - case blockProc := <-blockProcCh: - s.filterMaps.SetBlockProcessing(blockProc) - case <-time.After(time.Second * 10): - setHead(s.blockchain.CurrentBlock()) - case ch := <-s.closeFilterMaps: - close(ch) - return - } - } -} - func (s *Ethereum) setupDiscovery() error { eth.StartENRUpdater(s.blockchain, s.p2pServer.LocalNode()) @@ -575,10 +502,6 @@ func (s *Ethereum) Stop() error { s.handler.Stop() // Then stop everything else. - ch := make(chan struct{}) - s.closeFilterMaps <- ch - <-ch - s.filterMaps.Stop() s.txPool.Close() s.blockchain.Stop() s.engine.Close() diff --git a/eth/filters/filter.go b/eth/filters/filter.go index 422e5cd67b..a8e3396896 100644 --- a/eth/filters/filter.go +++ b/eth/filters/filter.go @@ -19,9 +19,11 @@ package filters import ( "context" "errors" + "fmt" "math" "math/big" "slices" + "sync" "time" "github.com/ethereum/go-ethereum/common" @@ -43,7 +45,12 @@ type Filter struct { block *common.Hash // Block hash if filtering a single block begin, end int64 // Range interval if filtering multiple blocks - rangeLogsTestHook chan rangeLogsTestEvent + testFilterRanges []testFilterRange +} + +type testFilterRange struct { + begin, end uint64 + indexed bool } // NewRangeFilter creates a new filter which uses a bloom filter on blocks to @@ -146,272 +153,101 @@ func (f *Filter) Logs(ctx context.Context) ([]*types.Log, error) { return f.rangeLogs(ctx, begin, end) } -const ( - rangeLogsTestDone = iota // zero range - rangeLogsTestSync // before sync; zero range - rangeLogsTestSynced // after sync; valid blocks range - rangeLogsTestIndexed // individual search range - rangeLogsTestUnindexed // individual search range - rangeLogsTestResults // results range after search iteration - rangeLogsTestReorg // results range trimmed by reorg -) - -type rangeLogsTestEvent struct { - event int - blocks common.Range[uint64] -} - -// searchSession represents a single search session. -type searchSession struct { - ctx context.Context - filter *Filter - mb filtermaps.MatcherBackend - syncRange filtermaps.SyncRange // latest synchronized state with the matcher - chainView *filtermaps.ChainView // can be more recent than the indexed view in syncRange - // block ranges always refer to the current chainView - firstBlock, lastBlock uint64 // specified search range; MaxUint64 means latest block - searchRange common.Range[uint64] // actual search range; end trimmed to latest head - matchRange common.Range[uint64] // range in which we have results (subset of searchRange) - matches []*types.Log // valid set of matches in matchRange - forceUnindexed bool // revert to unindexed search -} - -// newSearchSession returns a new searchSession. -func newSearchSession(ctx context.Context, filter *Filter, mb filtermaps.MatcherBackend, firstBlock, lastBlock uint64) (*searchSession, error) { - s := &searchSession{ - ctx: ctx, - filter: filter, - mb: mb, - firstBlock: firstBlock, - lastBlock: lastBlock, - } - // enforce a consistent state before starting the search in order to be able - // to determine valid range later - var err error - s.syncRange, err = s.mb.SyncLogIndex(s.ctx) - if err != nil { - return nil, err - } - if err := s.updateChainView(); err != nil { - return nil, err - } - return s, nil -} - -// updateChainView updates to the latest view of the underlying chain and sets -// searchRange by replacing MaxUint64 (meaning latest block) with actual head -// number in the specified search range. -// If the session already had an existing chain view and set of matches then -// it also trims part of the match set that a chain reorg might have invalidated. -func (s *searchSession) updateChainView() error { - // update chain view based on current chain head (might be more recent than - // the indexed view of syncRange as the indexer updates it asynchronously - // with some delay - newChainView := s.filter.sys.backend.CurrentView() - if newChainView == nil { - return errors.New("head block not available") - } - head := newChainView.HeadNumber() - - // update actual search range based on current head number - firstBlock, lastBlock := s.firstBlock, s.lastBlock - if firstBlock == math.MaxUint64 { - firstBlock = head - } - if lastBlock == math.MaxUint64 { - lastBlock = head - } - if firstBlock > lastBlock || lastBlock > head { - return errInvalidBlockRange - } - s.searchRange = common.NewRange(firstBlock, lastBlock+1-firstBlock) - - // Trim existing match set in case a reorg may have invalidated some results - if !s.matchRange.IsEmpty() { - trimRange := newChainView.SharedRange(s.chainView).Intersection(s.searchRange) - s.matchRange, s.matches = s.trimMatches(trimRange, s.matchRange, s.matches) - } - s.chainView = newChainView - return nil -} - -// trimMatches removes any entries from the specified set of matches that is -// outside the given range. -func (s *searchSession) trimMatches(trimRange, matchRange common.Range[uint64], matches []*types.Log) (common.Range[uint64], []*types.Log) { - newRange := matchRange.Intersection(trimRange) - if newRange == matchRange { - return matchRange, matches - } - if newRange.IsEmpty() { - return newRange, nil - } - for len(matches) > 0 && matches[0].BlockNumber < newRange.First() { - matches = matches[1:] - } - for len(matches) > 0 && matches[len(matches)-1].BlockNumber > newRange.Last() { - matches = matches[:len(matches)-1] - } - return newRange, matches -} - -// searchInRange performs a single range search, either indexed or unindexed. -func (s *searchSession) searchInRange(r common.Range[uint64], indexed bool) (common.Range[uint64], []*types.Log, error) { - if indexed { - if s.filter.rangeLogsTestHook != nil { - s.filter.rangeLogsTestHook <- rangeLogsTestEvent{rangeLogsTestIndexed, r} - } - results, err := s.filter.indexedLogs(s.ctx, s.mb, r.First(), r.Last()) - if err != nil && !errors.Is(err, filtermaps.ErrMatchAll) { - return common.Range[uint64]{}, nil, err - } - if err == nil { - // sync with filtermaps matcher - if s.filter.rangeLogsTestHook != nil { - s.filter.rangeLogsTestHook <- rangeLogsTestEvent{rangeLogsTestSync, common.Range[uint64]{}} - } - var syncErr error - if s.syncRange, syncErr = s.mb.SyncLogIndex(s.ctx); syncErr != nil { - return common.Range[uint64]{}, nil, syncErr - } - if s.filter.rangeLogsTestHook != nil { - s.filter.rangeLogsTestHook <- rangeLogsTestEvent{rangeLogsTestSynced, s.syncRange.ValidBlocks} - } - // discard everything that might be invalid - trimRange := s.syncRange.ValidBlocks.Intersection(s.chainView.SharedRange(s.syncRange.IndexedView)) - matchRange, matches := s.trimMatches(trimRange, r, results) - return matchRange, matches, nil - } - // "match all" filters are not supported by filtermaps; fall back to - // unindexed search which is the most efficient in this case - s.forceUnindexed = true - // fall through to unindexed case - } - if s.filter.rangeLogsTestHook != nil { - s.filter.rangeLogsTestHook <- rangeLogsTestEvent{rangeLogsTestUnindexed, r} - } - matches, err := s.filter.unindexedLogs(s.ctx, s.chainView, r.First(), r.Last()) - if err != nil { - return common.Range[uint64]{}, nil, err - } - return r, matches, nil -} - -// doSearchIteration performs a search on a range missing from an incomplete set -// of results, adds the new section and removes invalidated entries. -func (s *searchSession) doSearchIteration() error { - switch { - case s.matchRange.IsEmpty(): - // no results yet; try search in entire range - indexedSearchRange := s.searchRange.Intersection(s.syncRange.IndexedBlocks) - if s.forceUnindexed = indexedSearchRange.IsEmpty(); !s.forceUnindexed { - // indexed search on the intersection of indexed and searched range - matchRange, matches, err := s.searchInRange(indexedSearchRange, true) - if err != nil { - return err - } - s.matchRange = matchRange - s.matches = matches - return nil - } else { - // no intersection of indexed and searched range; unindexed search on - // the whole searched range - matchRange, matches, err := s.searchInRange(s.searchRange, false) - if err != nil { - return err - } - s.matchRange = matchRange - s.matches = matches - return nil - } - - case !s.matchRange.IsEmpty() && s.matchRange.First() > s.searchRange.First(): - // Results are available, but the tail section is missing. Perform an unindexed - // search for the missing tail, while still allowing indexed search for the head. - // - // The unindexed search is necessary because the tail portion of the indexes - // has been pruned. - tailRange := common.NewRange(s.searchRange.First(), s.matchRange.First()-s.searchRange.First()) - _, tailMatches, err := s.searchInRange(tailRange, false) - if err != nil { - return err - } - s.matches = append(tailMatches, s.matches...) - s.matchRange = tailRange.Union(s.matchRange) - return nil - - case !s.matchRange.IsEmpty() && s.matchRange.First() == s.searchRange.First() && s.searchRange.AfterLast() > s.matchRange.AfterLast(): - // Results are available, but the head section is missing. Try to perform - // the indexed search for the missing head, or fallback to unindexed search - // if the tail portion of indexed range has been pruned. - headRange := common.NewRange(s.matchRange.AfterLast(), s.searchRange.AfterLast()-s.matchRange.AfterLast()) - if !s.forceUnindexed { - indexedHeadRange := headRange.Intersection(s.syncRange.IndexedBlocks) - if !indexedHeadRange.IsEmpty() && indexedHeadRange.First() == headRange.First() { - headRange = indexedHeadRange - } else { - // The tail portion of the indexes has been pruned, falling back - // to unindexed search. - s.forceUnindexed = true - } - } - headMatchRange, headMatches, err := s.searchInRange(headRange, !s.forceUnindexed) - if err != nil { - return err - } - if headMatchRange.First() != s.matchRange.AfterLast() { - // improbable corner case, first part of new head range invalidated by tail unindexing - s.matches, s.matchRange = headMatches, headMatchRange - return nil - } - s.matches = append(s.matches, headMatches...) - s.matchRange = s.matchRange.Union(headMatchRange) - return nil - - default: - panic("invalid search session state") - } -} - func (f *Filter) rangeLogs(ctx context.Context, firstBlock, lastBlock uint64) ([]*types.Log, error) { - if f.rangeLogsTestHook != nil { - defer func() { - f.rangeLogsTestHook <- rangeLogsTestEvent{rangeLogsTestDone, common.Range[uint64]{}} - close(f.rangeLogsTestHook) - }() - } - if firstBlock > lastBlock { return nil, nil } - mb := f.sys.backend.NewMatcherBackend() - defer mb.Close() + chainView := f.sys.backend.CurrentChainView() - session, err := newSearchSession(ctx, f, mb, firstBlock, lastBlock) + if firstBlock > chainView.HeadNumber() { + firstBlock = chainView.HeadNumber() + } + if lastBlock > chainView.HeadNumber() { + lastBlock = chainView.HeadNumber() + } + + indexView := f.sys.backend.GetIndexView(chainView.BlockHash(chainView.HeadNumber())) + if indexView == nil { + return f.unindexedLogs(ctx, chainView, firstBlock, lastBlock) + } + searchRange := common.NewRange[uint64](firstBlock, lastBlock+1-firstBlock) + indexedRange := indexView.BlockRange().Intersection(searchRange) + if indexedRange.IsEmpty() { + indexView.Release() + return f.unindexedLogs(ctx, chainView, firstBlock, lastBlock) + } + var ( + res1, res2, res3 []*types.Log + err error + ) + res2, err = f.indexedLogs(ctx, chainView, indexView, indexedRange.First(), indexedRange.Last()) + indexView.Release() + if err == filtermaps.ErrMatchAll { + return f.unindexedLogs(ctx, chainView, firstBlock, lastBlock) + } if err != nil { return nil, err } - for session.searchRange != session.matchRange { - if err := session.doSearchIteration(); err != nil { + if searchRange.First() < indexedRange.First() { + res1, err = f.unindexedLogs(ctx, chainView, searchRange.First(), indexedRange.First()-1) + if err != nil { return nil, err } - if f.rangeLogsTestHook != nil { - f.rangeLogsTestHook <- rangeLogsTestEvent{rangeLogsTestResults, session.matchRange} - } - mr := session.matchRange - if err := session.updateChainView(); err != nil { - return nil, err - } - if f.rangeLogsTestHook != nil && session.matchRange != mr { - f.rangeLogsTestHook <- rangeLogsTestEvent{rangeLogsTestReorg, session.matchRange} - } } - return session.matches, nil + if indexedRange.Last() < searchRange.Last() { + res3, err = f.unindexedLogs(ctx, chainView, indexedRange.AfterLast(), searchRange.Last()) + if err != nil { + return nil, err + } + } + return append(append(res1, res2...), res3...), nil } -func (f *Filter) indexedLogs(ctx context.Context, mb filtermaps.MatcherBackend, begin, end uint64) ([]*types.Log, error) { +func (f *Filter) indexedLogs(ctx context.Context, chainView *filtermaps.ChainView, indexView *filtermaps.IndexView, begin, end uint64) ([]*types.Log, error) { + if f.testFilterRanges != nil { + f.testFilterRanges = append(f.testFilterRanges, testFilterRange{begin: begin, end: end, indexed: true}) + } start := time.Now() - potentialMatches, err := filtermaps.GetPotentialMatches(ctx, mb, begin, end, f.addresses, f.topics) - matches := filterLogs(potentialMatches, nil, nil, f.addresses, f.topics) + potentialMatches, err := filtermaps.GetPotentialMatches(ctx, indexView, begin, end, f.addresses, f.topics) + potentialLogs := make([]*types.Log, len(potentialMatches)) + indexCh := make(chan int) + errCh := make(chan error, 1) + var wg sync.WaitGroup + + worker := func() { + defer wg.Done() + for i := range indexCh { + logPosition := potentialMatches[i] + receipts := chainView.Receipts(logPosition.BlockNumber) + if receipts == nil { + select { + case errCh <- fmt.Errorf("receipts for block #%d not found", logPosition.BlockNumber): + default: + } + return + } + log, err := logPosition.GetLog(receipts) + if err != nil { + select { + case errCh <- err: + default: + } + return + } + potentialLogs[i] = log + } + } + + for range 4 { //TODO + wg.Add(1) + go worker() + } + for i := range potentialMatches { + indexCh <- i + } + close(indexCh) + wg.Wait() + matches := filterLogs(potentialLogs, nil, nil, f.addresses, f.topics) log.Trace("Performed indexed log search", "begin", begin, "end", end, "true matches", len(matches), "false positives", len(potentialMatches)-len(matches), "elapsed", common.PrettyDuration(time.Since(start))) return matches, err } @@ -419,6 +255,9 @@ func (f *Filter) indexedLogs(ctx context.Context, mb filtermaps.MatcherBackend, // unindexedLogs returns the logs matching the filter criteria based on raw block // iteration and bloom matching. func (f *Filter) unindexedLogs(ctx context.Context, chainView *filtermaps.ChainView, begin, end uint64) ([]*types.Log, error) { + if f.testFilterRanges != nil { + f.testFilterRanges = append(f.testFilterRanges, testFilterRange{begin: begin, end: end, indexed: false}) + } start := time.Now() log.Debug("Performing unindexed log search", "begin", begin, "end", end) var matches []*types.Log @@ -517,7 +356,7 @@ func filterLogs(logs []*types.Log, fromBlock, toBlock *big.Int, addresses []comm } var ret []*types.Log for _, log := range logs { - if check(log) { + if log != nil && check(log) { ret = append(ret, log) } } diff --git a/eth/filters/filter_system.go b/eth/filters/filter_system.go index f10e6a277b..3cada008bf 100644 --- a/eth/filters/filter_system.go +++ b/eth/filters/filter_system.go @@ -72,8 +72,8 @@ type Backend interface { SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription - CurrentView() *filtermaps.ChainView - NewMatcherBackend() filtermaps.MatcherBackend + CurrentChainView() *filtermaps.ChainView + GetIndexView(headBlockHash common.Hash) *filtermaps.IndexView } // FilterSystem holds resources shared by all filters. diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index e5a1a2b25f..8eb9f76c8e 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -42,7 +42,7 @@ import ( type testBackend struct { db ethdb.Database - fm *filtermaps.FilterMaps + logIndexer *filtermaps.Indexer txFeed event.Feed logsFeed event.Feed rmLogsFeed event.Feed @@ -160,31 +160,51 @@ func (b *testBackend) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subsc return b.chainFeed.Subscribe(ch) } -func (b *testBackend) CurrentView() *filtermaps.ChainView { +func (b *testBackend) CurrentChainView() *filtermaps.ChainView { head := b.CurrentBlock() return filtermaps.NewChainView(b, head.Number.Uint64(), head.Hash()) } -func (b *testBackend) NewMatcherBackend() filtermaps.MatcherBackend { - return b.fm.NewMatcherBackend() +func (b *testBackend) GetIndexView(headBlockHash common.Hash) *filtermaps.IndexView { + return b.logIndexer.GetIndexView(headBlockHash) } func (b *testBackend) startFilterMaps(history uint64, disabled bool, params filtermaps.Params) { head := b.CurrentBlock() - chainView := filtermaps.NewChainView(b, head.Number.Uint64(), head.Hash()) config := filtermaps.Config{ History: history, Disabled: disabled, ExportFileName: "", } - b.fm, _ = filtermaps.NewFilterMaps(b.db, chainView, 0, 0, params, config) - b.fm.Start() - b.fm.WaitIdle() + b.logIndexer = filtermaps.NewIndexer(b.db, params, config) + if !disabled { + b.indexHead(head.Number.Uint64()) + } +} + +func (b *testBackend) indexHead(head uint64) { + header, _ := b.HeaderByNumber(context.Background(), rpc.BlockNumber(head)) + receipts, _ := b.GetReceipts(context.Background(), header.Hash()) + b.logIndexer.AddBlockData([]*types.Header{header}, []types.Receipts{receipts}) + b.backfillBlocks() +} + +func (b *testBackend) backfillBlocks() { + ready, needBlocks := b.logIndexer.Status() + for !ready || !needBlocks.IsEmpty() { + for !ready { + time.Sleep(time.Millisecond) + ready, needBlocks = b.logIndexer.Status() + } + header, _ := b.HeaderByNumber(context.Background(), rpc.BlockNumber(needBlocks.First())) + receipts, _ := b.GetReceipts(context.Background(), header.Hash()) + ready, needBlocks = b.logIndexer.AddBlockData([]*types.Header{header}, []types.Receipts{receipts}) + } } func (b *testBackend) stopFilterMaps() { - b.fm.Stop() - b.fm = nil + b.logIndexer.Stop() + b.logIndexer = nil } func (b *testBackend) setPending(block *types.Block, receipts types.Receipts) { diff --git a/eth/filters/filter_test.go b/eth/filters/filter_test.go index edec3e027f..64a5032657 100644 --- a/eth/filters/filter_test.go +++ b/eth/filters/filter_test.go @@ -20,6 +20,7 @@ import ( "context" "encoding/json" "math/big" + "slices" "strings" "testing" "time" @@ -431,173 +432,45 @@ func TestRangeLogs(t *testing.T) { t.Fatal(err) } chain, _ := core.GenerateChain(gspec.Config, gspec.ToBlock(), ethash.NewFaker(), db, 1000, func(i int, gen *core.BlockGen) {}) - options := core.DefaultConfig().WithStateScheme(rawdb.HashScheme) options.TxLookupLimit = 0 // index all txs bc, err := core.NewBlockChain(db, gspec, ethash.NewFaker(), options) if err != nil { t.Fatal(err) } - _, err = bc.InsertChain(chain[:600]) - if err != nil { - t.Fatal(err) - } - backend.startFilterMaps(200, false, filtermaps.RangeTestParams) - defer backend.stopFilterMaps() - - var ( - testCase, event int - filter *Filter - addresses = []common.Address{{}} - ) - - expEvent := func(expEvent int, expFirst, expAfterLast uint64) { - exp := rangeLogsTestEvent{expEvent, common.NewRange[uint64](expFirst, expAfterLast-expFirst)} - event++ - ev := <-filter.rangeLogsTestHook - if ev != exp { - t.Fatalf("Test case #%d: wrong test event #%d received (got %v, expected %v)", testCase, event, ev, exp) + rangeTest := func(testCase int, begin, end int64, expRanges []testFilterRange) { + filter := sys.NewRangeFilter(begin, end, []common.Address{{}}, nil) + filter.testFilterRanges = []testFilterRange{} + filter.Logs(context.Background()) + if !slices.Equal(filter.testFilterRanges, expRanges) { + t.Fatalf("Test case #%d: wrong list of filtered ranges received (got %v, expected %v)", testCase, filter.testFilterRanges, expRanges) } } - newFilter := func(begin, end int64) { - testCase++ - event = 0 - filter = sys.NewRangeFilter(begin, end, addresses, nil) - filter.rangeLogsTestHook = make(chan rangeLogsTestEvent) - go func(filter *Filter) { - filter.Logs(context.Background()) - // ensure that filter will not be blocked if we exit early - for range filter.rangeLogsTestHook { - } - }(filter) - } + bc.InsertChain(chain[:600]) + backend.startFilterMaps(200, false, filtermaps.RangeTestParams) + defer backend.stopFilterMaps() - updateHead := func() { - head := bc.CurrentBlock() - backend.fm.SetTarget(filtermaps.NewChainView(backend, head.Number.Uint64(), head.Hash()), 0, 0) - backend.fm.WaitIdle() - } + rangeTest(1, 0, 300, []testFilterRange{{0, 300, false}}) + rangeTest(2, 300, 500, []testFilterRange{{400, 500, true}, {300, 399, false}}) + rangeTest(3, 400, 10000, []testFilterRange{{400, 600, true}}) + bc.SetCanonical(chain[99]) + backend.logIndexer.Revert(100) + rangeTest(4, 0, 10000, []testFilterRange{{0, 100, false}}) + backend.backfillBlocks() + bc.InsertChain(chain[100:120]) + backend.indexHead(120) + rangeTest(5, 0, 10000, []testFilterRange{{0, 120, true}}) + bc.InsertChain(chain[120:800]) + backend.indexHead(800) + rangeTest(6, 400, 800, []testFilterRange{{600, 800, true}, {400, 599, false}}) + backend.stopFilterMaps() - // test case #1 - newFilter(300, 500) - expEvent(rangeLogsTestIndexed, 401, 501) - expEvent(rangeLogsTestSync, 0, 0) - expEvent(rangeLogsTestSynced, 401, 601) - expEvent(rangeLogsTestResults, 401, 501) - expEvent(rangeLogsTestUnindexed, 300, 401) - if _, err := bc.InsertChain(chain[600:700]); err != nil { - t.Fatal(err) - } - updateHead() - expEvent(rangeLogsTestResults, 300, 501) - expEvent(rangeLogsTestDone, 0, 0) + backend.startFilterMaps(0, true, filtermaps.RangeTestParams) + rangeTest(7, 0, 10000, []testFilterRange{{0, 800, false}}) + backend.stopFilterMaps() - // test case #2 - newFilter(400, int64(rpc.LatestBlockNumber)) - expEvent(rangeLogsTestIndexed, 501, 701) - if _, err := bc.InsertChain(chain[700:800]); err != nil { - t.Fatal(err) - } - updateHead() - expEvent(rangeLogsTestSync, 0, 0) - expEvent(rangeLogsTestSynced, 601, 699) - expEvent(rangeLogsTestResults, 601, 699) - expEvent(rangeLogsTestUnindexed, 400, 601) - expEvent(rangeLogsTestResults, 400, 699) - expEvent(rangeLogsTestIndexed, 699, 801) - if _, err := bc.SetCanonical(chain[749]); err != nil { // set head to block 750 - t.Fatal(err) - } - updateHead() - expEvent(rangeLogsTestSync, 0, 0) - expEvent(rangeLogsTestSynced, 601, 749) - expEvent(rangeLogsTestResults, 400, 749) - expEvent(rangeLogsTestIndexed, 749, 751) - expEvent(rangeLogsTestSync, 0, 0) - expEvent(rangeLogsTestSynced, 551, 751) - expEvent(rangeLogsTestResults, 400, 751) - expEvent(rangeLogsTestDone, 0, 0) - - // test case #3 - newFilter(int64(rpc.LatestBlockNumber), int64(rpc.LatestBlockNumber)) - expEvent(rangeLogsTestIndexed, 750, 751) - if _, err := bc.SetCanonical(chain[739]); err != nil { - t.Fatal(err) - } - updateHead() - expEvent(rangeLogsTestSync, 0, 0) - expEvent(rangeLogsTestSynced, 551, 739) - expEvent(rangeLogsTestResults, 0, 0) - expEvent(rangeLogsTestIndexed, 740, 741) - if _, err := bc.InsertChain(chain[740:750]); err != nil { - t.Fatal(err) - } - updateHead() - expEvent(rangeLogsTestSync, 0, 0) - expEvent(rangeLogsTestSynced, 551, 739) - expEvent(rangeLogsTestResults, 0, 0) - expEvent(rangeLogsTestIndexed, 750, 751) - expEvent(rangeLogsTestSync, 0, 0) - expEvent(rangeLogsTestSynced, 551, 751) - expEvent(rangeLogsTestResults, 750, 751) - expEvent(rangeLogsTestDone, 0, 0) - - // test case #4 - if _, err := bc.SetCanonical(chain[499]); err != nil { - t.Fatal(err) - } - updateHead() - newFilter(400, int64(rpc.LatestBlockNumber)) - expEvent(rangeLogsTestIndexed, 400, 501) - if _, err := bc.InsertChain(chain[500:650]); err != nil { - t.Fatal(err) - } - updateHead() - expEvent(rangeLogsTestSync, 0, 0) - expEvent(rangeLogsTestSynced, 451, 499) - expEvent(rangeLogsTestResults, 451, 499) - expEvent(rangeLogsTestUnindexed, 400, 451) - expEvent(rangeLogsTestResults, 400, 499) - // indexed head extension seems possible - expEvent(rangeLogsTestIndexed, 499, 651) - // further head extension causes tail unindexing in searched range - if _, err := bc.InsertChain(chain[650:750]); err != nil { - t.Fatal(err) - } - updateHead() - expEvent(rangeLogsTestSync, 0, 0) - expEvent(rangeLogsTestSynced, 551, 649) - // tail trimmed to 551; cannot merge with existing results - expEvent(rangeLogsTestResults, 551, 649) - expEvent(rangeLogsTestUnindexed, 400, 551) - expEvent(rangeLogsTestResults, 400, 649) - expEvent(rangeLogsTestIndexed, 649, 751) - expEvent(rangeLogsTestSync, 0, 0) - expEvent(rangeLogsTestSynced, 551, 751) - expEvent(rangeLogsTestResults, 400, 751) - expEvent(rangeLogsTestDone, 0, 0) - - // test case #5 - newFilter(400, int64(rpc.LatestBlockNumber)) - expEvent(rangeLogsTestIndexed, 551, 751) - expEvent(rangeLogsTestSync, 0, 0) - expEvent(rangeLogsTestSynced, 551, 751) - expEvent(rangeLogsTestResults, 551, 751) - expEvent(rangeLogsTestUnindexed, 400, 551) - if _, err := bc.InsertChain(chain[750:1000]); err != nil { - t.Fatal(err) - } - updateHead() - expEvent(rangeLogsTestResults, 400, 751) - // indexed tail already beyond results head; revert to unindexed head search - expEvent(rangeLogsTestUnindexed, 751, 1001) - if _, err := bc.SetCanonical(chain[899]); err != nil { - t.Fatal(err) - } - updateHead() - expEvent(rangeLogsTestResults, 400, 1001) - expEvent(rangeLogsTestReorg, 400, 901) - expEvent(rangeLogsTestDone, 0, 0) + backend.startFilterMaps(0, false, filtermaps.RangeTestParams) + rangeTest(8, 0, 10000, []testFilterRange{{0, 800, true}}) } diff --git a/internal/ethapi/api_test.go b/internal/ethapi/api_test.go index aaa002b5ec..2c12a927ec 100644 --- a/internal/ethapi/api_test.go +++ b/internal/ethapi/api_test.go @@ -690,10 +690,10 @@ func (b testBackend) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) func (b testBackend) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription { panic("implement me") } -func (b testBackend) CurrentView() *filtermaps.ChainView { +func (b testBackend) CurrentChainView() *filtermaps.ChainView { panic("implement me") } -func (b testBackend) NewMatcherBackend() filtermaps.MatcherBackend { +func (b testBackend) GetIndexView(headBlockHash common.Hash) *filtermaps.IndexView { panic("implement me") } diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index af3d592b82..db2399cc02 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -99,8 +99,8 @@ type Backend interface { SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription - CurrentView() *filtermaps.ChainView - NewMatcherBackend() filtermaps.MatcherBackend + CurrentChainView() *filtermaps.ChainView + GetIndexView(headBlockHash common.Hash) *filtermaps.IndexView } func GetAPIs(apiBackend Backend) []rpc.API { diff --git a/internal/ethapi/transaction_args_test.go b/internal/ethapi/transaction_args_test.go index 30791f32b5..dc59ed531c 100644 --- a/internal/ethapi/transaction_args_test.go +++ b/internal/ethapi/transaction_args_test.go @@ -407,7 +407,7 @@ func (b *backendMock) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) func (b *backendMock) Engine() consensus.Engine { return nil } -func (b *backendMock) CurrentView() *filtermaps.ChainView { return nil } -func (b *backendMock) NewMatcherBackend() filtermaps.MatcherBackend { return nil } +func (b *backendMock) CurrentChainView() *filtermaps.ChainView { return nil } +func (b *backendMock) GetIndexView(headBlockHash common.Hash) *filtermaps.IndexView { return nil } func (b *backendMock) HistoryPruningCutoff() uint64 { return 0 }