diff --git a/common/range.go b/common/range.go
index c3a26ea7f5..aefd0c16d9 100644
--- a/common/range.go
+++ b/common/range.go
@@ -27,7 +27,11 @@ type Range[T uint32 | uint64] struct {
// NewRange creates a new range based of first element and number of elements.
func NewRange[T uint32 | uint64](first, count T) Range[T] {
- return Range[T]{first, first + count}
+ afterLast := first + count
+ if afterLast < first {
+ panic("range overflow")
+ }
+ return Range[T]{first, afterLast}
}
// First returns the first element of the range.
@@ -97,6 +101,12 @@ func (r Range[T]) Intersection(q Range[T]) Range[T] {
// Union returns the union of two ranges. Panics for gapped ranges.
func (r Range[T]) Union(q Range[T]) Range[T] {
+ if r.IsEmpty() {
+ return q
+ }
+ if q.IsEmpty() {
+ return r
+ }
if max(r.first, q.first) > min(r.afterLast, q.afterLast) {
panic("cannot create union; gap between ranges")
}
diff --git a/core/filtermaps/chain_view.go b/core/filtermaps/chain_view.go
index 35c5ed22a5..bbc60fe313 100644
--- a/core/filtermaps/chain_view.go
+++ b/core/filtermaps/chain_view.go
@@ -32,12 +32,11 @@ type blockchain interface {
GetRawReceipts(hash common.Hash, number uint64) types.Receipts
}
-// ChainView represents an immutable view of a chain with a block id and a set
+// ChainView represents an immutable view of a chain with a block hash and a set
// of receipts associated to each block number and a block hash associated with
// all block numbers except the head block. This is because in the future
// ChainView might represent a view where the head block is currently being
-// created. Block id is a unique identifier that can also be calculated for the
-// head block.
+// created.
// Note that the view's head does not have to be the current canonical head
// of the underlying blockchain, it should only possess the block headers
// and receipts up until the expected chain view head.
@@ -79,21 +78,6 @@ func (cv *ChainView) BlockHash(number uint64) common.Hash {
return cv.blockHash(number)
}
-// BlockId returns the unique block id belonging to the given block number.
-// Note that it is currently equal to the block hash. In the future it might
-// be a different id for future blocks if the log index root becomes part of
-// consensus and therefore rendering the index with the new head will happen
-// before the hash of that new head is available.
-func (cv *ChainView) BlockId(number uint64) common.Hash {
- cv.lock.Lock()
- defer cv.lock.Unlock()
-
- if number > cv.headNumber {
- panic("invalid block number")
- }
- return cv.blockHash(number)
-}
-
// Header returns the block header at the given block number.
func (cv *ChainView) Header(number uint64) *types.Header {
return cv.chain.GetHeader(cv.BlockHash(number), number)
@@ -128,7 +112,7 @@ func (cv *ChainView) SharedRange(cv2 *ChainView) common.Range[uint64] {
return common.Range[uint64]{}
}
sharedLen := min(cv.headNumber, cv2.headNumber) + 1
- for sharedLen > 0 && cv.BlockId(sharedLen-1) != cv2.BlockId(sharedLen-1) {
+ for sharedLen > 0 && cv.BlockHash(sharedLen-1) != cv2.BlockHash(sharedLen-1) {
sharedLen--
}
return common.NewRange(0, sharedLen)
@@ -139,7 +123,7 @@ func equalViews(cv1, cv2 *ChainView) bool {
if cv1 == nil || cv2 == nil {
return false
}
- return cv1.headNumber == cv2.headNumber && cv1.BlockId(cv1.headNumber) == cv2.BlockId(cv2.headNumber)
+ return cv1.headNumber == cv2.headNumber && cv1.BlockHash(cv1.headNumber) == cv2.BlockHash(cv2.headNumber)
}
// matchViews returns true if the two chain views are equivalent up until the
@@ -152,12 +136,7 @@ func matchViews(cv1, cv2 *ChainView, number uint64) bool {
if cv1.headNumber < number || cv2.headNumber < number {
return false
}
- var h1, h2 common.Hash
- if number == cv1.headNumber || number == cv2.headNumber {
- h1, h2 = cv1.BlockId(number), cv2.BlockId(number)
- } else {
- h1, h2 = cv1.BlockHash(number), cv2.BlockHash(number)
- }
+ h1, h2 := cv1.BlockHash(number), cv2.BlockHash(number)
return h1 == h2 && h1 != common.Hash{}
}
diff --git a/core/filtermaps/checkpoints.go b/core/filtermaps/checkpoints.go
index face4ea5d3..a3d86d685e 100644
--- a/core/filtermaps/checkpoints.go
+++ b/core/filtermaps/checkpoints.go
@@ -33,7 +33,7 @@ type checkpointList []epochCheckpoint
// be initialized at the epoch boundary.
type epochCheckpoint struct {
BlockNumber uint64 // block that generated the last log value of the given epoch
- BlockId common.Hash
+ BlockHash common.Hash
FirstIndex uint64 // first log value index of the given block
}
diff --git a/core/filtermaps/checkpoints_holesky.json b/core/filtermaps/checkpoints_holesky.json
index 481c71a114..31948d48b3 100644
--- a/core/filtermaps/checkpoints_holesky.json
+++ b/core/filtermaps/checkpoints_holesky.json
@@ -1,28 +1,28 @@
[
-{"blockNumber": 814410, "blockId": "0x6c38f0d4ff2c23ae187f581cebb0963c0ec2dbf051b289de1582c369c98a3244", "firstIndex": 67107349},
-{"blockNumber": 914268, "blockId": "0xeff161573a11eb6e2ab930674a5245b2c5ffc5e9e63093503344ec6fa60578a3", "firstIndex": 134216064},
-{"blockNumber": 1048866, "blockId": "0xebd3b95415dad9ab7f2b1d25e48c791e299063c09427bbde54217029c3e215ad", "firstIndex": 201325914},
-{"blockNumber": 1144433, "blockId": "0x0c895438ef12a2c835b4372c209e40a499ba2b18ed0e658846813784de391392", "firstIndex": 268434935},
-{"blockNumber": 1230406, "blockId": "0xd3512e7241efc9853e46b1ad565403d302f62457b6dd84917c31ff375fabf487", "firstIndex": 335544096},
-{"blockNumber": 1309104, "blockId": "0xcf108c6e002e7a5657bf7587adde03edf5f20d03e95b66a194eb8080daaf4f97", "firstIndex": 402653143},
-{"blockNumber": 1380499, "blockId": "0x984b0f8b6bf06f1240bbca8c1df9bef3880bedafd5b5c2a55cbc2f64c9efb974", "firstIndex": 469759822},
-{"blockNumber": 1476950, "blockId": "0x8323b528bb8d80a96b172de92452be0f45078a9ca311cb4be6675f089e25a426", "firstIndex": 536870335},
-{"blockNumber": 1533506, "blockId": "0x737dc416070aa3b522a0c2ab813a70c1820f062c718f27597394f309f0004106", "firstIndex": 603979618},
-{"blockNumber": 1613764, "blockId": "0x9d6a5a505afdda86b1ebcc2b10cb687a1f89c2e2b74315945a3ddc6a0b3c9cd6", "firstIndex": 671088253},
-{"blockNumber": 1719073, "blockId": "0x17bf6a51d9c55a908c3e9e652f80b2aa464b049c697abf3bd5a537e461aff0b9", "firstIndex": 738197366},
-{"blockNumber": 1973133, "blockId": "0x14b288d70e688de3334d09283670301aacfed21c193da8a56fd63767f8177705", "firstIndex": 805305624},
-{"blockNumber": 2274761, "blockId": "0xf02790b273b04219e001d2fcc93e8e47708cb228364e96ad754bc8050d08a9aa", "firstIndex": 872414871},
-{"blockNumber": 2530470, "blockId": "0x157ea98b1a7e66dd3f045da8e25219800671f9a000211d3d94006efd33d89a80", "firstIndex": 939523234},
-{"blockNumber": 2781706, "blockId": "0xa723663a584e280700d2302eba03d1b3b6549333db70c6152576d33e2911f594", "firstIndex": 1006632936},
-{"blockNumber": 3101669, "blockId": "0xa6e66a18fed64d33178c7088f273c404be597662c34f6e6884cf2e24f3ea4ace", "firstIndex": 1073741353},
-{"blockNumber": 3221725, "blockId": "0xe771f897dece48b1583cc1d1d10de8015da57407eb1fdf239fdbe46eaab85143", "firstIndex": 1140850137},
-{"blockNumber": 3357164, "blockId": "0x6252d0aa54c79623b0680069c88d7b5c47983f0d5c4845b6c811b8d9b5e8ff3c", "firstIndex": 1207959453},
-{"blockNumber": 3447019, "blockId": "0xeb7d585e1e063f3cc05ed399fbf6c2df63c271f62f030acb804e9fb1e74b6dc1", "firstIndex": 1275067542},
-{"blockNumber": 3546397, "blockId": "0xdabdef7defa4281180a57c5af121877b82274f15ccf074ea0096146f4c246df2", "firstIndex": 1342176778},
-{"blockNumber": 3867885, "blockId": "0x8be069dd7a3e2ffb869ee164d11b28555233d2510b134ab9d5484fdae55d2225", "firstIndex": 1409285539},
-{"blockNumber": 3935446, "blockId": "0xc91a61bc215bbcccc3020c62e5c8153162df0d8bcc59813d74671b2d24903ed7", "firstIndex": 1476394742},
-{"blockNumber": 3989508, "blockId": "0xc85dec36a767e44237842ef51915944c2a49780c8c394a3aa6cfb013c99cf58b", "firstIndex": 1543503452},
-{"blockNumber": 4057078, "blockId": "0xccdb79f6705629cb6ab1667a1244938f60911236549143fcff23a3989213e67e", "firstIndex": 1610612030},
-{"blockNumber": 4126499, "blockId": "0x92f2ef21fc911e87e81e38373d5f2915587b9648a0ab3cf4fcfe3e5aaffe7b85", "firstIndex": 1677720416},
-{"blockNumber": 4239335, "blockId": "0x64fbd22965eb583a584552b7edb9b7ce26fb6aad247c1063d0d5a4d11cbcc58c", "firstIndex": 1744830176}
+{"blockNumber": 814410, "blockHash": "0x6c38f0d4ff2c23ae187f581cebb0963c0ec2dbf051b289de1582c369c98a3244", "firstIndex": 67107349},
+{"blockNumber": 914268, "blockHash": "0xeff161573a11eb6e2ab930674a5245b2c5ffc5e9e63093503344ec6fa60578a3", "firstIndex": 134216064},
+{"blockNumber": 1048866, "blockHash": "0xebd3b95415dad9ab7f2b1d25e48c791e299063c09427bbde54217029c3e215ad", "firstIndex": 201325914},
+{"blockNumber": 1144433, "blockHash": "0x0c895438ef12a2c835b4372c209e40a499ba2b18ed0e658846813784de391392", "firstIndex": 268434935},
+{"blockNumber": 1230406, "blockHash": "0xd3512e7241efc9853e46b1ad565403d302f62457b6dd84917c31ff375fabf487", "firstIndex": 335544096},
+{"blockNumber": 1309104, "blockHash": "0xcf108c6e002e7a5657bf7587adde03edf5f20d03e95b66a194eb8080daaf4f97", "firstIndex": 402653143},
+{"blockNumber": 1380499, "blockHash": "0x984b0f8b6bf06f1240bbca8c1df9bef3880bedafd5b5c2a55cbc2f64c9efb974", "firstIndex": 469759822},
+{"blockNumber": 1476950, "blockHash": "0x8323b528bb8d80a96b172de92452be0f45078a9ca311cb4be6675f089e25a426", "firstIndex": 536870335},
+{"blockNumber": 1533506, "blockHash": "0x737dc416070aa3b522a0c2ab813a70c1820f062c718f27597394f309f0004106", "firstIndex": 603979618},
+{"blockNumber": 1613764, "blockHash": "0x9d6a5a505afdda86b1ebcc2b10cb687a1f89c2e2b74315945a3ddc6a0b3c9cd6", "firstIndex": 671088253},
+{"blockNumber": 1719073, "blockHash": "0x17bf6a51d9c55a908c3e9e652f80b2aa464b049c697abf3bd5a537e461aff0b9", "firstIndex": 738197366},
+{"blockNumber": 1973133, "blockHash": "0x14b288d70e688de3334d09283670301aacfed21c193da8a56fd63767f8177705", "firstIndex": 805305624},
+{"blockNumber": 2274761, "blockHash": "0xf02790b273b04219e001d2fcc93e8e47708cb228364e96ad754bc8050d08a9aa", "firstIndex": 872414871},
+{"blockNumber": 2530470, "blockHash": "0x157ea98b1a7e66dd3f045da8e25219800671f9a000211d3d94006efd33d89a80", "firstIndex": 939523234},
+{"blockNumber": 2781706, "blockHash": "0xa723663a584e280700d2302eba03d1b3b6549333db70c6152576d33e2911f594", "firstIndex": 1006632936},
+{"blockNumber": 3101669, "blockHash": "0xa6e66a18fed64d33178c7088f273c404be597662c34f6e6884cf2e24f3ea4ace", "firstIndex": 1073741353},
+{"blockNumber": 3221725, "blockHash": "0xe771f897dece48b1583cc1d1d10de8015da57407eb1fdf239fdbe46eaab85143", "firstIndex": 1140850137},
+{"blockNumber": 3357164, "blockHash": "0x6252d0aa54c79623b0680069c88d7b5c47983f0d5c4845b6c811b8d9b5e8ff3c", "firstIndex": 1207959453},
+{"blockNumber": 3447019, "blockHash": "0xeb7d585e1e063f3cc05ed399fbf6c2df63c271f62f030acb804e9fb1e74b6dc1", "firstIndex": 1275067542},
+{"blockNumber": 3546397, "blockHash": "0xdabdef7defa4281180a57c5af121877b82274f15ccf074ea0096146f4c246df2", "firstIndex": 1342176778},
+{"blockNumber": 3867885, "blockHash": "0x8be069dd7a3e2ffb869ee164d11b28555233d2510b134ab9d5484fdae55d2225", "firstIndex": 1409285539},
+{"blockNumber": 3935446, "blockHash": "0xc91a61bc215bbcccc3020c62e5c8153162df0d8bcc59813d74671b2d24903ed7", "firstIndex": 1476394742},
+{"blockNumber": 3989508, "blockHash": "0xc85dec36a767e44237842ef51915944c2a49780c8c394a3aa6cfb013c99cf58b", "firstIndex": 1543503452},
+{"blockNumber": 4057078, "blockHash": "0xccdb79f6705629cb6ab1667a1244938f60911236549143fcff23a3989213e67e", "firstIndex": 1610612030},
+{"blockNumber": 4126499, "blockHash": "0x92f2ef21fc911e87e81e38373d5f2915587b9648a0ab3cf4fcfe3e5aaffe7b85", "firstIndex": 1677720416},
+{"blockNumber": 4239335, "blockHash": "0x64fbd22965eb583a584552b7edb9b7ce26fb6aad247c1063d0d5a4d11cbcc58c", "firstIndex": 1744830176}
]
diff --git a/core/filtermaps/checkpoints_mainnet.json b/core/filtermaps/checkpoints_mainnet.json
index 2ea065ddb7..bb84950f86 100644
--- a/core/filtermaps/checkpoints_mainnet.json
+++ b/core/filtermaps/checkpoints_mainnet.json
@@ -1,292 +1,292 @@
[
-{"blockNumber": 4166212, "blockId": "0xd94b724fc1c7dceb3251b51b81f7a0f3220ae5b9add1e62917004f14f2a3532c", "firstIndex": 67108840},
-{"blockNumber": 4513996, "blockId": "0xc0fd25fef5888609d05fa8c5620d8e18c31cdd675dd4ee926ff966668f0e5d76", "firstIndex": 134217480},
-{"blockNumber": 4817399, "blockId": "0x8573c3166fb7f716e97cf6109d382f86441238e0b85828944a072ae8583a6cfa", "firstIndex": 201326547},
-{"blockNumber": 5087706, "blockId": "0xcd2dfae45901e299b25faa04c10df60983d5c0a3d0e478e789c7aeec980cf691", "firstIndex": 268435422},
-{"blockNumber": 5306085, "blockId": "0xe8026984c85873f2b24018a7e0bdf8c2df136b2fc49666b4addf0d2e067890da", "firstIndex": 335544018},
-{"blockNumber": 5509898, "blockId": "0x41bc63dc8184f9a7d4b9a5a554e00eee32fee60ccfa2339264be11270ac28de1", "firstIndex": 402652789},
-{"blockNumber": 5670367, "blockId": "0xe61c2ba463f2f458817ed1bf0ff9fb9d07e9d7354f46b48175b2d784b817bf3b", "firstIndex": 469761896},
-{"blockNumber": 5826113, "blockId": "0xfbbb1fc5e03a5562bf6b7f1799d7a572d9ef66c7fd0e0b9f4fed7262767b5c86", "firstIndex": 536870852},
-{"blockNumber": 5953008, "blockId": "0x6bbdccd094928e846de3c615a33f2952e3259afaa1929f4ee241a56907d0593b", "firstIndex": 603979150},
-{"blockNumber": 6102812, "blockId": "0x2403f2502088a1fa26c1294f5245032fe3b7f3a8bb14c12d0ba8d577156bbc1b", "firstIndex": 671087962},
-{"blockNumber": 6276672, "blockId": "0xeaa05a0e0574fbb164244628a7d14d9d47f1692941098aa737059d44104401df", "firstIndex": 738197399},
-{"blockNumber": 6448662, "blockId": "0xe6f097fa18a2cef425514e863659e42e1aa8d74df49cb0bed5877e82a08d1f84", "firstIndex": 805306056},
-{"blockNumber": 6655925, "blockId": "0xee5c9d87b06dc0c2e850fead07bcd13f85d45ad6b5a6e7ebee53cc8618772786", "firstIndex": 872415229},
-{"blockNumber": 6873878, "blockId": "0x9a0ea103146da21fa1d9cab7ff609ec2c5e7f1856288a1f744239588e33904c2", "firstIndex": 939524080},
-{"blockNumber": 7080893, "blockId": "0x35ad45055d7a1a3ef94efeb05e4122757bcc126ff1bc2b4ac72dc78ab3c0fd75", "firstIndex": 1006632327},
-{"blockNumber": 7266955, "blockId": "0x7b80f70520f2c16eb5890f4963af8f60446f50af82b589ceaf009d09e704b302", "firstIndex": 1073741608},
-{"blockNumber": 7466649, "blockId": "0xea8980b5c692c729013f9cc4c9f66e94d5ef2c5caad23f370be0953b6d85f32c", "firstIndex": 1140850335},
-{"blockNumber": 7661736, "blockId": "0x615cdb03dbf29a22d59bbc769c03f3647c2c9abb7d1767f3f6529708209e7073", "firstIndex": 1207959232},
-{"blockNumber": 7834494, "blockId": "0x05b7dd13e2eb8a833e4b278f9151ca01b294ae5fc5617769422ec85ffb446215", "firstIndex": 1275068098},
-{"blockNumber": 7989998, "blockId": "0x4b5026add7f2fc2c02993b82c61bceba1b75cfcb8a78c5112cc2832bc0413be6", "firstIndex": 1342177025},
-{"blockNumber": 8142950, "blockId": "0x9f56b7753a0e9964973f3e096e1385215e3aef232b448359e5bec05046b016a3", "firstIndex": 1409285545},
-{"blockNumber": 8297598, "blockId": "0x2ddfeb161fe34ff73c4a995bb94256c3b522b39348169edb97bb3d876ccdf77d", "firstIndex": 1476394898},
-{"blockNumber": 8465061, "blockId": "0xfdbe5435da5ae9fb34a9154e5ebe515afac9a128e383f798f1e69952be404fcd", "firstIndex": 1543503805},
-{"blockNumber": 8655740, "blockId": "0x305137325a39a44aba997e214c05f32d965658f7e3fe20322f2e57e72f239977", "firstIndex": 1610612406},
-{"blockNumber": 8807102, "blockId": "0xc32f9fd9a43d2b502dbcc4a683ffe324fca9a871753478bcb1f4fb9573da96a5", "firstIndex": 1677721343},
-{"blockNumber": 8911110, "blockId": "0x806eabee7dc429b57afcb6c0b72b61728b4f13c62ee9942f1813790463f9ab5b", "firstIndex": 1744830172},
-{"blockNumber": 8960262, "blockId": "0xa69f24032e9d5761565e726d484926a0932c6ca7fdb4c04a55bfc5e8f5879c4d", "firstIndex": 1811938729},
-{"blockNumber": 9085906, "blockId": "0x922b4d6aeca64517f5048fa1ffa98d3e813bc415716a1763d07cf2bdea236896", "firstIndex": 1879048062},
-{"blockNumber": 9230838, "blockId": "0xd892f513c48c95a859ae59c4e00ec4be0d684c549456c90e9c36669c11092f50", "firstIndex": 1946156580},
-{"blockNumber": 9390449, "blockId": "0xcc0ced78fa66b570f338d129794d574560a23958e28d0d5288003d4542cec734", "firstIndex": 2013264376},
-{"blockNumber": 9515554, "blockId": "0xc06d04737813c3e2c8e910b2d4543c5c684c7bfe235bb501445f68fd0663f3d9", "firstIndex": 2080374749},
-{"blockNumber": 9659367, "blockId": "0x765fa33d9418b1ed648e88b497b4bf4aa66990e6413dfc155525cda61e3cc813", "firstIndex": 2147483344},
-{"blockNumber": 9793940, "blockId": "0x9527dcbd85a100c51aee4ba90828fee23312f5f041859f3416a54fd87a437ce2", "firstIndex": 2214591669},
-{"blockNumber": 9923756, "blockId": "0xc4da5fdd2de077459fe84f721d7fc01ef69683e03e634b0c5443186486792246", "firstIndex": 2281700949},
-{"blockNumber": 10042325, "blockId": "0x6f554f195d20e4ba1262f73cf2c1cd44b14a71befe5ef0a5318fc309f8cde9fa", "firstIndex": 2348810131},
-{"blockNumber": 10168768, "blockId": "0xdcdf88ec2c197d552343c80d01375a35d6462aff39c5bd976160f7176d2add64", "firstIndex": 2415918723},
-{"blockNumber": 10289467, "blockId": "0x5ece23babcf6b8838695b891b044c3dcf8120cdc15acd83018e96ffc590f6cd0", "firstIndex": 2483027720},
-{"blockNumber": 10386601, "blockId": "0xd398ee129aea1247019fa39dc2f5083819687d8f5ad17ec39ba1803d178fbcda", "firstIndex": 2550136043},
-{"blockNumber": 10479586, "blockId": "0x61c1cccd60a546eacc29f2ae2a6facab402b4f35e2986c6629b302e6b1ebca3e", "firstIndex": 2617245577},
-{"blockNumber": 10562579, "blockId": "0x14fd6a079050e3e110140372605c050945d6db03746fa5b84a8718cb2a0e9b86", "firstIndex": 2684353802},
-{"blockNumber": 10641438, "blockId": "0x1998dcae09111b2c1bf6d0592e2f0b4a0143fe7db7115ff94b2b73b74542cee4", "firstIndex": 2751463334},
-{"blockNumber": 10717094, "blockId": "0x9ce45a150bb6bd13687d0692c86716911ffc6aff5831a03f4015bdffb85f5093", "firstIndex": 2818571144},
-{"blockNumber": 10784494, "blockId": "0xf65c936cf578e673696477f7db7189b47222fdf7ed13096564b9d47b15c86eb9", "firstIndex": 2885680709},
-{"blockNumber": 10848591, "blockId": "0x84ea8e9d5789302ebe88888115488f91cfb75fd89e282dfbd29ba9d6225ab0ca", "firstIndex": 2952789491},
-{"blockNumber": 10909102, "blockId": "0x22275047f643f4a3b82c349274a53d56a00da25beac89655877fb8cbfa3182c8", "firstIndex": 3019898024},
-{"blockNumber": 10972828, "blockId": "0x38e8acfec19bb53ab9fe70dd3577ccd8bd7c41b4d468b8f099a19bf06ce56518", "firstIndex": 3087006561},
-{"blockNumber": 11036532, "blockId": "0x478dac03f520e987a6e845cac1b3a756d69e4f5257f20758b8fc38456e7b6884", "firstIndex": 3154116463},
-{"blockNumber": 11102457, "blockId": "0x1a36616b03351ebe481330fc86578f57d9da5c0dda59f4406d49b53a2fa9b6f6", "firstIndex": 3221224961},
-{"blockNumber": 11168092, "blockId": "0xb23af483ca6dbc2fb8d2fada1f8189de2e155758ed9ecbbb20c234a30c7e093c", "firstIndex": 3288333510},
-{"blockNumber": 11233640, "blockId": "0xdc4c80d8370823773425b31e70b3af59582b556514857d1e9a9b4c53b880a424", "firstIndex": 3355442901},
-{"blockNumber": 11300454, "blockId": "0xdbd4d8e9c1756093037f431330db4604e5a3e67bb74f425f667633d5298fe34c", "firstIndex": 3422551962},
-{"blockNumber": 11367550, "blockId": "0x9e00816bd5b39353657a99421d278091599a39f246bbf8c0898f83e6d70b7c7c", "firstIndex": 3489660743},
-{"blockNumber": 11431820, "blockId": "0xd32d5e5871ce71acc9dcc7cf9d8cb9d964646182529c327a0ddf617814e27d74", "firstIndex": 3556769150},
-{"blockNumber": 11497031, "blockId": "0xed823054d4b293294e3301d3131e2ae9f0ce63c9f07d2d80fc144e42d535d61d", "firstIndex": 3623878513},
-{"blockNumber": 11560027, "blockId": "0x7098177efe77cfb5912dc63069fe366033111041ca5c539907a89e439c861d8f", "firstIndex": 3690987092},
-{"blockNumber": 11625049, "blockId": "0x61fccb47104a076b905d390498a1b38c317348bb1567e69aa1b670bed7a6a13f", "firstIndex": 3758096043},
-{"blockNumber": 11690298, "blockId": "0xaf6f9e9c92db9e155f1ffb35eb68765c09de1206c353c9a56a5d5fa53e2d88d2", "firstIndex": 3825204430},
-{"blockNumber": 11755003, "blockId": "0xf9efb54f08a2b59086a607541005928f1bda54fa2c46fcc48352fa6aa461c03a", "firstIndex": 3892313541},
-{"blockNumber": 11819591, "blockId": "0xa551deb3bd2ec0ca2683cb76adb8fb295d171a3787aa5ff071bd1fa94d3af82e", "firstIndex": 3959422322},
-{"blockNumber": 11883278, "blockId": "0x60c7a359d4aeb09b63b64487fa7794aee87c41b11446f69ba83e24d5bce6947f", "firstIndex": 4026531239},
-{"blockNumber": 11948436, "blockId": "0x1c46fb22653e5f9eb9503fe532151519845d05b121291766715973e027eb9c0c", "firstIndex": 4093639741},
-{"blockNumber": 12013084, "blockId": "0x0b8d1da9e70c4494ba61190390c891230846e69d9b0be605df8b04c4199ffd1b", "firstIndex": 4160749561},
-{"blockNumber": 12078612, "blockId": "0xb049af260be25c6153b67c227689b0556d8fb7d6033c3f61b8218f5c4ac5917a", "firstIndex": 4227857310},
-{"blockNumber": 12143549, "blockId": "0x685dce41efbe2bab0ac9a4b9026d9e5faa3e01ae4637c8181cd2f1eac326fe6a", "firstIndex": 4294966232},
-{"blockNumber": 12207918, "blockId": "0x53644ea8f781c6270f1136ed4bbcb956c98f12a4793c2f56f49b3105282b13ea", "firstIndex": 4362075410},
-{"blockNumber": 12272370, "blockId": "0x503523e1a01cb7d68be0eb22f36280c0ee7a1fc802061a704e326fbb8efcfe27", "firstIndex": 4429184831},
-{"blockNumber": 12329342, "blockId": "0xe6a1bf78dd827def078e23c768cf6e96666ee06b5fdc113380a4769d85f4bd33", "firstIndex": 4496293151},
-{"blockNumber": 12382311, "blockId": "0x352fe793c16b1d8aea195af86ee9aec3dbaa83e7a32872a0d222a77e96fc0e57", "firstIndex": 4563402486},
-{"blockNumber": 12436970, "blockId": "0x90276349edf0941bb3327dc1d749d0371fe550eaddafb5141fcacdd281de0976", "firstIndex": 4630510187},
-{"blockNumber": 12489954, "blockId": "0x71894b1e1608aeba777662a6055e601ce6f17950dbe3844c3601e0f88cbd5848", "firstIndex": 4697619586},
-{"blockNumber": 12541655, "blockId": "0xc34c7299041df72a81c4b86e81639d2dfcebfae603e287fc143881a636ce1188", "firstIndex": 4764728948},
-{"blockNumber": 12597319, "blockId": "0x4c4dc71d8d2f1223ad0469ddfe09fa8934f64e3712ed83708d6e1b9c64eceebb", "firstIndex": 4831837390},
-{"blockNumber": 12651863, "blockId": "0x406a642c8364fb61cbd896512a2d19ecdba3932ab09ac67cf48d91aa1a03f24e", "firstIndex": 4898946362},
-{"blockNumber": 12706391, "blockId": "0xa7fc3739898f7ff5733896313b248fb0b306a132681609b8b9e2671b42dac3a1", "firstIndex": 4966054748},
-{"blockNumber": 12762840, "blockId": "0x5a1048966dfa55a48248517c45683ce342671bf544117d2bac13d8e343563136", "firstIndex": 5033164303},
-{"blockNumber": 12816597, "blockId": "0x97f5b87f422870a8ffeff70c12f4377dde96d248aa912cb64ebfaea51dc086b6", "firstIndex": 5100272541},
-{"blockNumber": 12872315, "blockId": "0x432f7df7973fdea4e5abe2c09b634ca3b724e37a19d5ec23ca5810e4854c6b33", "firstIndex": 5167382243},
-{"blockNumber": 12929620, "blockId": "0xa4524e3b4c5dadac91df2f29696532cfed377f47469fe7cfabf10baac945eea2", "firstIndex": 5234490748},
-{"blockNumber": 12988653, "blockId": "0x9be70e31aeea164c370d8d183a1af45120693b13d360edc78214535c40ef1d63", "firstIndex": 5301600248},
-{"blockNumber": 13049074, "blockId": "0x3076857bd17959c2a3df18357eab3ecbb7ca25457785b07074d2e885eb566fc2", "firstIndex": 5368707746},
-{"blockNumber": 13108823, "blockId": "0x67b2764f6b53779e3d683def2c7659c6d92814152574073d24f10cd67200a22b", "firstIndex": 5435817680},
-{"blockNumber": 13175389, "blockId": "0x0d41e34dbca58dec45e0509b6cbf027b2377220845a85caebbf3a14b8505e8ce", "firstIndex": 5502925085},
-{"blockNumber": 13237624, "blockId": "0x723e0818d54ab1b1c53efc2f2f448c16c1f943af2033575c612c3c71228d4ddf", "firstIndex": 5570034948},
-{"blockNumber": 13298658, "blockId": "0xd7f9f477b20bcf1379ccaaae701c9658a02b177861e0af9e873c63fe8b854f4b", "firstIndex": 5637144562},
-{"blockNumber": 13361166, "blockId": "0x9f9e4130093269d59045f91344a13eb2786023e02099f53dd44854d50a7f5ffb", "firstIndex": 5704253260},
-{"blockNumber": 13421716, "blockId": "0x28752f35ad2adeba342ef6aa51bca1c0abc757c4f712cf620c8a5e6a8576636e", "firstIndex": 5771361775},
-{"blockNumber": 13480516, "blockId": "0x43f4ecf24c17b7a77621e2ca09c88c53b3407fb018f4bb3a032c212015b4e18d", "firstIndex": 5838469616},
-{"blockNumber": 13535688, "blockId": "0x99ffcd9982219a4be45fa5f15622b288afe4f85da536956d39dccac1ec416444", "firstIndex": 5905579739},
-{"blockNumber": 13590487, "blockId": "0xff7b74464a318037e0670c0be7a5b1b79418d1a42c7a3587c46adc1ef720c819", "firstIndex": 5972688013},
-{"blockNumber": 13646596, "blockId": "0xd555ce5f2b65ffeec9b9e3e75a56f5f23d7353a1014bcc861efd7df6e5dc2c92", "firstIndex": 6039797525},
-{"blockNumber": 13702902, "blockId": "0x3e8feab590c6631c63931fee83bf7e10234890bdf377b1b374fb70221b9ac9c0", "firstIndex": 6106905641},
-{"blockNumber": 13760908, "blockId": "0x18e96fa585c57aad1f2a8dc9639d9f4fe53c3f38894862471b7cec17f97f3cb1", "firstIndex": 6174014330},
-{"blockNumber": 13819340, "blockId": "0x25384bf2cef178e30c4d9e0abae59357055241fc715133b2bb078e8278e5296a", "firstIndex": 6241123906},
-{"blockNumber": 13877817, "blockId": "0x4a84a0a7035dbfed0e1281fb1df162a195f2c516686a2e2a6ad605e6f2b3cf57", "firstIndex": 6308232690},
-{"blockNumber": 13935260, "blockId": "0x6c14a8e563614edbad66698ebe911e997e27844f9d33c7069795379540e71a44", "firstIndex": 6375341148},
-{"blockNumber": 13993914, "blockId": "0xc0991d1bc431c5215fb0218b0b541f2b4fae0994775e5eddb016965fe5b4a0c7", "firstIndex": 6442448947},
-{"blockNumber": 14051007, "blockId": "0x854803ce364b6fe4446800e74448a5ee883e7d472e8a0347f1ad13bd347fa69f", "firstIndex": 6509558964},
-{"blockNumber": 14109063, "blockId": "0xa3fadce7ab18a5fedcfed9e3b1f0cf2fb32151172c362327c602343bc26dde8f", "firstIndex": 6576668017},
-{"blockNumber": 14166701, "blockId": "0xaa2443a38e2b0d6b01cf4b73de54d2b7e92c99475bd0802beebd86845c126d1b", "firstIndex": 6643774648},
-{"blockNumber": 14222367, "blockId": "0xa27c78134c1fbf432c8caee7c9d2ef323fdde519a065804c9c8da6ca1615e04f", "firstIndex": 6710886238},
-{"blockNumber": 14277029, "blockId": "0x5d0ed08b65db01e382c339dcb2715b2927fb472b0c1f4256841c386d785c3f3a", "firstIndex": 6777995247},
-{"blockNumber": 14330955, "blockId": "0x3e5014d82c416549acfe2e26b13ff5ddf29ad74943ef5b94a65eca0a4d72d82e", "firstIndex": 6845102969},
-{"blockNumber": 14384139, "blockId": "0x720e1fd687292019c3fa1812c39c73d20fde783c0196f43378a51cb4e0b46d65", "firstIndex": 6912212078},
-{"blockNumber": 14437534, "blockId": "0xe4342ab6f363066d9c2514a870241ceb63df232d33659fdb3928faf433360bb4", "firstIndex": 6979321493},
-{"blockNumber": 14489073, "blockId": "0xeb2f9923d816e773460eccfa42eb4ff2bbfc353b756ba274e98dc85c64ab1536", "firstIndex": 7046428879},
-{"blockNumber": 14541559, "blockId": "0x55cc33721fa9022964f47c30f4146fee7cc80f5b4158d7ecea0ff661cbba3487", "firstIndex": 7113538857},
-{"blockNumber": 14594432, "blockId": "0x676441bdc1ec430a9fbc05e56a0437a475606a253f9fdffc86f5f6286c159b5d", "firstIndex": 7180647956},
-{"blockNumber": 14644942, "blockId": "0x03232a1d3c9b2f1f0c2a4b780c83bce66238d9abdeed71c2a8af9e97a53011b0", "firstIndex": 7247754981},
-{"blockNumber": 14695793, "blockId": "0x6017f6d1d79188692e1a1a18de5f93b83b97905c8ff5bed5cef4b2e9f4d4d41b", "firstIndex": 7314863646},
-{"blockNumber": 14746186, "blockId": "0x332c8ac0b2960fb1627407d45443201a0f6de7421b52af077b32fa6e018e5401", "firstIndex": 7381974499},
-{"blockNumber": 14798403, "blockId": "0x4507ae1d9f27a3eada874dcf04112a5d4ee7155701afd3fb857fe58201a2dede", "firstIndex": 7449082752},
-{"blockNumber": 14848093, "blockId": "0xff98c820c66c3beaaa3b195158c94f099148573a7f032295ba7156c13f78eaa8", "firstIndex": 7516191978},
-{"blockNumber": 14897508, "blockId": "0x49c31d3433b197585f733d7c382ae16d9d78bf59c333a9b76362cc6c5fed270c", "firstIndex": 7583300663},
-{"blockNumber": 14950638, "blockId": "0xf1a1609206788977841927271808fe1c42d059026385d025bf4cea0c0a05966b", "firstIndex": 7650408992},
-{"blockNumber": 15003958, "blockId": "0xf9e6b41108d907672f29ba1df97ed83aca2ba933335e6b5ccfb5d4001807b33f", "firstIndex": 7717519105},
-{"blockNumber": 15056783, "blockId": "0x126a5a5c22e878a4c42bea890d48cfe00edf7fbc27d805c88a5269540bc94a42", "firstIndex": 7784628173},
-{"blockNumber": 15108163, "blockId": "0x5f8494ccacefce9fef8a95a52247f2670ed30bb5a3b3deb85c645e72c115d506", "firstIndex": 7851735261},
-{"blockNumber": 15159385, "blockId": "0x0fed6c5aafb53e9cdb0d8620c3eef9b619fb3ae9559296618f634eddae33f0c6", "firstIndex": 7918844299},
-{"blockNumber": 15210872, "blockId": "0x37bdb94b57044b67fbd9b9a7cdc728fcfd2066cf1904cab478fd42d41ec6e1f0", "firstIndex": 7985954119},
-{"blockNumber": 15264226, "blockId": "0xc173ad8a791e2293efd8e0b2c9f088fac49233aabb1e72db6a045aa9904f4728", "firstIndex": 8053061840},
-{"blockNumber": 15315693, "blockId": "0x985cf97bf28f0becb2b16abfe629a5f520596e45fc23f4936bc2f6298b8145f9", "firstIndex": 8120172048},
-{"blockNumber": 15364606, "blockId": "0x593362ef8a82140f3dd3ff13d0ce50287dcb0b8b3d31ffcc7813c6be16f7438d", "firstIndex": 8187281275},
-{"blockNumber": 15412727, "blockId": "0x5a6ff10acc48e9e8bcf963b042012ea094628b9488b8bf569783b3297c67eedd", "firstIndex": 8254388575},
-{"blockNumber": 15462642, "blockId": "0x137f15c5d112b0fdc6e4c84393032030646122c3b0d86286143a1ab8bfacbac4", "firstIndex": 8321497214},
-{"blockNumber": 15509781, "blockId": "0xe58a8ca223b7c03755d046a6212bc33fa9814f73efe52804d8eea7ae4b4bd7f9", "firstIndex": 8388606496},
-{"blockNumber": 15558625, "blockId": "0x4127d527b58c7685a1237d228508ea77d40f33dbeffb5cc30e47e83a4b6152a5", "firstIndex": 8455714910},
-{"blockNumber": 15604075, "blockId": "0x7d1c1bccfe5c23a1edd81246e6e55277f271cffef30bde22c9d1202f764dd96f", "firstIndex": 8522823324},
-{"blockNumber": 15651742, "blockId": "0x732e5203a504fd97510325c1fb876476ca132831e78182367cd4b4efae1232c3", "firstIndex": 8589934155},
-{"blockNumber": 15700810, "blockId": "0xe031decb5e42a69dafcc814441227085508cf51c9c77f35d06487db19bd05a6a", "firstIndex": 8657041374},
-{"blockNumber": 15762855, "blockId": "0xbdff9b1e193c44f35bf38945d2523c74ef59c4f352efc93fad02ef34eef1c5f0", "firstIndex": 8724150924},
-{"blockNumber": 15814304, "blockId": "0x524a226d9f48e37db00e252c7f6a1350d1829d38fe83c756021cc11c56d3b2dd", "firstIndex": 8791259373},
-{"blockNumber": 15865514, "blockId": "0x4e4ad46f3ae7fb2a2e7cb94aa804929cb7353c79c78cc2cbcee710da5f0ec4b5", "firstIndex": 8858369571},
-{"blockNumber": 15920411, "blockId": "0xa39e4cbd8e84b427e44b8d06b5b1149652f3af5fa88633b802665b8677772bc5", "firstIndex": 8925477782},
-{"blockNumber": 15974193, "blockId": "0x4a3f82935e6fcda6ce7636001989b65b79422d362e60833e01185a7960b2fa55", "firstIndex": 8992587369},
-{"blockNumber": 16032739, "blockId": "0x7f50bb76dfd01125e0a4a7595106e43329eb8bd02e819be2ade4248a403c714a", "firstIndex": 9059696611},
-{"blockNumber": 16090896, "blockId": "0xdc4db130973ef793d2223844e35122a2477e642d86c69616fb3604d2a79805c4", "firstIndex": 9126804425},
-{"blockNumber": 16150783, "blockId": "0xb46294fe1cbcbbdff7309ebf0bf481e78e2589d386b95fa4c781c437e02e0aec", "firstIndex": 9193912139},
-{"blockNumber": 16207278, "blockId": "0x8a3af0387572b56989bb6c98d8860e07b5cdefb2ebe7b1dc58be07c24a53e056", "firstIndex": 9261022215},
-{"blockNumber": 16262389, "blockId": "0xec5a03d88cfc1bf8c278ddce7ace5032c3f7ed5032b4575df54779ec1abca131", "firstIndex": 9328130680},
-{"blockNumber": 16319525, "blockId": "0x74ae702375e242205fe4986448b0ee494d4a38ccc813981489c4b0488e58a949", "firstIndex": 9395240779},
-{"blockNumber": 16373441, "blockId": "0x5f404555cf45c547ef0fc044f2701bc7a28ce1e432de06e0264dbfd1c17447d0", "firstIndex": 9462347671},
-{"blockNumber": 16423343, "blockId": "0xe4af890b6c4dd3b10af2e6c08f14563bd2a3121b22a3cc6a0db29d438c1dcc59", "firstIndex": 9529457704},
-{"blockNumber": 16474687, "blockId": "0x019a8b374cf150fcda1c8af2a3e7ba63603ae81744d6343897c4b17ee58411d5", "firstIndex": 9596566169},
-{"blockNumber": 16525504, "blockId": "0xad394ce7e84e1c904cc5dd6dabd51b1a575c0b2fb006f10f272875b7f921904e", "firstIndex": 9663675287},
-{"blockNumber": 16574051, "blockId": "0x098e2dca2eda0d700aa3127bdfb679ce16e3ca297a305978c6ecfff044269511", "firstIndex": 9730784563},
-{"blockNumber": 16622457, "blockId": "0x8627f1237731c1127116898724f1b6420b8e7bcd8e18a0ea568f192bbd2cd55d", "firstIndex": 9797893241},
-{"blockNumber": 16672410, "blockId": "0x3e32b46faa670b191b89bea84a68022d5ea8b90008b388a948ff27f871d17292", "firstIndex": 9865002964},
-{"blockNumber": 16719964, "blockId": "0x1668c3ea6cb256f306e93851cc2c00612f85df5cf7b5527ffce14c89088d7fdf", "firstIndex": 9932111708},
-{"blockNumber": 16769014, "blockId": "0xed9a6f88e47b5bfe4d5bc92e1c1bfeb252f716f0990ffa1f907655c0f5e3a365", "firstIndex": 9999220312},
-{"blockNumber": 16818134, "blockId": "0x93504ba617500698a1b46f3110cd26827b8efe468190b2a91d3595784e34a1b6", "firstIndex": 10066329167},
-{"blockNumber": 16869369, "blockId": "0x94ca4b395a5c36c4e9d07cbb08271f0680455af4e4659906b647e16abebd7d4e", "firstIndex": 10133438415},
-{"blockNumber": 16921428, "blockId": "0x66650775bff4cb95de56099c10b06f3dfe302a731b391f406d1aa551121308ba", "firstIndex": 10200547148},
-{"blockNumber": 16974677, "blockId": "0x58d5fc4cd45047dac10d55a036e0a4448faacc1041b29c4c7fb41a6e9394c3ff", "firstIndex": 10267655623},
-{"blockNumber": 17031050, "blockId": "0x9d15b831979e95fc4149ecae46478cac1e54470b277d57d7102d19a72f28eeba", "firstIndex": 10334764441},
-{"blockNumber": 17086163, "blockId": "0x978ff48863fcdb50b274ea4220e7635c13c41bae91912db9e24205fb917fde85", "firstIndex": 10401873176},
-{"blockNumber": 17141086, "blockId": "0x4084d442499893361731f17534944d260cccd8e07d41c52d7160910555770b90", "firstIndex": 10468982462},
-{"blockNumber": 17190735, "blockId": "0x42ed86fb42def23870df6ec23204f557669ec7cd8de59c9f3ef9ab90704e4b11", "firstIndex": 10536090726},
-{"blockNumber": 17237026, "blockId": "0x9a49d943f82a7335f8049df4d953080f25920e3b098f6a2d872520166b225961", "firstIndex": 10603199938},
-{"blockNumber": 17287007, "blockId": "0xac3d189676d24eadc62adcf97d8730a001a7766fe3ac27b18e745ca9727f6c46", "firstIndex": 10670308186},
-{"blockNumber": 17338498, "blockId": "0xe50e251aa0a3209fa8a0030b237cf8357b08eeb80492a432c8aa5969a820ee42", "firstIndex": 10737417623},
-{"blockNumber": 17389178, "blockId": "0x1e18cedcfa6c81bc845c9136fc92f9f27ca8fff8847289bb139f374f0f6ba733", "firstIndex": 10804526769},
-{"blockNumber": 17442038, "blockId": "0x07f47ebbee99938fb96beb1ffadf8266f71d0d73f393172f0e24facd40d45c97", "firstIndex": 10871635940},
-{"blockNumber": 17497589, "blockId": "0x247a5e852e36e46f7897ad1875e93d810ff4434bb835b469afae3a3d3d4a9c9a", "firstIndex": 10938744251},
-{"blockNumber": 17554111, "blockId": "0x420e420a1b599e4dc3bd115652b4d6f5eedc391086ef064f97a9cbffc02b62f9", "firstIndex": 11005853686},
-{"blockNumber": 17608282, "blockId": "0xb2ba644d3593d3c832f932f14ffec532e26e9e1cdcd8fbe7477f46e238c6bffd", "firstIndex": 11072962303},
-{"blockNumber": 17664091, "blockId": "0x6d0bfff8145652a617bfe03ab87e3af09219cdb822c9c214b42b16f20742c1eb", "firstIndex": 11140070260},
-{"blockNumber": 17714917, "blockId": "0x61509544b5174d3ff5d48816ff54f2929c8a400ef21454116056baa136b607b6", "firstIndex": 11207179013},
-{"blockNumber": 17763850, "blockId": "0x8da83012621f42670601406c902ce6e48e77c6285dcc6983e108e69b3a8d5d2d", "firstIndex": 11274288732},
-{"blockNumber": 17814186, "blockId": "0xc37bda200856589c220dc206e31186d6c12761fd193f5bc7123cf331fbbed7f5", "firstIndex": 11341397343},
-{"blockNumber": 17864461, "blockId": "0x99af43893dc9f54c55abbdbd7b000e27248145371e9b00db116dfb961b36749c", "firstIndex": 11408506122},
-{"blockNumber": 17913212, "blockId": "0x9b642236d63d756fcd3404713bc6abe97804b46a9e665ae5f1d9aa48e693fe78", "firstIndex": 11475614347},
-{"blockNumber": 17961497, "blockId": "0xaec9b1e851cf51646673ec7698e1d5964d44eb04729ad11385f6ae84d497070a", "firstIndex": 11542723713},
-{"blockNumber": 18010859, "blockId": "0xb15fb7fc308516cad5f1dadfb7dd92fd3fe612308f67154cf2a241990aa15bc4", "firstIndex": 11609833204},
-{"blockNumber": 18061052, "blockId": "0xd8ea94b8b8a7380d21dc433abb69973f3de2eeb339f4aaffdb79f85a5d86929d", "firstIndex": 11676938565},
-{"blockNumber": 18111466, "blockId": "0x5b5cc6d182bffaf06692a7efda8cad5d4742fdbce5c1146b3be3c5be10ea046a", "firstIndex": 11744048539},
-{"blockNumber": 18166001, "blockId": "0x901d8401c795542c75069cd917781fa587ad416da0a5298f144fae2f14305705", "firstIndex": 11811159949},
-{"blockNumber": 18218505, "blockId": "0x5894285023f34ec6fd501cd90b605e118b9688ca7c01082289bf69a6987142bc", "firstIndex": 11878268395},
-{"blockNumber": 18271037, "blockId": "0x3f0fce315fd993d966b6c599ebd360462466e67046d512dc430ac07f75243bae", "firstIndex": 11945377050},
-{"blockNumber": 18322778, "blockId": "0xfa6536b3b06e7b579e53a8d01368182808c950586e39e12485be3145c3acbf43", "firstIndex": 12012486036},
-{"blockNumber": 18372240, "blockId": "0x5531859d5b8a4093020e2fb971808d0eeaddd7d61dc58d70f34923081f97336f", "firstIndex": 12079595358},
-{"blockNumber": 18421630, "blockId": "0xfeadff676fe8f04d0362b77ea2d7c3ea59b05d68cc6b2cc95cfe751558b1bfa7", "firstIndex": 12146703872},
-{"blockNumber": 18471472, "blockId": "0x352b0a17e4160d334b876eb180d686fd67c7ddc40a78219b0bc4b59f5ae5912d", "firstIndex": 12213813174},
-{"blockNumber": 18522088, "blockId": "0xf7d408309246089531b47c787049c7ffd2820c1b87b4804033a458bc193a3432", "firstIndex": 12280921181},
-{"blockNumber": 18572725, "blockId": "0x44126d4a5b43594b09d26e0b0a5dc859c074865cb9b11ab78c80f28406fd0240", "firstIndex": 12348029615},
-{"blockNumber": 18623223, "blockId": "0xd3aec1f98cea30e870d0f530194e9a72e225eb45eefe435e38da5967eb2fa470", "firstIndex": 12415139522},
-{"blockNumber": 18675228, "blockId": "0x24deaef2508038bddf8e9020f95128ae34eb1786f786c3f98df0bdf95f3e97cc", "firstIndex": 12482248123},
-{"blockNumber": 18725495, "blockId": "0x2e5a7edd7337b20e862741e116680a472089426dbc2b0eb5f1c984f8d6da935b", "firstIndex": 12549357121},
-{"blockNumber": 18778174, "blockId": "0xe734a1084fa5eeff427bba8e8d3623181a30dd9bad128fc22de9b15dffd0e549", "firstIndex": 12616466131},
-{"blockNumber": 18834807, "blockId": "0x0924a55ef4f9c2605261433c5b236d47645484efa44b34dc9f15b7c2e6b47a74", "firstIndex": 12683575000},
-{"blockNumber": 18883054, "blockId": "0xef638ec491a1cf685b22bf4072e41710440d6133a53e31250762825247d3eb64", "firstIndex": 12750683712},
-{"blockNumber": 18933405, "blockId": "0x6e9eefc2e121d13e6c04544ea370fe8543afeb36fef3a8e2ea68afb079738c8c", "firstIndex": 12817792510},
-{"blockNumber": 18988033, "blockId": "0x4c98c0dbb6a7217609c4b277fe7ff9e2c32c8d000ee628a3f37b2b296e1ca421", "firstIndex": 12884901389},
-{"blockNumber": 19041096, "blockId": "0x61a219a23d111cd07861ff47b89d4b2840c0ab5421d5d55a6f1fe43da3f33b98", "firstIndex": 12952010147},
-{"blockNumber": 19088914, "blockId": "0xefb26ba58447e50e9b820e0b0d16c08da45793a824af12a2e1f38b6f46b7e36a", "firstIndex": 13019118891},
-{"blockNumber": 19140413, "blockId": "0x91dbb13623fc4d5a4e462dfa0a03c92bf5301948dd4b2c3e5fddd2bc88856d47", "firstIndex": 13086228137},
-{"blockNumber": 19191876, "blockId": "0xdd7f97af7ad206b2636656ed4722fe1c47c6ba9fa5531066785acbe4f8d4c4ad", "firstIndex": 13153336551},
-{"blockNumber": 19237651, "blockId": "0xf1e875f314314abe3654b667c6443c772dd67c525c52877b06515ea8fc5a391b", "firstIndex": 13220444812},
-{"blockNumber": 19291032, "blockId": "0x9eda269ac47957b619d2a8c408b5efc8cfb168a46bfde351f43325fa337fce47", "firstIndex": 13287553967},
-{"blockNumber": 19344225, "blockId": "0x62859b78d45f589f679fc6a3bf266a2e0aa906fd61d787755ec51cfadc05eaf6", "firstIndex": 13354663338},
-{"blockNumber": 19394709, "blockId": "0x7cda3d193bcfe8ddbc89b85e98c814dbbe31a54ccab52ab7738ecd003061f3c8", "firstIndex": 13421772549},
-{"blockNumber": 19442850, "blockId": "0xd95f3f8bab5b91c3e1f848c9455936850bf92cf6a5086b817f6f81031bc0f81e", "firstIndex": 13488880704},
-{"blockNumber": 19488177, "blockId": "0x3a1a9623ee332d6decada9359cf84eeafb81fffa997d6cf4f3efcc92e98e9310", "firstIndex": 13555989570},
-{"blockNumber": 19534365, "blockId": "0x5c7a8ee7cbba4894ea412cc335895a9a7d3369d35779bcdc97a6c24b6a81a7ba", "firstIndex": 13623097446},
-{"blockNumber": 19579409, "blockId": "0x4280f5f76cb0f9947f9da1cb9de6c4b6e65b37b3a3c174f5fb29d360cf389ffb", "firstIndex": 13690206790},
-{"blockNumber": 19621406, "blockId": "0x0a98ca72e97fd295c7c73263b17185d74dbe427f41c616a5d5454faa5a1c63a0", "firstIndex": 13757316653},
-{"blockNumber": 19664868, "blockId": "0xc8c53d89b7a849861f3daaf9503e6397a0340cecb085850c3636bee6ea2b7189", "firstIndex": 13824424608},
-{"blockNumber": 19709011, "blockId": "0xf65babc88ef6a9eb2ffce9a6844adf895995f65a2facd1956f5df17ca68dbf94", "firstIndex": 13891534141},
-{"blockNumber": 19755170, "blockId": "0xf903882449eeb86d69af33b6797099275e82d8787334a02a42a2168dbd739b95", "firstIndex": 13958642627},
-{"blockNumber": 19803666, "blockId": "0xa8a975f570db082a3771790bb1fe405667dde4674ff49e31092687fa92727e36", "firstIndex": 14025752433},
-{"blockNumber": 19847730, "blockId": "0x89d479a4b4d9faa7af3e3f61d13a7a9c80a11c4adf7cd018859eb0ddba30bdd5", "firstIndex": 14092860598},
-{"blockNumber": 19894138, "blockId": "0x740278e6dc79cfd2b030780868a1b8e1520dbae0e8d3ee0a3ef43c84c83c6e0a", "firstIndex": 14159952129},
-{"blockNumber": 19938334, "blockId": "0x05ae7ce9813235c19baa57658bc5c8e160130328a1fd2cc58d2baf2d9a4ae097", "firstIndex": 14227078393},
-{"blockNumber": 19985138, "blockId": "0x511c8876a93a77fa473ae0d29e25d8163162c09b7212f506be0ea73840ea3971", "firstIndex": 14294187397},
-{"blockNumber": 20028221, "blockId": "0x64379baedf429cfa65df076142a7d9909d7e369d6aa6b8e161bb686944d169ff", "firstIndex": 14361296446},
-{"blockNumber": 20071555, "blockId": "0x4f43d8b1091d2bd0c6b080ce51bce08aae6be605e00889be3cdd17cb80614ff8", "firstIndex": 14428405409},
-{"blockNumber": 20113611, "blockId": "0xebbf87409ef6732a53513414926b0aa998a7269f9e9717a3acbe96a33c2de937", "firstIndex": 14495514255},
-{"blockNumber": 20156628, "blockId": "0x70218108c0669f42c4ac1afff80947cf4a0a2fd8e9300b31d45bbe330a5159c9", "firstIndex": 14562622257},
-{"blockNumber": 20199886, "blockId": "0x1d6aaa4b041eb0cc9caed34347bb943e50a139caf8998d781ff4f06d4aa727dc", "firstIndex": 14629732017},
-{"blockNumber": 20244148, "blockId": "0x37fed57b6945a815ec56c84c6261e90cf9f3883d9cbffc62b70d9722c1f0f363", "firstIndex": 14696839730},
-{"blockNumber": 20288249, "blockId": "0x1d9fe664431e093c4ef6139ad35fea767a2e66825e605cfea9e91d1738665619", "firstIndex": 14763948803},
-{"blockNumber": 20333358, "blockId": "0x1ac7c1362791507057b7e87d7118b4f61c2919c8ebdb498ecf7d64a59ba3d2e2", "firstIndex": 14831058814},
-{"blockNumber": 20376855, "blockId": "0xf6a19eac8a402f591079a4aa7ff57349c350ec96d683a031e4d85c05bc0541b2", "firstIndex": 14898167076},
-{"blockNumber": 20421474, "blockId": "0xe8767aa7516b632dc8c0fed3a366273ef14bd7b8c2212998b7105fb80b4c0d9b", "firstIndex": 14965276285},
-{"blockNumber": 20466974, "blockId": "0xafa2d2e6c9594762c163c339069cb15aa9fd27d301268892e12f99a3f2912ade", "firstIndex": 15032384674},
-{"blockNumber": 20512183, "blockId": "0xc1c51f9372d781d7ac0a1a7aa052503bbde3e1706bf65f225b0f6a6eed93d055", "firstIndex": 15099493832},
-{"blockNumber": 20557251, "blockId": "0xa905087c15e2b12b19cd23f20089b471cd9815dcff73842d2043e13a276fcc55", "firstIndex": 15166601921},
-{"blockNumber": 20595654, "blockId": "0x9e7772dd946f65e7569f34f6896d8e597ac8ff88700b871de21ff578c0c995e6", "firstIndex": 15233711474},
-{"blockNumber": 20638340, "blockId": "0xc08b46af83db6f0e5d1acad7020bacd59d0f18df72651d32c9b60edd0351a886", "firstIndex": 15300820584},
-{"blockNumber": 20683359, "blockId": "0x7ec93a1f1d211c24d93ce7e344634efef680a6388374cf3831d250ecd63711f9", "firstIndex": 15367929168},
-{"blockNumber": 20728401, "blockId": "0x4ad4124c2595abcf4477cac4eb70c0c9372bbd48f2066c7ac5b8e9f94817ab48", "firstIndex": 15435038303},
-{"blockNumber": 20771272, "blockId": "0x68fd6b12fdb4add8b2957105a21b571c7dcb114c6c2ca02f5fa40730a108f864", "firstIndex": 15502146509},
-{"blockNumber": 20814862, "blockId": "0x4e7f640bf3c135b2442ca4c744023021af163e8d45e8d776fb7051c49d79fcff", "firstIndex": 15569256173},
-{"blockNumber": 20857627, "blockId": "0x9b023bffed12530b5a68db263bb621a147e0cf02b30ec39a82b82a1fdeb7ca36", "firstIndex": 15636364955},
-{"blockNumber": 20896662, "blockId": "0x8c3adf694b8520ed1edd38282b2f58cf383426176628b5be6330a56882e2a07e", "firstIndex": 15703473445},
-{"blockNumber": 20939156, "blockId": "0x0649e2559f767da945a674c9d6c700628109e18d0757ad33496a228a827413a4", "firstIndex": 15770581979},
-{"blockNumber": 20981054, "blockId": "0xebf0ecca671b833f77df70106cf1171d4dd3fa0f062607e4845569073997f1cd", "firstIndex": 15837691319},
-{"blockNumber": 21022980, "blockId": "0x07479520c08a727c42e3616aca7fc4435773f4ef277d84369190263159606d71", "firstIndex": 15904799391},
-{"blockNumber": 21068139, "blockId": "0x669cfd03b293a61815588cb9cef928a512d328c1fe9191fe6b281573f429f8b8", "firstIndex": 15971908604},
-{"blockNumber": 21112368, "blockId": "0x98957aedaf8b0483be208e8d16a3e6d73ff217e732f3649650671c92c8fb57a5", "firstIndex": 16039007815},
-{"blockNumber": 21155756, "blockId": "0xc301e40988fbe19b1254ab3abe2f78fe0cd35259898be02476179a041926f777", "firstIndex": 16106127130},
-{"blockNumber": 21200717, "blockId": "0x8d35b199ee65c5ccf4ca0fc22b50282850cd88b72580679f1568eea67d3431e8", "firstIndex": 16173235593},
-{"blockNumber": 21244396, "blockId": "0xe7ec084eb8a7125fe26c4ef64fdbb3e0a727b2d882fb1e4fdad1d7fa3dc55fd2", "firstIndex": 16240343803},
-{"blockNumber": 21290031, "blockId": "0x8bb31b883d30875b7ed8a01c35a6f51e603b8d974c4e74af20fb9c723e5ac5df", "firstIndex": 16307452772},
-{"blockNumber": 21335893, "blockId": "0xd0976b7c946a7556e607e47a02d2cfcccd259508e2828b9c4faa5c7105a597a8", "firstIndex": 16374562657},
-{"blockNumber": 21378611, "blockId": "0x72d42953f013aeb7c84bb22c9e07828b92f73885b20c208af612874df8fe85ed", "firstIndex": 16441671192},
-{"blockNumber": 21421343, "blockId": "0x663b3cddcf3f7cfa59ed29b4894e43fa784c07576cfec897ff1a3838235ecb50", "firstIndex": 16508779766},
-{"blockNumber": 21466598, "blockId": "0xe04ebd60289dbcc09a27e053e7b240a59e928e2a76a85a6fa4044ee387c46d18", "firstIndex": 16575888836},
-{"blockNumber": 21511717, "blockId": "0x5ea9863a0b504f381010ff383d60a56390ba9cef162d9d5a077b1884d2adac98", "firstIndex": 16642998234},
-{"blockNumber": 21550050, "blockId": "0x2a7b3e635c80521887ca3ebda04541090ec720dd817cd06fcba61e7ae0f78a4f", "firstIndex": 16710106733},
-{"blockNumber": 21592440, "blockId": "0xc7095268d0d53cb5a069c2ea7107cc42172bb1b817f04154938e6b52bdbb369c", "firstIndex": 16777215261},
-{"blockNumber": 21636007, "blockId": "0x55f463b932053ebfc02728a12c2cdfb2cb4700a115cde686c9ab22bb4c336ecb", "firstIndex": 16844324844},
-{"blockNumber": 21680799, "blockId": "0x4521d473aa7f5150f920c66fe639b20f30cb373b32e425fcb7dcce9803b506be", "firstIndex": 16911431255},
-{"blockNumber": 21725330, "blockId": "0x793876007ce3594df603628766b139a02273ceee43d5b3ca2d0ca2f13a5fc5d9", "firstIndex": 16978541626},
-{"blockNumber": 21771310, "blockId": "0xd0db23bb28d6e39917348b1879dd560c73354ea6d7d0bbee0b7a8fcbcede1091", "firstIndex": 17045650902},
-{"blockNumber": 21811348, "blockId": "0x8165948c08db34aeb8cc2a3e331a00af2d09fbff9c466019fb2f2892ca3f7b71", "firstIndex": 17112760024},
-{"blockNumber": 21851615, "blockId": "0x3083e8f4fbb4d726ef19b1a5bce20e639a0418eac384f9967f24c7697a7e2dfc", "firstIndex": 17179868870},
-{"blockNumber": 21894041, "blockId": "0x9b0776a020374e4985f6a7e1199596bedd387ff085f65224d724ab11955c56fe", "firstIndex": 17246977342},
-{"blockNumber": 21932218, "blockId": "0xe9c50eddc21759d3ab790e2babba56f4b8d38eb739f1b561bc337a1ba6e15393", "firstIndex": 17314085541},
-{"blockNumber": 21959188, "blockId": "0x6b9c482cc4822af2ad62a359b923062556ab6a046d1a39a8243b764848690601", "firstIndex": 17381193886},
-{"blockNumber": 21994911, "blockId": "0x0d99ef9dd42dd1c62fc5249eb4f618e7672ab93fa0ba7545c77360371cf972e5", "firstIndex": 17448302795},
-{"blockNumber": 22026007, "blockId": "0xf7cdc7f6694f2c2ed31fa8a607f65cfa59d0dd7d7424ab5c017f959ae2982c71", "firstIndex": 17515413036},
-{"blockNumber": 22059890, "blockId": "0x82b768a0dddefda2eefd3a33596ea2be075312f1dd4b01f6b0d377faca2af98b", "firstIndex": 17582521768},
-{"blockNumber": 22090784, "blockId": "0xf97c2eaf9a550360ac24000c0ff17ffa388a2bdd6f73f2f36718e332edfa107a", "firstIndex": 17649630983},
-{"blockNumber": 22121157, "blockId": "0xa790025235db782e899f23d8b09663ec2d74ec149e4125d62989f98829b08e2d", "firstIndex": 17716724973},
-{"blockNumber": 22148056, "blockId": "0xbe25ac4f1bdd89a7db5782bf1157e6c4378d80220d67a029397853ef16cd1a4c", "firstIndex": 17783838366},
-{"blockNumber": 22168652, "blockId": "0x6ae43618c915e636794e2cc2d75dde9992766881c7405fe6479c045ed4bee57e", "firstIndex": 17850956277},
-{"blockNumber": 22190975, "blockId": "0x9437121647899a4b7b84d67fbea7cc6ff967481c2eab4328ccd86e2cefe19420", "firstIndex": 17918066140},
-{"blockNumber": 22234357, "blockId": "0x036030830134f9224160d5a0b62da35ec7813dc8855d554bd22e9d38545243ed", "firstIndex": 17985175075},
-{"blockNumber": 22276736, "blockId": "0x5ceb96d98aa1b4c1c2f2fa253ae9cdb1b04e0420c11bf795400e8762c0a1635c", "firstIndex": 18052284344},
-{"blockNumber": 22321282, "blockId": "0x8a601ebf6a757020c6d43a978f0bd2c150c4acc1ffdd50c7ee88afc78b0c11f8", "firstIndex": 18119392242},
-{"blockNumber": 22349231, "blockId": "0xb751c026a92ba5be95ad7ea4e2729a175b0d0e11a4c108f47cab232b4715d1a2", "firstIndex": 18186501218},
-{"blockNumber": 22377469, "blockId": "0xa47916860a22f7e26761ec2d7f717410791bd3ed0237b2f6266750214c7bbf08", "firstIndex": 18253610249},
-{"blockNumber": 22422685, "blockId": "0x8beaee39603af55fad222730f556c840c41cd76a5eef0bad367ac94d3b86c7aa", "firstIndex": 18320716377},
-{"blockNumber": 22462378, "blockId": "0x6dba9c5d2949f5a6a072267b590e8b15e6fb157a0fc22719387f1fd6bfcd8d5d", "firstIndex": 18387828426},
-{"blockNumber": 22500185, "blockId": "0x2484c380df0a8f7edfdf8d917570d23fab8499aea80c35b6cf4e5fe1e34106e9", "firstIndex": 18454936227},
-{"blockNumber": 22539624, "blockId": "0xd418071906803d25afc3842a6a6468ad3b5fea27107b314ce4e2ccf08b478acf", "firstIndex": 18522044531},
-{"blockNumber": 22577021, "blockId": "0xff222982693f3ff60d2097822171f80a6ddd979080aeb7e995bfb1b973497c84", "firstIndex": 18589154438},
-{"blockNumber": 22614525, "blockId": "0x9868da1fea2ffca3f67e35570f02eb5707b27f6967ea4a109eb4ddbf24566efd", "firstIndex": 18656264174},
-{"blockNumber": 22652848, "blockId": "0x060a911da11ab0f1dda307f5196e622d23901d198925749e70ab58a439477c5a", "firstIndex": 18723372617},
-{"blockNumber": 22692432, "blockId": "0x6a937f2c283aba8c778c1f5ef340b225fd820f8a7dfa6f24f5fe541994f32f2d", "firstIndex": 18790480232},
-{"blockNumber": 22731200, "blockId": "0x00d57a9e7a2dad252436fe9f0382c6a8860d301a9f9ffe6d7ac64c82b95300f8", "firstIndex": 18857590076},
-{"blockNumber": 22769000, "blockId": "0xa48db20307c19c373ef2d31d85088ea14b8df0450491c31982504c87b04edbc0", "firstIndex": 18924699130},
-{"blockNumber": 22808126, "blockId": "0x1419c64ff003edca0586f1c8ec3063da5c54c57ff826cfb34bc866cc18949653", "firstIndex": 18991807807},
-{"blockNumber": 22845231, "blockId": "0x691f87217e61c5d7ae9ad53a44d30e1ab6b1cc3f2b689b9fbf7c38fbacacfe3e", "firstIndex": 19058917062},
-{"blockNumber": 22884189, "blockId": "0x7f102d44c0ea7803f5b0e1a98a6abf0e8383eb99fb114d6f7b4591753ce8bba3", "firstIndex": 19126024122},
-{"blockNumber": 22920923, "blockId": "0x04fe6179495016fc3fe56d8ef5311c360a5761a898262173849c3494fdd73d92", "firstIndex": 19193134595},
-{"blockNumber": 22958100, "blockId": "0xe38e0ff7b0c4065ca42ea577bc32f2566ca46f2ddeedcc4bc1f8fb00e7f26329", "firstIndex": 19260242424},
-{"blockNumber": 22988600, "blockId": "0x04ca74758b22e0ea54b8c992022ff21c16a2af9c45144c3b0f80de921a7eee82", "firstIndex": 19327351273},
-{"blockNumber": 23018392, "blockId": "0x61cc979b00bc97b48356f986a5b9ec997d674bc904c2a2e4b0f17de08e50b3bb", "firstIndex": 19394459627},
-{"blockNumber": 23048524, "blockId": "0x489de15d95739ede4ab15e8b5151d80d4dc85ae10e7be800b1a4723094a678df", "firstIndex": 19461570073}
+{"blockNumber": 4166212, "blockHash": "0xd94b724fc1c7dceb3251b51b81f7a0f3220ae5b9add1e62917004f14f2a3532c", "firstIndex": 67108840},
+{"blockNumber": 4513996, "blockHash": "0xc0fd25fef5888609d05fa8c5620d8e18c31cdd675dd4ee926ff966668f0e5d76", "firstIndex": 134217480},
+{"blockNumber": 4817399, "blockHash": "0x8573c3166fb7f716e97cf6109d382f86441238e0b85828944a072ae8583a6cfa", "firstIndex": 201326547},
+{"blockNumber": 5087706, "blockHash": "0xcd2dfae45901e299b25faa04c10df60983d5c0a3d0e478e789c7aeec980cf691", "firstIndex": 268435422},
+{"blockNumber": 5306085, "blockHash": "0xe8026984c85873f2b24018a7e0bdf8c2df136b2fc49666b4addf0d2e067890da", "firstIndex": 335544018},
+{"blockNumber": 5509898, "blockHash": "0x41bc63dc8184f9a7d4b9a5a554e00eee32fee60ccfa2339264be11270ac28de1", "firstIndex": 402652789},
+{"blockNumber": 5670367, "blockHash": "0xe61c2ba463f2f458817ed1bf0ff9fb9d07e9d7354f46b48175b2d784b817bf3b", "firstIndex": 469761896},
+{"blockNumber": 5826113, "blockHash": "0xfbbb1fc5e03a5562bf6b7f1799d7a572d9ef66c7fd0e0b9f4fed7262767b5c86", "firstIndex": 536870852},
+{"blockNumber": 5953008, "blockHash": "0x6bbdccd094928e846de3c615a33f2952e3259afaa1929f4ee241a56907d0593b", "firstIndex": 603979150},
+{"blockNumber": 6102812, "blockHash": "0x2403f2502088a1fa26c1294f5245032fe3b7f3a8bb14c12d0ba8d577156bbc1b", "firstIndex": 671087962},
+{"blockNumber": 6276672, "blockHash": "0xeaa05a0e0574fbb164244628a7d14d9d47f1692941098aa737059d44104401df", "firstIndex": 738197399},
+{"blockNumber": 6448662, "blockHash": "0xe6f097fa18a2cef425514e863659e42e1aa8d74df49cb0bed5877e82a08d1f84", "firstIndex": 805306056},
+{"blockNumber": 6655925, "blockHash": "0xee5c9d87b06dc0c2e850fead07bcd13f85d45ad6b5a6e7ebee53cc8618772786", "firstIndex": 872415229},
+{"blockNumber": 6873878, "blockHash": "0x9a0ea103146da21fa1d9cab7ff609ec2c5e7f1856288a1f744239588e33904c2", "firstIndex": 939524080},
+{"blockNumber": 7080893, "blockHash": "0x35ad45055d7a1a3ef94efeb05e4122757bcc126ff1bc2b4ac72dc78ab3c0fd75", "firstIndex": 1006632327},
+{"blockNumber": 7266955, "blockHash": "0x7b80f70520f2c16eb5890f4963af8f60446f50af82b589ceaf009d09e704b302", "firstIndex": 1073741608},
+{"blockNumber": 7466649, "blockHash": "0xea8980b5c692c729013f9cc4c9f66e94d5ef2c5caad23f370be0953b6d85f32c", "firstIndex": 1140850335},
+{"blockNumber": 7661736, "blockHash": "0x615cdb03dbf29a22d59bbc769c03f3647c2c9abb7d1767f3f6529708209e7073", "firstIndex": 1207959232},
+{"blockNumber": 7834494, "blockHash": "0x05b7dd13e2eb8a833e4b278f9151ca01b294ae5fc5617769422ec85ffb446215", "firstIndex": 1275068098},
+{"blockNumber": 7989998, "blockHash": "0x4b5026add7f2fc2c02993b82c61bceba1b75cfcb8a78c5112cc2832bc0413be6", "firstIndex": 1342177025},
+{"blockNumber": 8142950, "blockHash": "0x9f56b7753a0e9964973f3e096e1385215e3aef232b448359e5bec05046b016a3", "firstIndex": 1409285545},
+{"blockNumber": 8297598, "blockHash": "0x2ddfeb161fe34ff73c4a995bb94256c3b522b39348169edb97bb3d876ccdf77d", "firstIndex": 1476394898},
+{"blockNumber": 8465061, "blockHash": "0xfdbe5435da5ae9fb34a9154e5ebe515afac9a128e383f798f1e69952be404fcd", "firstIndex": 1543503805},
+{"blockNumber": 8655740, "blockHash": "0x305137325a39a44aba997e214c05f32d965658f7e3fe20322f2e57e72f239977", "firstIndex": 1610612406},
+{"blockNumber": 8807102, "blockHash": "0xc32f9fd9a43d2b502dbcc4a683ffe324fca9a871753478bcb1f4fb9573da96a5", "firstIndex": 1677721343},
+{"blockNumber": 8911110, "blockHash": "0x806eabee7dc429b57afcb6c0b72b61728b4f13c62ee9942f1813790463f9ab5b", "firstIndex": 1744830172},
+{"blockNumber": 8960262, "blockHash": "0xa69f24032e9d5761565e726d484926a0932c6ca7fdb4c04a55bfc5e8f5879c4d", "firstIndex": 1811938729},
+{"blockNumber": 9085906, "blockHash": "0x922b4d6aeca64517f5048fa1ffa98d3e813bc415716a1763d07cf2bdea236896", "firstIndex": 1879048062},
+{"blockNumber": 9230838, "blockHash": "0xd892f513c48c95a859ae59c4e00ec4be0d684c549456c90e9c36669c11092f50", "firstIndex": 1946156580},
+{"blockNumber": 9390449, "blockHash": "0xcc0ced78fa66b570f338d129794d574560a23958e28d0d5288003d4542cec734", "firstIndex": 2013264376},
+{"blockNumber": 9515554, "blockHash": "0xc06d04737813c3e2c8e910b2d4543c5c684c7bfe235bb501445f68fd0663f3d9", "firstIndex": 2080374749},
+{"blockNumber": 9659367, "blockHash": "0x765fa33d9418b1ed648e88b497b4bf4aa66990e6413dfc155525cda61e3cc813", "firstIndex": 2147483344},
+{"blockNumber": 9793940, "blockHash": "0x9527dcbd85a100c51aee4ba90828fee23312f5f041859f3416a54fd87a437ce2", "firstIndex": 2214591669},
+{"blockNumber": 9923756, "blockHash": "0xc4da5fdd2de077459fe84f721d7fc01ef69683e03e634b0c5443186486792246", "firstIndex": 2281700949},
+{"blockNumber": 10042325, "blockHash": "0x6f554f195d20e4ba1262f73cf2c1cd44b14a71befe5ef0a5318fc309f8cde9fa", "firstIndex": 2348810131},
+{"blockNumber": 10168768, "blockHash": "0xdcdf88ec2c197d552343c80d01375a35d6462aff39c5bd976160f7176d2add64", "firstIndex": 2415918723},
+{"blockNumber": 10289467, "blockHash": "0x5ece23babcf6b8838695b891b044c3dcf8120cdc15acd83018e96ffc590f6cd0", "firstIndex": 2483027720},
+{"blockNumber": 10386601, "blockHash": "0xd398ee129aea1247019fa39dc2f5083819687d8f5ad17ec39ba1803d178fbcda", "firstIndex": 2550136043},
+{"blockNumber": 10479586, "blockHash": "0x61c1cccd60a546eacc29f2ae2a6facab402b4f35e2986c6629b302e6b1ebca3e", "firstIndex": 2617245577},
+{"blockNumber": 10562579, "blockHash": "0x14fd6a079050e3e110140372605c050945d6db03746fa5b84a8718cb2a0e9b86", "firstIndex": 2684353802},
+{"blockNumber": 10641438, "blockHash": "0x1998dcae09111b2c1bf6d0592e2f0b4a0143fe7db7115ff94b2b73b74542cee4", "firstIndex": 2751463334},
+{"blockNumber": 10717094, "blockHash": "0x9ce45a150bb6bd13687d0692c86716911ffc6aff5831a03f4015bdffb85f5093", "firstIndex": 2818571144},
+{"blockNumber": 10784494, "blockHash": "0xf65c936cf578e673696477f7db7189b47222fdf7ed13096564b9d47b15c86eb9", "firstIndex": 2885680709},
+{"blockNumber": 10848591, "blockHash": "0x84ea8e9d5789302ebe88888115488f91cfb75fd89e282dfbd29ba9d6225ab0ca", "firstIndex": 2952789491},
+{"blockNumber": 10909102, "blockHash": "0x22275047f643f4a3b82c349274a53d56a00da25beac89655877fb8cbfa3182c8", "firstIndex": 3019898024},
+{"blockNumber": 10972828, "blockHash": "0x38e8acfec19bb53ab9fe70dd3577ccd8bd7c41b4d468b8f099a19bf06ce56518", "firstIndex": 3087006561},
+{"blockNumber": 11036532, "blockHash": "0x478dac03f520e987a6e845cac1b3a756d69e4f5257f20758b8fc38456e7b6884", "firstIndex": 3154116463},
+{"blockNumber": 11102457, "blockHash": "0x1a36616b03351ebe481330fc86578f57d9da5c0dda59f4406d49b53a2fa9b6f6", "firstIndex": 3221224961},
+{"blockNumber": 11168092, "blockHash": "0xb23af483ca6dbc2fb8d2fada1f8189de2e155758ed9ecbbb20c234a30c7e093c", "firstIndex": 3288333510},
+{"blockNumber": 11233640, "blockHash": "0xdc4c80d8370823773425b31e70b3af59582b556514857d1e9a9b4c53b880a424", "firstIndex": 3355442901},
+{"blockNumber": 11300454, "blockHash": "0xdbd4d8e9c1756093037f431330db4604e5a3e67bb74f425f667633d5298fe34c", "firstIndex": 3422551962},
+{"blockNumber": 11367550, "blockHash": "0x9e00816bd5b39353657a99421d278091599a39f246bbf8c0898f83e6d70b7c7c", "firstIndex": 3489660743},
+{"blockNumber": 11431820, "blockHash": "0xd32d5e5871ce71acc9dcc7cf9d8cb9d964646182529c327a0ddf617814e27d74", "firstIndex": 3556769150},
+{"blockNumber": 11497031, "blockHash": "0xed823054d4b293294e3301d3131e2ae9f0ce63c9f07d2d80fc144e42d535d61d", "firstIndex": 3623878513},
+{"blockNumber": 11560027, "blockHash": "0x7098177efe77cfb5912dc63069fe366033111041ca5c539907a89e439c861d8f", "firstIndex": 3690987092},
+{"blockNumber": 11625049, "blockHash": "0x61fccb47104a076b905d390498a1b38c317348bb1567e69aa1b670bed7a6a13f", "firstIndex": 3758096043},
+{"blockNumber": 11690298, "blockHash": "0xaf6f9e9c92db9e155f1ffb35eb68765c09de1206c353c9a56a5d5fa53e2d88d2", "firstIndex": 3825204430},
+{"blockNumber": 11755003, "blockHash": "0xf9efb54f08a2b59086a607541005928f1bda54fa2c46fcc48352fa6aa461c03a", "firstIndex": 3892313541},
+{"blockNumber": 11819591, "blockHash": "0xa551deb3bd2ec0ca2683cb76adb8fb295d171a3787aa5ff071bd1fa94d3af82e", "firstIndex": 3959422322},
+{"blockNumber": 11883278, "blockHash": "0x60c7a359d4aeb09b63b64487fa7794aee87c41b11446f69ba83e24d5bce6947f", "firstIndex": 4026531239},
+{"blockNumber": 11948436, "blockHash": "0x1c46fb22653e5f9eb9503fe532151519845d05b121291766715973e027eb9c0c", "firstIndex": 4093639741},
+{"blockNumber": 12013084, "blockHash": "0x0b8d1da9e70c4494ba61190390c891230846e69d9b0be605df8b04c4199ffd1b", "firstIndex": 4160749561},
+{"blockNumber": 12078612, "blockHash": "0xb049af260be25c6153b67c227689b0556d8fb7d6033c3f61b8218f5c4ac5917a", "firstIndex": 4227857310},
+{"blockNumber": 12143549, "blockHash": "0x685dce41efbe2bab0ac9a4b9026d9e5faa3e01ae4637c8181cd2f1eac326fe6a", "firstIndex": 4294966232},
+{"blockNumber": 12207918, "blockHash": "0x53644ea8f781c6270f1136ed4bbcb956c98f12a4793c2f56f49b3105282b13ea", "firstIndex": 4362075410},
+{"blockNumber": 12272370, "blockHash": "0x503523e1a01cb7d68be0eb22f36280c0ee7a1fc802061a704e326fbb8efcfe27", "firstIndex": 4429184831},
+{"blockNumber": 12329342, "blockHash": "0xe6a1bf78dd827def078e23c768cf6e96666ee06b5fdc113380a4769d85f4bd33", "firstIndex": 4496293151},
+{"blockNumber": 12382311, "blockHash": "0x352fe793c16b1d8aea195af86ee9aec3dbaa83e7a32872a0d222a77e96fc0e57", "firstIndex": 4563402486},
+{"blockNumber": 12436970, "blockHash": "0x90276349edf0941bb3327dc1d749d0371fe550eaddafb5141fcacdd281de0976", "firstIndex": 4630510187},
+{"blockNumber": 12489954, "blockHash": "0x71894b1e1608aeba777662a6055e601ce6f17950dbe3844c3601e0f88cbd5848", "firstIndex": 4697619586},
+{"blockNumber": 12541655, "blockHash": "0xc34c7299041df72a81c4b86e81639d2dfcebfae603e287fc143881a636ce1188", "firstIndex": 4764728948},
+{"blockNumber": 12597319, "blockHash": "0x4c4dc71d8d2f1223ad0469ddfe09fa8934f64e3712ed83708d6e1b9c64eceebb", "firstIndex": 4831837390},
+{"blockNumber": 12651863, "blockHash": "0x406a642c8364fb61cbd896512a2d19ecdba3932ab09ac67cf48d91aa1a03f24e", "firstIndex": 4898946362},
+{"blockNumber": 12706391, "blockHash": "0xa7fc3739898f7ff5733896313b248fb0b306a132681609b8b9e2671b42dac3a1", "firstIndex": 4966054748},
+{"blockNumber": 12762840, "blockHash": "0x5a1048966dfa55a48248517c45683ce342671bf544117d2bac13d8e343563136", "firstIndex": 5033164303},
+{"blockNumber": 12816597, "blockHash": "0x97f5b87f422870a8ffeff70c12f4377dde96d248aa912cb64ebfaea51dc086b6", "firstIndex": 5100272541},
+{"blockNumber": 12872315, "blockHash": "0x432f7df7973fdea4e5abe2c09b634ca3b724e37a19d5ec23ca5810e4854c6b33", "firstIndex": 5167382243},
+{"blockNumber": 12929620, "blockHash": "0xa4524e3b4c5dadac91df2f29696532cfed377f47469fe7cfabf10baac945eea2", "firstIndex": 5234490748},
+{"blockNumber": 12988653, "blockHash": "0x9be70e31aeea164c370d8d183a1af45120693b13d360edc78214535c40ef1d63", "firstIndex": 5301600248},
+{"blockNumber": 13049074, "blockHash": "0x3076857bd17959c2a3df18357eab3ecbb7ca25457785b07074d2e885eb566fc2", "firstIndex": 5368707746},
+{"blockNumber": 13108823, "blockHash": "0x67b2764f6b53779e3d683def2c7659c6d92814152574073d24f10cd67200a22b", "firstIndex": 5435817680},
+{"blockNumber": 13175389, "blockHash": "0x0d41e34dbca58dec45e0509b6cbf027b2377220845a85caebbf3a14b8505e8ce", "firstIndex": 5502925085},
+{"blockNumber": 13237624, "blockHash": "0x723e0818d54ab1b1c53efc2f2f448c16c1f943af2033575c612c3c71228d4ddf", "firstIndex": 5570034948},
+{"blockNumber": 13298658, "blockHash": "0xd7f9f477b20bcf1379ccaaae701c9658a02b177861e0af9e873c63fe8b854f4b", "firstIndex": 5637144562},
+{"blockNumber": 13361166, "blockHash": "0x9f9e4130093269d59045f91344a13eb2786023e02099f53dd44854d50a7f5ffb", "firstIndex": 5704253260},
+{"blockNumber": 13421716, "blockHash": "0x28752f35ad2adeba342ef6aa51bca1c0abc757c4f712cf620c8a5e6a8576636e", "firstIndex": 5771361775},
+{"blockNumber": 13480516, "blockHash": "0x43f4ecf24c17b7a77621e2ca09c88c53b3407fb018f4bb3a032c212015b4e18d", "firstIndex": 5838469616},
+{"blockNumber": 13535688, "blockHash": "0x99ffcd9982219a4be45fa5f15622b288afe4f85da536956d39dccac1ec416444", "firstIndex": 5905579739},
+{"blockNumber": 13590487, "blockHash": "0xff7b74464a318037e0670c0be7a5b1b79418d1a42c7a3587c46adc1ef720c819", "firstIndex": 5972688013},
+{"blockNumber": 13646596, "blockHash": "0xd555ce5f2b65ffeec9b9e3e75a56f5f23d7353a1014bcc861efd7df6e5dc2c92", "firstIndex": 6039797525},
+{"blockNumber": 13702902, "blockHash": "0x3e8feab590c6631c63931fee83bf7e10234890bdf377b1b374fb70221b9ac9c0", "firstIndex": 6106905641},
+{"blockNumber": 13760908, "blockHash": "0x18e96fa585c57aad1f2a8dc9639d9f4fe53c3f38894862471b7cec17f97f3cb1", "firstIndex": 6174014330},
+{"blockNumber": 13819340, "blockHash": "0x25384bf2cef178e30c4d9e0abae59357055241fc715133b2bb078e8278e5296a", "firstIndex": 6241123906},
+{"blockNumber": 13877817, "blockHash": "0x4a84a0a7035dbfed0e1281fb1df162a195f2c516686a2e2a6ad605e6f2b3cf57", "firstIndex": 6308232690},
+{"blockNumber": 13935260, "blockHash": "0x6c14a8e563614edbad66698ebe911e997e27844f9d33c7069795379540e71a44", "firstIndex": 6375341148},
+{"blockNumber": 13993914, "blockHash": "0xc0991d1bc431c5215fb0218b0b541f2b4fae0994775e5eddb016965fe5b4a0c7", "firstIndex": 6442448947},
+{"blockNumber": 14051007, "blockHash": "0x854803ce364b6fe4446800e74448a5ee883e7d472e8a0347f1ad13bd347fa69f", "firstIndex": 6509558964},
+{"blockNumber": 14109063, "blockHash": "0xa3fadce7ab18a5fedcfed9e3b1f0cf2fb32151172c362327c602343bc26dde8f", "firstIndex": 6576668017},
+{"blockNumber": 14166701, "blockHash": "0xaa2443a38e2b0d6b01cf4b73de54d2b7e92c99475bd0802beebd86845c126d1b", "firstIndex": 6643774648},
+{"blockNumber": 14222367, "blockHash": "0xa27c78134c1fbf432c8caee7c9d2ef323fdde519a065804c9c8da6ca1615e04f", "firstIndex": 6710886238},
+{"blockNumber": 14277029, "blockHash": "0x5d0ed08b65db01e382c339dcb2715b2927fb472b0c1f4256841c386d785c3f3a", "firstIndex": 6777995247},
+{"blockNumber": 14330955, "blockHash": "0x3e5014d82c416549acfe2e26b13ff5ddf29ad74943ef5b94a65eca0a4d72d82e", "firstIndex": 6845102969},
+{"blockNumber": 14384139, "blockHash": "0x720e1fd687292019c3fa1812c39c73d20fde783c0196f43378a51cb4e0b46d65", "firstIndex": 6912212078},
+{"blockNumber": 14437534, "blockHash": "0xe4342ab6f363066d9c2514a870241ceb63df232d33659fdb3928faf433360bb4", "firstIndex": 6979321493},
+{"blockNumber": 14489073, "blockHash": "0xeb2f9923d816e773460eccfa42eb4ff2bbfc353b756ba274e98dc85c64ab1536", "firstIndex": 7046428879},
+{"blockNumber": 14541559, "blockHash": "0x55cc33721fa9022964f47c30f4146fee7cc80f5b4158d7ecea0ff661cbba3487", "firstIndex": 7113538857},
+{"blockNumber": 14594432, "blockHash": "0x676441bdc1ec430a9fbc05e56a0437a475606a253f9fdffc86f5f6286c159b5d", "firstIndex": 7180647956},
+{"blockNumber": 14644942, "blockHash": "0x03232a1d3c9b2f1f0c2a4b780c83bce66238d9abdeed71c2a8af9e97a53011b0", "firstIndex": 7247754981},
+{"blockNumber": 14695793, "blockHash": "0x6017f6d1d79188692e1a1a18de5f93b83b97905c8ff5bed5cef4b2e9f4d4d41b", "firstIndex": 7314863646},
+{"blockNumber": 14746186, "blockHash": "0x332c8ac0b2960fb1627407d45443201a0f6de7421b52af077b32fa6e018e5401", "firstIndex": 7381974499},
+{"blockNumber": 14798403, "blockHash": "0x4507ae1d9f27a3eada874dcf04112a5d4ee7155701afd3fb857fe58201a2dede", "firstIndex": 7449082752},
+{"blockNumber": 14848093, "blockHash": "0xff98c820c66c3beaaa3b195158c94f099148573a7f032295ba7156c13f78eaa8", "firstIndex": 7516191978},
+{"blockNumber": 14897508, "blockHash": "0x49c31d3433b197585f733d7c382ae16d9d78bf59c333a9b76362cc6c5fed270c", "firstIndex": 7583300663},
+{"blockNumber": 14950638, "blockHash": "0xf1a1609206788977841927271808fe1c42d059026385d025bf4cea0c0a05966b", "firstIndex": 7650408992},
+{"blockNumber": 15003958, "blockHash": "0xf9e6b41108d907672f29ba1df97ed83aca2ba933335e6b5ccfb5d4001807b33f", "firstIndex": 7717519105},
+{"blockNumber": 15056783, "blockHash": "0x126a5a5c22e878a4c42bea890d48cfe00edf7fbc27d805c88a5269540bc94a42", "firstIndex": 7784628173},
+{"blockNumber": 15108163, "blockHash": "0x5f8494ccacefce9fef8a95a52247f2670ed30bb5a3b3deb85c645e72c115d506", "firstIndex": 7851735261},
+{"blockNumber": 15159385, "blockHash": "0x0fed6c5aafb53e9cdb0d8620c3eef9b619fb3ae9559296618f634eddae33f0c6", "firstIndex": 7918844299},
+{"blockNumber": 15210872, "blockHash": "0x37bdb94b57044b67fbd9b9a7cdc728fcfd2066cf1904cab478fd42d41ec6e1f0", "firstIndex": 7985954119},
+{"blockNumber": 15264226, "blockHash": "0xc173ad8a791e2293efd8e0b2c9f088fac49233aabb1e72db6a045aa9904f4728", "firstIndex": 8053061840},
+{"blockNumber": 15315693, "blockHash": "0x985cf97bf28f0becb2b16abfe629a5f520596e45fc23f4936bc2f6298b8145f9", "firstIndex": 8120172048},
+{"blockNumber": 15364606, "blockHash": "0x593362ef8a82140f3dd3ff13d0ce50287dcb0b8b3d31ffcc7813c6be16f7438d", "firstIndex": 8187281275},
+{"blockNumber": 15412727, "blockHash": "0x5a6ff10acc48e9e8bcf963b042012ea094628b9488b8bf569783b3297c67eedd", "firstIndex": 8254388575},
+{"blockNumber": 15462642, "blockHash": "0x137f15c5d112b0fdc6e4c84393032030646122c3b0d86286143a1ab8bfacbac4", "firstIndex": 8321497214},
+{"blockNumber": 15509781, "blockHash": "0xe58a8ca223b7c03755d046a6212bc33fa9814f73efe52804d8eea7ae4b4bd7f9", "firstIndex": 8388606496},
+{"blockNumber": 15558625, "blockHash": "0x4127d527b58c7685a1237d228508ea77d40f33dbeffb5cc30e47e83a4b6152a5", "firstIndex": 8455714910},
+{"blockNumber": 15604075, "blockHash": "0x7d1c1bccfe5c23a1edd81246e6e55277f271cffef30bde22c9d1202f764dd96f", "firstIndex": 8522823324},
+{"blockNumber": 15651742, "blockHash": "0x732e5203a504fd97510325c1fb876476ca132831e78182367cd4b4efae1232c3", "firstIndex": 8589934155},
+{"blockNumber": 15700810, "blockHash": "0xe031decb5e42a69dafcc814441227085508cf51c9c77f35d06487db19bd05a6a", "firstIndex": 8657041374},
+{"blockNumber": 15762855, "blockHash": "0xbdff9b1e193c44f35bf38945d2523c74ef59c4f352efc93fad02ef34eef1c5f0", "firstIndex": 8724150924},
+{"blockNumber": 15814304, "blockHash": "0x524a226d9f48e37db00e252c7f6a1350d1829d38fe83c756021cc11c56d3b2dd", "firstIndex": 8791259373},
+{"blockNumber": 15865514, "blockHash": "0x4e4ad46f3ae7fb2a2e7cb94aa804929cb7353c79c78cc2cbcee710da5f0ec4b5", "firstIndex": 8858369571},
+{"blockNumber": 15920411, "blockHash": "0xa39e4cbd8e84b427e44b8d06b5b1149652f3af5fa88633b802665b8677772bc5", "firstIndex": 8925477782},
+{"blockNumber": 15974193, "blockHash": "0x4a3f82935e6fcda6ce7636001989b65b79422d362e60833e01185a7960b2fa55", "firstIndex": 8992587369},
+{"blockNumber": 16032739, "blockHash": "0x7f50bb76dfd01125e0a4a7595106e43329eb8bd02e819be2ade4248a403c714a", "firstIndex": 9059696611},
+{"blockNumber": 16090896, "blockHash": "0xdc4db130973ef793d2223844e35122a2477e642d86c69616fb3604d2a79805c4", "firstIndex": 9126804425},
+{"blockNumber": 16150783, "blockHash": "0xb46294fe1cbcbbdff7309ebf0bf481e78e2589d386b95fa4c781c437e02e0aec", "firstIndex": 9193912139},
+{"blockNumber": 16207278, "blockHash": "0x8a3af0387572b56989bb6c98d8860e07b5cdefb2ebe7b1dc58be07c24a53e056", "firstIndex": 9261022215},
+{"blockNumber": 16262389, "blockHash": "0xec5a03d88cfc1bf8c278ddce7ace5032c3f7ed5032b4575df54779ec1abca131", "firstIndex": 9328130680},
+{"blockNumber": 16319525, "blockHash": "0x74ae702375e242205fe4986448b0ee494d4a38ccc813981489c4b0488e58a949", "firstIndex": 9395240779},
+{"blockNumber": 16373441, "blockHash": "0x5f404555cf45c547ef0fc044f2701bc7a28ce1e432de06e0264dbfd1c17447d0", "firstIndex": 9462347671},
+{"blockNumber": 16423343, "blockHash": "0xe4af890b6c4dd3b10af2e6c08f14563bd2a3121b22a3cc6a0db29d438c1dcc59", "firstIndex": 9529457704},
+{"blockNumber": 16474687, "blockHash": "0x019a8b374cf150fcda1c8af2a3e7ba63603ae81744d6343897c4b17ee58411d5", "firstIndex": 9596566169},
+{"blockNumber": 16525504, "blockHash": "0xad394ce7e84e1c904cc5dd6dabd51b1a575c0b2fb006f10f272875b7f921904e", "firstIndex": 9663675287},
+{"blockNumber": 16574051, "blockHash": "0x098e2dca2eda0d700aa3127bdfb679ce16e3ca297a305978c6ecfff044269511", "firstIndex": 9730784563},
+{"blockNumber": 16622457, "blockHash": "0x8627f1237731c1127116898724f1b6420b8e7bcd8e18a0ea568f192bbd2cd55d", "firstIndex": 9797893241},
+{"blockNumber": 16672410, "blockHash": "0x3e32b46faa670b191b89bea84a68022d5ea8b90008b388a948ff27f871d17292", "firstIndex": 9865002964},
+{"blockNumber": 16719964, "blockHash": "0x1668c3ea6cb256f306e93851cc2c00612f85df5cf7b5527ffce14c89088d7fdf", "firstIndex": 9932111708},
+{"blockNumber": 16769014, "blockHash": "0xed9a6f88e47b5bfe4d5bc92e1c1bfeb252f716f0990ffa1f907655c0f5e3a365", "firstIndex": 9999220312},
+{"blockNumber": 16818134, "blockHash": "0x93504ba617500698a1b46f3110cd26827b8efe468190b2a91d3595784e34a1b6", "firstIndex": 10066329167},
+{"blockNumber": 16869369, "blockHash": "0x94ca4b395a5c36c4e9d07cbb08271f0680455af4e4659906b647e16abebd7d4e", "firstIndex": 10133438415},
+{"blockNumber": 16921428, "blockHash": "0x66650775bff4cb95de56099c10b06f3dfe302a731b391f406d1aa551121308ba", "firstIndex": 10200547148},
+{"blockNumber": 16974677, "blockHash": "0x58d5fc4cd45047dac10d55a036e0a4448faacc1041b29c4c7fb41a6e9394c3ff", "firstIndex": 10267655623},
+{"blockNumber": 17031050, "blockHash": "0x9d15b831979e95fc4149ecae46478cac1e54470b277d57d7102d19a72f28eeba", "firstIndex": 10334764441},
+{"blockNumber": 17086163, "blockHash": "0x978ff48863fcdb50b274ea4220e7635c13c41bae91912db9e24205fb917fde85", "firstIndex": 10401873176},
+{"blockNumber": 17141086, "blockHash": "0x4084d442499893361731f17534944d260cccd8e07d41c52d7160910555770b90", "firstIndex": 10468982462},
+{"blockNumber": 17190735, "blockHash": "0x42ed86fb42def23870df6ec23204f557669ec7cd8de59c9f3ef9ab90704e4b11", "firstIndex": 10536090726},
+{"blockNumber": 17237026, "blockHash": "0x9a49d943f82a7335f8049df4d953080f25920e3b098f6a2d872520166b225961", "firstIndex": 10603199938},
+{"blockNumber": 17287007, "blockHash": "0xac3d189676d24eadc62adcf97d8730a001a7766fe3ac27b18e745ca9727f6c46", "firstIndex": 10670308186},
+{"blockNumber": 17338498, "blockHash": "0xe50e251aa0a3209fa8a0030b237cf8357b08eeb80492a432c8aa5969a820ee42", "firstIndex": 10737417623},
+{"blockNumber": 17389178, "blockHash": "0x1e18cedcfa6c81bc845c9136fc92f9f27ca8fff8847289bb139f374f0f6ba733", "firstIndex": 10804526769},
+{"blockNumber": 17442038, "blockHash": "0x07f47ebbee99938fb96beb1ffadf8266f71d0d73f393172f0e24facd40d45c97", "firstIndex": 10871635940},
+{"blockNumber": 17497589, "blockHash": "0x247a5e852e36e46f7897ad1875e93d810ff4434bb835b469afae3a3d3d4a9c9a", "firstIndex": 10938744251},
+{"blockNumber": 17554111, "blockHash": "0x420e420a1b599e4dc3bd115652b4d6f5eedc391086ef064f97a9cbffc02b62f9", "firstIndex": 11005853686},
+{"blockNumber": 17608282, "blockHash": "0xb2ba644d3593d3c832f932f14ffec532e26e9e1cdcd8fbe7477f46e238c6bffd", "firstIndex": 11072962303},
+{"blockNumber": 17664091, "blockHash": "0x6d0bfff8145652a617bfe03ab87e3af09219cdb822c9c214b42b16f20742c1eb", "firstIndex": 11140070260},
+{"blockNumber": 17714917, "blockHash": "0x61509544b5174d3ff5d48816ff54f2929c8a400ef21454116056baa136b607b6", "firstIndex": 11207179013},
+{"blockNumber": 17763850, "blockHash": "0x8da83012621f42670601406c902ce6e48e77c6285dcc6983e108e69b3a8d5d2d", "firstIndex": 11274288732},
+{"blockNumber": 17814186, "blockHash": "0xc37bda200856589c220dc206e31186d6c12761fd193f5bc7123cf331fbbed7f5", "firstIndex": 11341397343},
+{"blockNumber": 17864461, "blockHash": "0x99af43893dc9f54c55abbdbd7b000e27248145371e9b00db116dfb961b36749c", "firstIndex": 11408506122},
+{"blockNumber": 17913212, "blockHash": "0x9b642236d63d756fcd3404713bc6abe97804b46a9e665ae5f1d9aa48e693fe78", "firstIndex": 11475614347},
+{"blockNumber": 17961497, "blockHash": "0xaec9b1e851cf51646673ec7698e1d5964d44eb04729ad11385f6ae84d497070a", "firstIndex": 11542723713},
+{"blockNumber": 18010859, "blockHash": "0xb15fb7fc308516cad5f1dadfb7dd92fd3fe612308f67154cf2a241990aa15bc4", "firstIndex": 11609833204},
+{"blockNumber": 18061052, "blockHash": "0xd8ea94b8b8a7380d21dc433abb69973f3de2eeb339f4aaffdb79f85a5d86929d", "firstIndex": 11676938565},
+{"blockNumber": 18111466, "blockHash": "0x5b5cc6d182bffaf06692a7efda8cad5d4742fdbce5c1146b3be3c5be10ea046a", "firstIndex": 11744048539},
+{"blockNumber": 18166001, "blockHash": "0x901d8401c795542c75069cd917781fa587ad416da0a5298f144fae2f14305705", "firstIndex": 11811159949},
+{"blockNumber": 18218505, "blockHash": "0x5894285023f34ec6fd501cd90b605e118b9688ca7c01082289bf69a6987142bc", "firstIndex": 11878268395},
+{"blockNumber": 18271037, "blockHash": "0x3f0fce315fd993d966b6c599ebd360462466e67046d512dc430ac07f75243bae", "firstIndex": 11945377050},
+{"blockNumber": 18322778, "blockHash": "0xfa6536b3b06e7b579e53a8d01368182808c950586e39e12485be3145c3acbf43", "firstIndex": 12012486036},
+{"blockNumber": 18372240, "blockHash": "0x5531859d5b8a4093020e2fb971808d0eeaddd7d61dc58d70f34923081f97336f", "firstIndex": 12079595358},
+{"blockNumber": 18421630, "blockHash": "0xfeadff676fe8f04d0362b77ea2d7c3ea59b05d68cc6b2cc95cfe751558b1bfa7", "firstIndex": 12146703872},
+{"blockNumber": 18471472, "blockHash": "0x352b0a17e4160d334b876eb180d686fd67c7ddc40a78219b0bc4b59f5ae5912d", "firstIndex": 12213813174},
+{"blockNumber": 18522088, "blockHash": "0xf7d408309246089531b47c787049c7ffd2820c1b87b4804033a458bc193a3432", "firstIndex": 12280921181},
+{"blockNumber": 18572725, "blockHash": "0x44126d4a5b43594b09d26e0b0a5dc859c074865cb9b11ab78c80f28406fd0240", "firstIndex": 12348029615},
+{"blockNumber": 18623223, "blockHash": "0xd3aec1f98cea30e870d0f530194e9a72e225eb45eefe435e38da5967eb2fa470", "firstIndex": 12415139522},
+{"blockNumber": 18675228, "blockHash": "0x24deaef2508038bddf8e9020f95128ae34eb1786f786c3f98df0bdf95f3e97cc", "firstIndex": 12482248123},
+{"blockNumber": 18725495, "blockHash": "0x2e5a7edd7337b20e862741e116680a472089426dbc2b0eb5f1c984f8d6da935b", "firstIndex": 12549357121},
+{"blockNumber": 18778174, "blockHash": "0xe734a1084fa5eeff427bba8e8d3623181a30dd9bad128fc22de9b15dffd0e549", "firstIndex": 12616466131},
+{"blockNumber": 18834807, "blockHash": "0x0924a55ef4f9c2605261433c5b236d47645484efa44b34dc9f15b7c2e6b47a74", "firstIndex": 12683575000},
+{"blockNumber": 18883054, "blockHash": "0xef638ec491a1cf685b22bf4072e41710440d6133a53e31250762825247d3eb64", "firstIndex": 12750683712},
+{"blockNumber": 18933405, "blockHash": "0x6e9eefc2e121d13e6c04544ea370fe8543afeb36fef3a8e2ea68afb079738c8c", "firstIndex": 12817792510},
+{"blockNumber": 18988033, "blockHash": "0x4c98c0dbb6a7217609c4b277fe7ff9e2c32c8d000ee628a3f37b2b296e1ca421", "firstIndex": 12884901389},
+{"blockNumber": 19041096, "blockHash": "0x61a219a23d111cd07861ff47b89d4b2840c0ab5421d5d55a6f1fe43da3f33b98", "firstIndex": 12952010147},
+{"blockNumber": 19088914, "blockHash": "0xefb26ba58447e50e9b820e0b0d16c08da45793a824af12a2e1f38b6f46b7e36a", "firstIndex": 13019118891},
+{"blockNumber": 19140413, "blockHash": "0x91dbb13623fc4d5a4e462dfa0a03c92bf5301948dd4b2c3e5fddd2bc88856d47", "firstIndex": 13086228137},
+{"blockNumber": 19191876, "blockHash": "0xdd7f97af7ad206b2636656ed4722fe1c47c6ba9fa5531066785acbe4f8d4c4ad", "firstIndex": 13153336551},
+{"blockNumber": 19237651, "blockHash": "0xf1e875f314314abe3654b667c6443c772dd67c525c52877b06515ea8fc5a391b", "firstIndex": 13220444812},
+{"blockNumber": 19291032, "blockHash": "0x9eda269ac47957b619d2a8c408b5efc8cfb168a46bfde351f43325fa337fce47", "firstIndex": 13287553967},
+{"blockNumber": 19344225, "blockHash": "0x62859b78d45f589f679fc6a3bf266a2e0aa906fd61d787755ec51cfadc05eaf6", "firstIndex": 13354663338},
+{"blockNumber": 19394709, "blockHash": "0x7cda3d193bcfe8ddbc89b85e98c814dbbe31a54ccab52ab7738ecd003061f3c8", "firstIndex": 13421772549},
+{"blockNumber": 19442850, "blockHash": "0xd95f3f8bab5b91c3e1f848c9455936850bf92cf6a5086b817f6f81031bc0f81e", "firstIndex": 13488880704},
+{"blockNumber": 19488177, "blockHash": "0x3a1a9623ee332d6decada9359cf84eeafb81fffa997d6cf4f3efcc92e98e9310", "firstIndex": 13555989570},
+{"blockNumber": 19534365, "blockHash": "0x5c7a8ee7cbba4894ea412cc335895a9a7d3369d35779bcdc97a6c24b6a81a7ba", "firstIndex": 13623097446},
+{"blockNumber": 19579409, "blockHash": "0x4280f5f76cb0f9947f9da1cb9de6c4b6e65b37b3a3c174f5fb29d360cf389ffb", "firstIndex": 13690206790},
+{"blockNumber": 19621406, "blockHash": "0x0a98ca72e97fd295c7c73263b17185d74dbe427f41c616a5d5454faa5a1c63a0", "firstIndex": 13757316653},
+{"blockNumber": 19664868, "blockHash": "0xc8c53d89b7a849861f3daaf9503e6397a0340cecb085850c3636bee6ea2b7189", "firstIndex": 13824424608},
+{"blockNumber": 19709011, "blockHash": "0xf65babc88ef6a9eb2ffce9a6844adf895995f65a2facd1956f5df17ca68dbf94", "firstIndex": 13891534141},
+{"blockNumber": 19755170, "blockHash": "0xf903882449eeb86d69af33b6797099275e82d8787334a02a42a2168dbd739b95", "firstIndex": 13958642627},
+{"blockNumber": 19803666, "blockHash": "0xa8a975f570db082a3771790bb1fe405667dde4674ff49e31092687fa92727e36", "firstIndex": 14025752433},
+{"blockNumber": 19847730, "blockHash": "0x89d479a4b4d9faa7af3e3f61d13a7a9c80a11c4adf7cd018859eb0ddba30bdd5", "firstIndex": 14092860598},
+{"blockNumber": 19894138, "blockHash": "0x740278e6dc79cfd2b030780868a1b8e1520dbae0e8d3ee0a3ef43c84c83c6e0a", "firstIndex": 14159952129},
+{"blockNumber": 19938334, "blockHash": "0x05ae7ce9813235c19baa57658bc5c8e160130328a1fd2cc58d2baf2d9a4ae097", "firstIndex": 14227078393},
+{"blockNumber": 19985138, "blockHash": "0x511c8876a93a77fa473ae0d29e25d8163162c09b7212f506be0ea73840ea3971", "firstIndex": 14294187397},
+{"blockNumber": 20028221, "blockHash": "0x64379baedf429cfa65df076142a7d9909d7e369d6aa6b8e161bb686944d169ff", "firstIndex": 14361296446},
+{"blockNumber": 20071555, "blockHash": "0x4f43d8b1091d2bd0c6b080ce51bce08aae6be605e00889be3cdd17cb80614ff8", "firstIndex": 14428405409},
+{"blockNumber": 20113611, "blockHash": "0xebbf87409ef6732a53513414926b0aa998a7269f9e9717a3acbe96a33c2de937", "firstIndex": 14495514255},
+{"blockNumber": 20156628, "blockHash": "0x70218108c0669f42c4ac1afff80947cf4a0a2fd8e9300b31d45bbe330a5159c9", "firstIndex": 14562622257},
+{"blockNumber": 20199886, "blockHash": "0x1d6aaa4b041eb0cc9caed34347bb943e50a139caf8998d781ff4f06d4aa727dc", "firstIndex": 14629732017},
+{"blockNumber": 20244148, "blockHash": "0x37fed57b6945a815ec56c84c6261e90cf9f3883d9cbffc62b70d9722c1f0f363", "firstIndex": 14696839730},
+{"blockNumber": 20288249, "blockHash": "0x1d9fe664431e093c4ef6139ad35fea767a2e66825e605cfea9e91d1738665619", "firstIndex": 14763948803},
+{"blockNumber": 20333358, "blockHash": "0x1ac7c1362791507057b7e87d7118b4f61c2919c8ebdb498ecf7d64a59ba3d2e2", "firstIndex": 14831058814},
+{"blockNumber": 20376855, "blockHash": "0xf6a19eac8a402f591079a4aa7ff57349c350ec96d683a031e4d85c05bc0541b2", "firstIndex": 14898167076},
+{"blockNumber": 20421474, "blockHash": "0xe8767aa7516b632dc8c0fed3a366273ef14bd7b8c2212998b7105fb80b4c0d9b", "firstIndex": 14965276285},
+{"blockNumber": 20466974, "blockHash": "0xafa2d2e6c9594762c163c339069cb15aa9fd27d301268892e12f99a3f2912ade", "firstIndex": 15032384674},
+{"blockNumber": 20512183, "blockHash": "0xc1c51f9372d781d7ac0a1a7aa052503bbde3e1706bf65f225b0f6a6eed93d055", "firstIndex": 15099493832},
+{"blockNumber": 20557251, "blockHash": "0xa905087c15e2b12b19cd23f20089b471cd9815dcff73842d2043e13a276fcc55", "firstIndex": 15166601921},
+{"blockNumber": 20595654, "blockHash": "0x9e7772dd946f65e7569f34f6896d8e597ac8ff88700b871de21ff578c0c995e6", "firstIndex": 15233711474},
+{"blockNumber": 20638340, "blockHash": "0xc08b46af83db6f0e5d1acad7020bacd59d0f18df72651d32c9b60edd0351a886", "firstIndex": 15300820584},
+{"blockNumber": 20683359, "blockHash": "0x7ec93a1f1d211c24d93ce7e344634efef680a6388374cf3831d250ecd63711f9", "firstIndex": 15367929168},
+{"blockNumber": 20728401, "blockHash": "0x4ad4124c2595abcf4477cac4eb70c0c9372bbd48f2066c7ac5b8e9f94817ab48", "firstIndex": 15435038303},
+{"blockNumber": 20771272, "blockHash": "0x68fd6b12fdb4add8b2957105a21b571c7dcb114c6c2ca02f5fa40730a108f864", "firstIndex": 15502146509},
+{"blockNumber": 20814862, "blockHash": "0x4e7f640bf3c135b2442ca4c744023021af163e8d45e8d776fb7051c49d79fcff", "firstIndex": 15569256173},
+{"blockNumber": 20857627, "blockHash": "0x9b023bffed12530b5a68db263bb621a147e0cf02b30ec39a82b82a1fdeb7ca36", "firstIndex": 15636364955},
+{"blockNumber": 20896662, "blockHash": "0x8c3adf694b8520ed1edd38282b2f58cf383426176628b5be6330a56882e2a07e", "firstIndex": 15703473445},
+{"blockNumber": 20939156, "blockHash": "0x0649e2559f767da945a674c9d6c700628109e18d0757ad33496a228a827413a4", "firstIndex": 15770581979},
+{"blockNumber": 20981054, "blockHash": "0xebf0ecca671b833f77df70106cf1171d4dd3fa0f062607e4845569073997f1cd", "firstIndex": 15837691319},
+{"blockNumber": 21022980, "blockHash": "0x07479520c08a727c42e3616aca7fc4435773f4ef277d84369190263159606d71", "firstIndex": 15904799391},
+{"blockNumber": 21068139, "blockHash": "0x669cfd03b293a61815588cb9cef928a512d328c1fe9191fe6b281573f429f8b8", "firstIndex": 15971908604},
+{"blockNumber": 21112368, "blockHash": "0x98957aedaf8b0483be208e8d16a3e6d73ff217e732f3649650671c92c8fb57a5", "firstIndex": 16039007815},
+{"blockNumber": 21155756, "blockHash": "0xc301e40988fbe19b1254ab3abe2f78fe0cd35259898be02476179a041926f777", "firstIndex": 16106127130},
+{"blockNumber": 21200717, "blockHash": "0x8d35b199ee65c5ccf4ca0fc22b50282850cd88b72580679f1568eea67d3431e8", "firstIndex": 16173235593},
+{"blockNumber": 21244396, "blockHash": "0xe7ec084eb8a7125fe26c4ef64fdbb3e0a727b2d882fb1e4fdad1d7fa3dc55fd2", "firstIndex": 16240343803},
+{"blockNumber": 21290031, "blockHash": "0x8bb31b883d30875b7ed8a01c35a6f51e603b8d974c4e74af20fb9c723e5ac5df", "firstIndex": 16307452772},
+{"blockNumber": 21335893, "blockHash": "0xd0976b7c946a7556e607e47a02d2cfcccd259508e2828b9c4faa5c7105a597a8", "firstIndex": 16374562657},
+{"blockNumber": 21378611, "blockHash": "0x72d42953f013aeb7c84bb22c9e07828b92f73885b20c208af612874df8fe85ed", "firstIndex": 16441671192},
+{"blockNumber": 21421343, "blockHash": "0x663b3cddcf3f7cfa59ed29b4894e43fa784c07576cfec897ff1a3838235ecb50", "firstIndex": 16508779766},
+{"blockNumber": 21466598, "blockHash": "0xe04ebd60289dbcc09a27e053e7b240a59e928e2a76a85a6fa4044ee387c46d18", "firstIndex": 16575888836},
+{"blockNumber": 21511717, "blockHash": "0x5ea9863a0b504f381010ff383d60a56390ba9cef162d9d5a077b1884d2adac98", "firstIndex": 16642998234},
+{"blockNumber": 21550050, "blockHash": "0x2a7b3e635c80521887ca3ebda04541090ec720dd817cd06fcba61e7ae0f78a4f", "firstIndex": 16710106733},
+{"blockNumber": 21592440, "blockHash": "0xc7095268d0d53cb5a069c2ea7107cc42172bb1b817f04154938e6b52bdbb369c", "firstIndex": 16777215261},
+{"blockNumber": 21636007, "blockHash": "0x55f463b932053ebfc02728a12c2cdfb2cb4700a115cde686c9ab22bb4c336ecb", "firstIndex": 16844324844},
+{"blockNumber": 21680799, "blockHash": "0x4521d473aa7f5150f920c66fe639b20f30cb373b32e425fcb7dcce9803b506be", "firstIndex": 16911431255},
+{"blockNumber": 21725330, "blockHash": "0x793876007ce3594df603628766b139a02273ceee43d5b3ca2d0ca2f13a5fc5d9", "firstIndex": 16978541626},
+{"blockNumber": 21771310, "blockHash": "0xd0db23bb28d6e39917348b1879dd560c73354ea6d7d0bbee0b7a8fcbcede1091", "firstIndex": 17045650902},
+{"blockNumber": 21811348, "blockHash": "0x8165948c08db34aeb8cc2a3e331a00af2d09fbff9c466019fb2f2892ca3f7b71", "firstIndex": 17112760024},
+{"blockNumber": 21851615, "blockHash": "0x3083e8f4fbb4d726ef19b1a5bce20e639a0418eac384f9967f24c7697a7e2dfc", "firstIndex": 17179868870},
+{"blockNumber": 21894041, "blockHash": "0x9b0776a020374e4985f6a7e1199596bedd387ff085f65224d724ab11955c56fe", "firstIndex": 17246977342},
+{"blockNumber": 21932218, "blockHash": "0xe9c50eddc21759d3ab790e2babba56f4b8d38eb739f1b561bc337a1ba6e15393", "firstIndex": 17314085541},
+{"blockNumber": 21959188, "blockHash": "0x6b9c482cc4822af2ad62a359b923062556ab6a046d1a39a8243b764848690601", "firstIndex": 17381193886},
+{"blockNumber": 21994911, "blockHash": "0x0d99ef9dd42dd1c62fc5249eb4f618e7672ab93fa0ba7545c77360371cf972e5", "firstIndex": 17448302795},
+{"blockNumber": 22026007, "blockHash": "0xf7cdc7f6694f2c2ed31fa8a607f65cfa59d0dd7d7424ab5c017f959ae2982c71", "firstIndex": 17515413036},
+{"blockNumber": 22059890, "blockHash": "0x82b768a0dddefda2eefd3a33596ea2be075312f1dd4b01f6b0d377faca2af98b", "firstIndex": 17582521768},
+{"blockNumber": 22090784, "blockHash": "0xf97c2eaf9a550360ac24000c0ff17ffa388a2bdd6f73f2f36718e332edfa107a", "firstIndex": 17649630983},
+{"blockNumber": 22121157, "blockHash": "0xa790025235db782e899f23d8b09663ec2d74ec149e4125d62989f98829b08e2d", "firstIndex": 17716724973},
+{"blockNumber": 22148056, "blockHash": "0xbe25ac4f1bdd89a7db5782bf1157e6c4378d80220d67a029397853ef16cd1a4c", "firstIndex": 17783838366},
+{"blockNumber": 22168652, "blockHash": "0x6ae43618c915e636794e2cc2d75dde9992766881c7405fe6479c045ed4bee57e", "firstIndex": 17850956277},
+{"blockNumber": 22190975, "blockHash": "0x9437121647899a4b7b84d67fbea7cc6ff967481c2eab4328ccd86e2cefe19420", "firstIndex": 17918066140},
+{"blockNumber": 22234357, "blockHash": "0x036030830134f9224160d5a0b62da35ec7813dc8855d554bd22e9d38545243ed", "firstIndex": 17985175075},
+{"blockNumber": 22276736, "blockHash": "0x5ceb96d98aa1b4c1c2f2fa253ae9cdb1b04e0420c11bf795400e8762c0a1635c", "firstIndex": 18052284344},
+{"blockNumber": 22321282, "blockHash": "0x8a601ebf6a757020c6d43a978f0bd2c150c4acc1ffdd50c7ee88afc78b0c11f8", "firstIndex": 18119392242},
+{"blockNumber": 22349231, "blockHash": "0xb751c026a92ba5be95ad7ea4e2729a175b0d0e11a4c108f47cab232b4715d1a2", "firstIndex": 18186501218},
+{"blockNumber": 22377469, "blockHash": "0xa47916860a22f7e26761ec2d7f717410791bd3ed0237b2f6266750214c7bbf08", "firstIndex": 18253610249},
+{"blockNumber": 22422685, "blockHash": "0x8beaee39603af55fad222730f556c840c41cd76a5eef0bad367ac94d3b86c7aa", "firstIndex": 18320716377},
+{"blockNumber": 22462378, "blockHash": "0x6dba9c5d2949f5a6a072267b590e8b15e6fb157a0fc22719387f1fd6bfcd8d5d", "firstIndex": 18387828426},
+{"blockNumber": 22500185, "blockHash": "0x2484c380df0a8f7edfdf8d917570d23fab8499aea80c35b6cf4e5fe1e34106e9", "firstIndex": 18454936227},
+{"blockNumber": 22539624, "blockHash": "0xd418071906803d25afc3842a6a6468ad3b5fea27107b314ce4e2ccf08b478acf", "firstIndex": 18522044531},
+{"blockNumber": 22577021, "blockHash": "0xff222982693f3ff60d2097822171f80a6ddd979080aeb7e995bfb1b973497c84", "firstIndex": 18589154438},
+{"blockNumber": 22614525, "blockHash": "0x9868da1fea2ffca3f67e35570f02eb5707b27f6967ea4a109eb4ddbf24566efd", "firstIndex": 18656264174},
+{"blockNumber": 22652848, "blockHash": "0x060a911da11ab0f1dda307f5196e622d23901d198925749e70ab58a439477c5a", "firstIndex": 18723372617},
+{"blockNumber": 22692432, "blockHash": "0x6a937f2c283aba8c778c1f5ef340b225fd820f8a7dfa6f24f5fe541994f32f2d", "firstIndex": 18790480232},
+{"blockNumber": 22731200, "blockHash": "0x00d57a9e7a2dad252436fe9f0382c6a8860d301a9f9ffe6d7ac64c82b95300f8", "firstIndex": 18857590076},
+{"blockNumber": 22769000, "blockHash": "0xa48db20307c19c373ef2d31d85088ea14b8df0450491c31982504c87b04edbc0", "firstIndex": 18924699130},
+{"blockNumber": 22808126, "blockHash": "0x1419c64ff003edca0586f1c8ec3063da5c54c57ff826cfb34bc866cc18949653", "firstIndex": 18991807807},
+{"blockNumber": 22845231, "blockHash": "0x691f87217e61c5d7ae9ad53a44d30e1ab6b1cc3f2b689b9fbf7c38fbacacfe3e", "firstIndex": 19058917062},
+{"blockNumber": 22884189, "blockHash": "0x7f102d44c0ea7803f5b0e1a98a6abf0e8383eb99fb114d6f7b4591753ce8bba3", "firstIndex": 19126024122},
+{"blockNumber": 22920923, "blockHash": "0x04fe6179495016fc3fe56d8ef5311c360a5761a898262173849c3494fdd73d92", "firstIndex": 19193134595},
+{"blockNumber": 22958100, "blockHash": "0xe38e0ff7b0c4065ca42ea577bc32f2566ca46f2ddeedcc4bc1f8fb00e7f26329", "firstIndex": 19260242424},
+{"blockNumber": 22988600, "blockHash": "0x04ca74758b22e0ea54b8c992022ff21c16a2af9c45144c3b0f80de921a7eee82", "firstIndex": 19327351273},
+{"blockNumber": 23018392, "blockHash": "0x61cc979b00bc97b48356f986a5b9ec997d674bc904c2a2e4b0f17de08e50b3bb", "firstIndex": 19394459627},
+{"blockNumber": 23048524, "blockHash": "0x489de15d95739ede4ab15e8b5151d80d4dc85ae10e7be800b1a4723094a678df", "firstIndex": 19461570073}
]
diff --git a/core/filtermaps/checkpoints_sepolia.json b/core/filtermaps/checkpoints_sepolia.json
index 234af955e8..7214a4df1f 100644
--- a/core/filtermaps/checkpoints_sepolia.json
+++ b/core/filtermaps/checkpoints_sepolia.json
@@ -1,99 +1,99 @@
[
-{"blockNumber": 3246675, "blockId": "0x36bf7de9e1f151963088ca3efa206b6e78411d699d2f64f3bf86895294275e0b", "firstIndex": 67108765},
-{"blockNumber": 3575560, "blockId": "0x4652e3bee440f946aeaa3358c9a62b9bc4d41f663737ab00fded00c2662bfa29", "firstIndex": 134217644},
-{"blockNumber": 3694262, "blockId": "0x433573f97e563ca04fa53e0d0eb019d0ffe9dd032a05ee5b33ce7705aedfd34d", "firstIndex": 201316990},
-{"blockNumber": 3725628, "blockId": "0x1e25ebd76b9e2a2007bfdd7c82433cce7dae8beb190d249c82bf83d0cd177735", "firstIndex": 268426577},
-{"blockNumber": 3795378, "blockId": "0xca999083e01d5947212fb878d529abecdd18a6012997b0d5326afcf6908f17e5", "firstIndex": 335543438},
-{"blockNumber": 3856681, "blockId": "0x48148cd35de3de6c626f76ce5d97cfc6acd88772af69c0aef3d640c29ba82096", "firstIndex": 402641717},
-{"blockNumber": 3869367, "blockId": "0x08bafccf9ece7b72e4e069fc3c7493fbf25732881758917df2e8f9a51fc2c75f", "firstIndex": 469761225},
-{"blockNumber": 3938346, "blockId": "0xc638eb890c0e65ce1169ef0bb6b74dede2f4f6144ff8a5739c4894ff41a22fa2", "firstIndex": 536865235},
-{"blockNumber": 3984892, "blockId": "0x6a7782f0765b0f0c406b3cb107fd8bb14c6906293d9bdc5cc00e081bfd1ae59f", "firstIndex": 603972697},
-{"blockNumber": 4002660, "blockId": "0xda473de053602bc42fcbb074a32c908303f7f6db2ed8c38192665dce80276702", "firstIndex": 671087183},
-{"blockNumber": 4113149, "blockId": "0xd1782f677262226194725028858da83e672748e409fe1c21bdd77c07be0662c8", "firstIndex": 738197365},
-{"blockNumber": 4260733, "blockId": "0xc356600819e49d311963312157c481fd9be43a36ad040a0512fd17559c78d394", "firstIndex": 805305957},
-{"blockNumber": 4391073, "blockId": "0x06f903dc765b4b0ca1a40847eb74d30b0b51904a70c46f2e836dc5b3ea3dd8d3", "firstIndex": 872415067},
-{"blockNumber": 4515567, "blockId": "0xe99d3849755a1b4ec9a65d003260c111d4886e29ad6243fc750c72dbc79dbe4c", "firstIndex": 939523746},
-{"blockNumber": 4634815, "blockId": "0x9f281e749b192ef58242ff9fccd496a9075c30bd28c61ed4ce6487fc2d167e5c", "firstIndex": 1006629595},
-{"blockNumber": 4718286, "blockId": "0x23d2a40ec3ae059a45006ba651a70f308b6b282829b9cfdf2534e4311e34050f", "firstIndex": 1073731435},
-{"blockNumber": 4753427, "blockId": "0x6df7295f9ccb45a95e6cc72a8ee56bda992513b3bf4baa3d62219871a0b5a912", "firstIndex": 1140843105},
-{"blockNumber": 4786511, "blockId": "0x705c79435411c1af81385a1eca34343427eceb3f6af72e7f665552d6b0eceb6c", "firstIndex": 1207957185},
-{"blockNumber": 4811696, "blockId": "0x69a6e60d1b958b469feafb1dca97f38f97620a90c21ed92f90080f6443ee6c78", "firstIndex": 1275061401},
-{"blockNumber": 4841770, "blockId": "0x0a0e9d2d3d177601a478428297b5b9a42124166fefe42f2221c7a612c31e7ca1", "firstIndex": 1342174442},
-{"blockNumber": 4914785, "blockId": "0x5ca0e12230aeacbce9064dff15dafb9f24e20b4a737f7d04df8549008b829fa3", "firstIndex": 1409284680},
-{"blockNumber": 4992516, "blockId": "0xb9cb1700d8b65615fc3a019b2e9b2ece3b4f1e0e2fcfc01ff40bcf53e7291b10", "firstIndex": 1476387066},
-{"blockNumber": 5088617, "blockId": "0x33122f693a264fb2ab626232fac5e2714010fd65bdceb772109511f7c9497333", "firstIndex": 1543503849},
-{"blockNumber": 5155024, "blockId": "0x187716822dd6ab9c21138ae5a8e763197a5e7ec48bf0e756c5e6fa2c9f2f24bf", "firstIndex": 1610604247},
-{"blockNumber": 5204373, "blockId": "0x753b6495979ca90a1b915edad6af33c58bbabe68b45f95c7645113ec1d35d0e6", "firstIndex": 1677721200},
-{"blockNumber": 5269952, "blockId": "0xcbb0409ed824631120d6951b55537143d9a1c82ca6dd6255d2eb31c57844685c", "firstIndex": 1744829248},
-{"blockNumber": 5337623, "blockId": "0x0b460658511c4ae5205c038430789f9b31f3c08ba4acc06d273b7a13a149cf8b", "firstIndex": 1811939255},
-{"blockNumber": 5399044, "blockId": "0x9d07e37321e2b019f4837cc4f95f103da87ecdcb77e131e9d42d8643b0446524", "firstIndex": 1879041131},
-{"blockNumber": 5422692, "blockId": "0x9d9e27614635989788e32f15cd7a0abf7470dccf3336b77547baeb3423b629ca", "firstIndex": 1946154968},
-{"blockNumber": 5454259, "blockId": "0x01e694443d471d3411fe0b52720fd4a189e687b14cc8ede116f5e93f0ad96bb6", "firstIndex": 2013265148},
-{"blockNumber": 5498863, "blockId": "0xc4cb906c96d9f80b830b8add25318c38b3ff857c1bd577cd9cae0a324a068ffc", "firstIndex": 2080373677},
-{"blockNumber": 5554788, "blockId": "0x9b934b2d66723559d82e0f4cd49a2497234fe892248cb2f46c13f6a9585cfddd", "firstIndex": 2147475372},
-{"blockNumber": 5594711, "blockId": "0x840c0b7889b8c882cfa0eceb516dd73043b51c0e5a0c1be849a0e2a5dda7e842", "firstIndex": 2214592267},
-{"blockNumber": 5645179, "blockId": "0x188873a43506fee33349bc33df4a55059401e98dc7f0a5d488a0bc5c87dfd431", "firstIndex": 2281695869},
-{"blockNumber": 5687634, "blockId": "0x7ea8d7a2afccf795a2a02eb00746d89db57f13f8936cf7ab0f0a7db1b1ceb341", "firstIndex": 2348806824},
-{"blockNumber": 5727808, "blockId": "0x3f3eaf36c8e0271403737676907eac15312927c1d60ce814337654b7d7f2b2bd", "firstIndex": 2415915517},
-{"blockNumber": 5784480, "blockId": "0x10ec5adf01449c29550f5117f09ecafc49fc614d7da5886cf01619950827f850", "firstIndex": 2483023750},
-{"blockNumber": 5843906, "blockId": "0x9e6e14b3fba395a9ae7b3f1295f33a12d0f8b9f10e1aaab1649a90a56a323676", "firstIndex": 2550136398},
-{"blockNumber": 5906280, "blockId": "0x2188869a164e9c19232f6702a5233dc27a2304839159dd7230cc93ebdfc6c048", "firstIndex": 2617245012},
-{"blockNumber": 5977929, "blockId": "0xb247181ed597d9473c2c6e5762ae45a05f328d8df215fdac83d19ae4c028109a", "firstIndex": 2684347033},
-{"blockNumber": 6051479, "blockId": "0x0e8634c95dac31b7c835b44d0a14290e9212b31b9c3f621997ad1cca2d8ab0ba", "firstIndex": 2751463372},
-{"blockNumber": 6118422, "blockId": "0xa642c4457d4b63d68a5502bdba2355616e34495bf3ab2d8518a45ded54a16be4", "firstIndex": 2818563217},
-{"blockNumber": 6174274, "blockId": "0xbb9808bc56b5f14636e654bdd6d50d35b31345255926341ef3ddb54840095bcc", "firstIndex": 2885680274},
-{"blockNumber": 6276207, "blockId": "0x7ae027a18889e1b60a46ea3e5f2759035f3d9041de68eca0b154068ec533db59", "firstIndex": 2952789656},
-{"blockNumber": 6368344, "blockId": "0x8b4401f89589d9e99259e6c05c18b1780e96b65027caf9d83edf80390d317f66", "firstIndex": 3019898378},
-{"blockNumber": 6470718, "blockId": "0x0e2f8de093392c38750581c41e93c2fb1c0f6676b65c9af52792f3c56e894457", "firstIndex": 3087007589},
-{"blockNumber": 6553237, "blockId": "0xa6c4f3a1fe3b4e116c367fcbb1ae98e0f25570fdb2f1eadee2681deb39462c1e", "firstIndex": 3154116356},
-{"blockNumber": 6663750, "blockId": "0x9049748ddf655c7cc2ccbe2d36010b3a67ae011d114d32b1c812a7a53102f27f", "firstIndex": 3221225375},
-{"blockNumber": 6766894, "blockId": "0xc98188fb313b1beb7fc4f415b4a74c363903b3b5e0172437d79446bc011238b5", "firstIndex": 3288334172},
-{"blockNumber": 6886576, "blockId": "0x4ae8237a94198c82c266f01e472fdcd3ae150c84ecf714205a9be672f1e705b8", "firstIndex": 3355443040},
-{"blockNumber": 6978746, "blockId": "0xb2b69ce29c5c9c46c5cbc6e96260c34313d8b0b4a739cf4091c98f37093aa732", "firstIndex": 3422552017},
-{"blockNumber": 7098871, "blockId": "0x9ab2b681feeb38287a1b689fa64a50efd3407b6331fd6896187cad98a29ce107", "firstIndex": 3489660879},
-{"blockNumber": 7203023, "blockId": "0xdedfff5985c1c93d476120d2afbaa61967bae4edabb9711c7794a1b1cd453aa1", "firstIndex": 3556769365},
-{"blockNumber": 7256709, "blockId": "0xc078041cffc1802b6342fd2ba5a92ff8076ac0f5b72b4a77ba75cd486fe1b3c9", "firstIndex": 3623876563},
-{"blockNumber": 7307744, "blockId": "0xdf8dcdd4af9213a79587eb2f4d91f4111da74f99207ee05e4ae50bcb4f8d5435", "firstIndex": 3690986225},
-{"blockNumber": 7369268, "blockId": "0x62315823621a38a1f138e193b85259988fcc15f4a74d0cf19a412920bcdcad98", "firstIndex": 3758096030},
-{"blockNumber": 7445109, "blockId": "0x8b668ac1274a8dcc9526eb115c2f67bd56f6657b5072c7542da2252daca10fef", "firstIndex": 3825204921},
-{"blockNumber": 7511512, "blockId": "0x056ad7411652aa64849c67705a86e42221cb88f7fb6a7012ab7d3d5b9a30eb76", "firstIndex": 3892313524},
-{"blockNumber": 7557259, "blockId": "0xef660b97661f073b7c7319b69451e16d26ecaed54f9237e209999d137da2bfd4", "firstIndex": 3959415096},
-{"blockNumber": 7606260, "blockId": "0x9f5f06ad381166261780864bf17ad4d3cdd475f87ba51fa7f9ba621bf0ac0acb", "firstIndex": 4026521703},
-{"blockNumber": 7654244, "blockId": "0xefe5b537435001203741bf445ca26466baf7ad3f63c2bba94f0f658aed8c244b", "firstIndex": 4093627258},
-{"blockNumber": 7700433, "blockId": "0x89af6e8c5aa934a9139944b3de7a15232069479dba387e0fdd043ad36156c8bf", "firstIndex": 4160749200},
-{"blockNumber": 7730079, "blockId": "0x08e0d511a93e2a9c004b3456d77d687ae86247417ada1cdd1f85332a62dfed1d", "firstIndex": 4227846795},
-{"blockNumber": 7777515, "blockId": "0xeba8faed2e12bb9ed5e7fad19dd82662f15f6f4f512a0d232eedf1e676712428", "firstIndex": 4294966082},
-{"blockNumber": 7828860, "blockId": "0x62c054bd24be351dc4777460ee922a75fdcea5c9830455cbac6fa29552719c85", "firstIndex": 4362076087},
-{"blockNumber": 7869890, "blockId": "0x5664bcfa6151f798781e22bea4f9a2c796d097402350cb131e4e3c78e13e461a", "firstIndex": 4429183267},
-{"blockNumber": 7911722, "blockId": "0x9a85e48e3135c97c51fc1786f2af0596c802e021b6c53cfca65a129cafcd23ed", "firstIndex": 4496287265},
-{"blockNumber": 7960147, "blockId": "0xc9359cc76d7090e1c8a031108f0ab7a8935d971efd4325fe53612a1d99562f6f", "firstIndex": 4563402388},
-{"blockNumber": 8030418, "blockId": "0x21867e68cd8327aed2da2601399d60f7f9e41dca4a4f2f9be982e5a2b9304a88", "firstIndex": 4630511616},
-{"blockNumber": 8087701, "blockId": "0x0fa8c8d7549cc9a8d308262706fe248efe759f8b63511efb1e7f3926e9af2dcb", "firstIndex": 4697614758},
-{"blockNumber": 8149130, "blockId": "0x655ea638fd9e35cc25f4332f260d7bf98f4f6fa9a72e1bff861209f18659e94c", "firstIndex": 4764727744},
-{"blockNumber": 8208672, "blockId": "0xb5847a670dc3b6181f9e2e40e4218548048366d237a0d12e938b9879bc8cf800", "firstIndex": 4831837882},
-{"blockNumber": 8271345, "blockId": "0x96797214946f29093883b877ccb0f2a9f771a9a3db3794a642b5dcb781c4d194", "firstIndex": 4898942160},
-{"blockNumber": 8302858, "blockId": "0x6a5977b3382ca69a9e0412333f97b911c1f69f857d8f31dd0fc930980e24f2fc", "firstIndex": 4966054626},
-{"blockNumber": 8333618, "blockId": "0x2547294aa23b67c42adbdddfcf424b17a95c4ff0f352a6a2442c529cfb0c892a", "firstIndex": 5033163605},
-{"blockNumber": 8360582, "blockId": "0xf34f5dceb0ef22e0f782b56c12790472acc675997b9c45075bd4e18a9dacd03c", "firstIndex": 5100273631},
-{"blockNumber": 8387230, "blockId": "0x0fbea42e87620b5beeb76b67febc173847c54333d7dce9fa2f8f2a3fa9c8c22a", "firstIndex": 5167381673},
-{"blockNumber": 8414795, "blockId": "0x6c9c000cf5e35da3a7e9e1cd56147c8ce9b43a76d6de945675efd9dc03b628c9", "firstIndex": 5234477010},
-{"blockNumber": 8444749, "blockId": "0xba85f8c9abaddc34e2113eb49385667ba4b008168ae701f46aa7a7ce78c633a1", "firstIndex": 5301598562},
-{"blockNumber": 8474551, "blockId": "0x720866a40242f087dd25b6f0dd79224884f435b114a39e60c5669f5c942c78c1", "firstIndex": 5368707262},
-{"blockNumber": 8501310, "blockId": "0x2b6da233532c701202fb5ac67e005f7d3eb71f88a9fac10c25d24dd11ada05e5", "firstIndex": 5435803858},
-{"blockNumber": 8526970, "blockId": "0x005f9bbad0a10234129d09894d7fcf04bf1398d326510eedb4195808c282802d", "firstIndex": 5502926509},
-{"blockNumber": 8550412, "blockId": "0x37c9f3efc9f33cf62f590087c8c9ac70011883f75e648647a6fd0fec00ca627c", "firstIndex": 5570034950},
-{"blockNumber": 8573540, "blockId": "0x81cfb46a07be7c70bb8a0f76b03a4cd502f92032bea68ad7ba10e26351673000", "firstIndex": 5637137662},
-{"blockNumber": 8590416, "blockId": "0x5c223d58ef22d7b0dd8c498e8498da4787b5dc706681c2bc83849441f5d0922d", "firstIndex": 5704252906},
-{"blockNumber": 8616793, "blockId": "0x9043ce02742fb5ec43a696602867b7ce6003a95b36cd28a37eeb9785a46ad49f", "firstIndex": 5771357264},
-{"blockNumber": 8647290, "blockId": "0xd90115193764b0a33f3f2a719381b3ddbce2532607c72fb287a864eb391eeada", "firstIndex": 5838466144},
-{"blockNumber": 8673192, "blockId": "0x9bc92d340cccaf4c8c03372efc24eb92c5159106729de8d2e9e064f5568d082b", "firstIndex": 5905577457},
-{"blockNumber": 8700694, "blockId": "0xb3d656a173b962bc6825198e94a4974289db06a8998060bd0f5ee2044a7a7deb", "firstIndex": 5972679345},
-{"blockNumber": 8724533, "blockId": "0x253ffc6d77b88fe18736e4c313e9930341c444bc87b2ee22b26cfe8d9d0b178d", "firstIndex": 6039795829},
-{"blockNumber": 8743948, "blockId": "0x04eb66d0261705d31e629193148d0685058d7759ba5f95d2d38e412dbadb8256", "firstIndex": 6106901747},
-{"blockNumber": 8758378, "blockId": "0x64adf54e662d11db716610157da672c3d8b45f001dbce40a269871b86a84d026", "firstIndex": 6174011544},
-{"blockNumber": 8777722, "blockId": "0x0a7f9a956024b404c915e70b42221aa027b2dd715b0697f099dccefae0b9af97", "firstIndex": 6241124215},
-{"blockNumber": 8800154, "blockId": "0x411f90dc18f2bca31fa63615c2866c907bbac1fae8c06782cabfaf788efba665", "firstIndex": 6308233216},
-{"blockNumber": 8829725, "blockId": "0x5686f3a5eec1b070d0113c588f8f4a560d57ad96b8045cedb5c08bbadaa0273e", "firstIndex": 6375340033},
-{"blockNumber": 8858036, "blockId": "0x4f9b5d9fac9c6f6e2224f613cda12e8ab95d636774ce87489dce8a9f805ee2e5", "firstIndex": 6442450330},
-{"blockNumber": 8884811, "blockId": "0x9cf74f978872683802c065e72b5a5326fdad95f19733c34d927b575cd85fd0bd", "firstIndex": 6509559380}
+{"blockNumber": 3246675, "blockHash": "0x36bf7de9e1f151963088ca3efa206b6e78411d699d2f64f3bf86895294275e0b", "firstIndex": 67108765},
+{"blockNumber": 3575560, "blockHash": "0x4652e3bee440f946aeaa3358c9a62b9bc4d41f663737ab00fded00c2662bfa29", "firstIndex": 134217644},
+{"blockNumber": 3694262, "blockHash": "0x433573f97e563ca04fa53e0d0eb019d0ffe9dd032a05ee5b33ce7705aedfd34d", "firstIndex": 201316990},
+{"blockNumber": 3725628, "blockHash": "0x1e25ebd76b9e2a2007bfdd7c82433cce7dae8beb190d249c82bf83d0cd177735", "firstIndex": 268426577},
+{"blockNumber": 3795378, "blockHash": "0xca999083e01d5947212fb878d529abecdd18a6012997b0d5326afcf6908f17e5", "firstIndex": 335543438},
+{"blockNumber": 3856681, "blockHash": "0x48148cd35de3de6c626f76ce5d97cfc6acd88772af69c0aef3d640c29ba82096", "firstIndex": 402641717},
+{"blockNumber": 3869367, "blockHash": "0x08bafccf9ece7b72e4e069fc3c7493fbf25732881758917df2e8f9a51fc2c75f", "firstIndex": 469761225},
+{"blockNumber": 3938346, "blockHash": "0xc638eb890c0e65ce1169ef0bb6b74dede2f4f6144ff8a5739c4894ff41a22fa2", "firstIndex": 536865235},
+{"blockNumber": 3984892, "blockHash": "0x6a7782f0765b0f0c406b3cb107fd8bb14c6906293d9bdc5cc00e081bfd1ae59f", "firstIndex": 603972697},
+{"blockNumber": 4002660, "blockHash": "0xda473de053602bc42fcbb074a32c908303f7f6db2ed8c38192665dce80276702", "firstIndex": 671087183},
+{"blockNumber": 4113149, "blockHash": "0xd1782f677262226194725028858da83e672748e409fe1c21bdd77c07be0662c8", "firstIndex": 738197365},
+{"blockNumber": 4260733, "blockHash": "0xc356600819e49d311963312157c481fd9be43a36ad040a0512fd17559c78d394", "firstIndex": 805305957},
+{"blockNumber": 4391073, "blockHash": "0x06f903dc765b4b0ca1a40847eb74d30b0b51904a70c46f2e836dc5b3ea3dd8d3", "firstIndex": 872415067},
+{"blockNumber": 4515567, "blockHash": "0xe99d3849755a1b4ec9a65d003260c111d4886e29ad6243fc750c72dbc79dbe4c", "firstIndex": 939523746},
+{"blockNumber": 4634815, "blockHash": "0x9f281e749b192ef58242ff9fccd496a9075c30bd28c61ed4ce6487fc2d167e5c", "firstIndex": 1006629595},
+{"blockNumber": 4718286, "blockHash": "0x23d2a40ec3ae059a45006ba651a70f308b6b282829b9cfdf2534e4311e34050f", "firstIndex": 1073731435},
+{"blockNumber": 4753427, "blockHash": "0x6df7295f9ccb45a95e6cc72a8ee56bda992513b3bf4baa3d62219871a0b5a912", "firstIndex": 1140843105},
+{"blockNumber": 4786511, "blockHash": "0x705c79435411c1af81385a1eca34343427eceb3f6af72e7f665552d6b0eceb6c", "firstIndex": 1207957185},
+{"blockNumber": 4811696, "blockHash": "0x69a6e60d1b958b469feafb1dca97f38f97620a90c21ed92f90080f6443ee6c78", "firstIndex": 1275061401},
+{"blockNumber": 4841770, "blockHash": "0x0a0e9d2d3d177601a478428297b5b9a42124166fefe42f2221c7a612c31e7ca1", "firstIndex": 1342174442},
+{"blockNumber": 4914785, "blockHash": "0x5ca0e12230aeacbce9064dff15dafb9f24e20b4a737f7d04df8549008b829fa3", "firstIndex": 1409284680},
+{"blockNumber": 4992516, "blockHash": "0xb9cb1700d8b65615fc3a019b2e9b2ece3b4f1e0e2fcfc01ff40bcf53e7291b10", "firstIndex": 1476387066},
+{"blockNumber": 5088617, "blockHash": "0x33122f693a264fb2ab626232fac5e2714010fd65bdceb772109511f7c9497333", "firstIndex": 1543503849},
+{"blockNumber": 5155024, "blockHash": "0x187716822dd6ab9c21138ae5a8e763197a5e7ec48bf0e756c5e6fa2c9f2f24bf", "firstIndex": 1610604247},
+{"blockNumber": 5204373, "blockHash": "0x753b6495979ca90a1b915edad6af33c58bbabe68b45f95c7645113ec1d35d0e6", "firstIndex": 1677721200},
+{"blockNumber": 5269952, "blockHash": "0xcbb0409ed824631120d6951b55537143d9a1c82ca6dd6255d2eb31c57844685c", "firstIndex": 1744829248},
+{"blockNumber": 5337623, "blockHash": "0x0b460658511c4ae5205c038430789f9b31f3c08ba4acc06d273b7a13a149cf8b", "firstIndex": 1811939255},
+{"blockNumber": 5399044, "blockHash": "0x9d07e37321e2b019f4837cc4f95f103da87ecdcb77e131e9d42d8643b0446524", "firstIndex": 1879041131},
+{"blockNumber": 5422692, "blockHash": "0x9d9e27614635989788e32f15cd7a0abf7470dccf3336b77547baeb3423b629ca", "firstIndex": 1946154968},
+{"blockNumber": 5454259, "blockHash": "0x01e694443d471d3411fe0b52720fd4a189e687b14cc8ede116f5e93f0ad96bb6", "firstIndex": 2013265148},
+{"blockNumber": 5498863, "blockHash": "0xc4cb906c96d9f80b830b8add25318c38b3ff857c1bd577cd9cae0a324a068ffc", "firstIndex": 2080373677},
+{"blockNumber": 5554788, "blockHash": "0x9b934b2d66723559d82e0f4cd49a2497234fe892248cb2f46c13f6a9585cfddd", "firstIndex": 2147475372},
+{"blockNumber": 5594711, "blockHash": "0x840c0b7889b8c882cfa0eceb516dd73043b51c0e5a0c1be849a0e2a5dda7e842", "firstIndex": 2214592267},
+{"blockNumber": 5645179, "blockHash": "0x188873a43506fee33349bc33df4a55059401e98dc7f0a5d488a0bc5c87dfd431", "firstIndex": 2281695869},
+{"blockNumber": 5687634, "blockHash": "0x7ea8d7a2afccf795a2a02eb00746d89db57f13f8936cf7ab0f0a7db1b1ceb341", "firstIndex": 2348806824},
+{"blockNumber": 5727808, "blockHash": "0x3f3eaf36c8e0271403737676907eac15312927c1d60ce814337654b7d7f2b2bd", "firstIndex": 2415915517},
+{"blockNumber": 5784480, "blockHash": "0x10ec5adf01449c29550f5117f09ecafc49fc614d7da5886cf01619950827f850", "firstIndex": 2483023750},
+{"blockNumber": 5843906, "blockHash": "0x9e6e14b3fba395a9ae7b3f1295f33a12d0f8b9f10e1aaab1649a90a56a323676", "firstIndex": 2550136398},
+{"blockNumber": 5906280, "blockHash": "0x2188869a164e9c19232f6702a5233dc27a2304839159dd7230cc93ebdfc6c048", "firstIndex": 2617245012},
+{"blockNumber": 5977929, "blockHash": "0xb247181ed597d9473c2c6e5762ae45a05f328d8df215fdac83d19ae4c028109a", "firstIndex": 2684347033},
+{"blockNumber": 6051479, "blockHash": "0x0e8634c95dac31b7c835b44d0a14290e9212b31b9c3f621997ad1cca2d8ab0ba", "firstIndex": 2751463372},
+{"blockNumber": 6118422, "blockHash": "0xa642c4457d4b63d68a5502bdba2355616e34495bf3ab2d8518a45ded54a16be4", "firstIndex": 2818563217},
+{"blockNumber": 6174274, "blockHash": "0xbb9808bc56b5f14636e654bdd6d50d35b31345255926341ef3ddb54840095bcc", "firstIndex": 2885680274},
+{"blockNumber": 6276207, "blockHash": "0x7ae027a18889e1b60a46ea3e5f2759035f3d9041de68eca0b154068ec533db59", "firstIndex": 2952789656},
+{"blockNumber": 6368344, "blockHash": "0x8b4401f89589d9e99259e6c05c18b1780e96b65027caf9d83edf80390d317f66", "firstIndex": 3019898378},
+{"blockNumber": 6470718, "blockHash": "0x0e2f8de093392c38750581c41e93c2fb1c0f6676b65c9af52792f3c56e894457", "firstIndex": 3087007589},
+{"blockNumber": 6553237, "blockHash": "0xa6c4f3a1fe3b4e116c367fcbb1ae98e0f25570fdb2f1eadee2681deb39462c1e", "firstIndex": 3154116356},
+{"blockNumber": 6663750, "blockHash": "0x9049748ddf655c7cc2ccbe2d36010b3a67ae011d114d32b1c812a7a53102f27f", "firstIndex": 3221225375},
+{"blockNumber": 6766894, "blockHash": "0xc98188fb313b1beb7fc4f415b4a74c363903b3b5e0172437d79446bc011238b5", "firstIndex": 3288334172},
+{"blockNumber": 6886576, "blockHash": "0x4ae8237a94198c82c266f01e472fdcd3ae150c84ecf714205a9be672f1e705b8", "firstIndex": 3355443040},
+{"blockNumber": 6978746, "blockHash": "0xb2b69ce29c5c9c46c5cbc6e96260c34313d8b0b4a739cf4091c98f37093aa732", "firstIndex": 3422552017},
+{"blockNumber": 7098871, "blockHash": "0x9ab2b681feeb38287a1b689fa64a50efd3407b6331fd6896187cad98a29ce107", "firstIndex": 3489660879},
+{"blockNumber": 7203023, "blockHash": "0xdedfff5985c1c93d476120d2afbaa61967bae4edabb9711c7794a1b1cd453aa1", "firstIndex": 3556769365},
+{"blockNumber": 7256709, "blockHash": "0xc078041cffc1802b6342fd2ba5a92ff8076ac0f5b72b4a77ba75cd486fe1b3c9", "firstIndex": 3623876563},
+{"blockNumber": 7307744, "blockHash": "0xdf8dcdd4af9213a79587eb2f4d91f4111da74f99207ee05e4ae50bcb4f8d5435", "firstIndex": 3690986225},
+{"blockNumber": 7369268, "blockHash": "0x62315823621a38a1f138e193b85259988fcc15f4a74d0cf19a412920bcdcad98", "firstIndex": 3758096030},
+{"blockNumber": 7445109, "blockHash": "0x8b668ac1274a8dcc9526eb115c2f67bd56f6657b5072c7542da2252daca10fef", "firstIndex": 3825204921},
+{"blockNumber": 7511512, "blockHash": "0x056ad7411652aa64849c67705a86e42221cb88f7fb6a7012ab7d3d5b9a30eb76", "firstIndex": 3892313524},
+{"blockNumber": 7557259, "blockHash": "0xef660b97661f073b7c7319b69451e16d26ecaed54f9237e209999d137da2bfd4", "firstIndex": 3959415096},
+{"blockNumber": 7606260, "blockHash": "0x9f5f06ad381166261780864bf17ad4d3cdd475f87ba51fa7f9ba621bf0ac0acb", "firstIndex": 4026521703},
+{"blockNumber": 7654244, "blockHash": "0xefe5b537435001203741bf445ca26466baf7ad3f63c2bba94f0f658aed8c244b", "firstIndex": 4093627258},
+{"blockNumber": 7700433, "blockHash": "0x89af6e8c5aa934a9139944b3de7a15232069479dba387e0fdd043ad36156c8bf", "firstIndex": 4160749200},
+{"blockNumber": 7730079, "blockHash": "0x08e0d511a93e2a9c004b3456d77d687ae86247417ada1cdd1f85332a62dfed1d", "firstIndex": 4227846795},
+{"blockNumber": 7777515, "blockHash": "0xeba8faed2e12bb9ed5e7fad19dd82662f15f6f4f512a0d232eedf1e676712428", "firstIndex": 4294966082},
+{"blockNumber": 7828860, "blockHash": "0x62c054bd24be351dc4777460ee922a75fdcea5c9830455cbac6fa29552719c85", "firstIndex": 4362076087},
+{"blockNumber": 7869890, "blockHash": "0x5664bcfa6151f798781e22bea4f9a2c796d097402350cb131e4e3c78e13e461a", "firstIndex": 4429183267},
+{"blockNumber": 7911722, "blockHash": "0x9a85e48e3135c97c51fc1786f2af0596c802e021b6c53cfca65a129cafcd23ed", "firstIndex": 4496287265},
+{"blockNumber": 7960147, "blockHash": "0xc9359cc76d7090e1c8a031108f0ab7a8935d971efd4325fe53612a1d99562f6f", "firstIndex": 4563402388},
+{"blockNumber": 8030418, "blockHash": "0x21867e68cd8327aed2da2601399d60f7f9e41dca4a4f2f9be982e5a2b9304a88", "firstIndex": 4630511616},
+{"blockNumber": 8087701, "blockHash": "0x0fa8c8d7549cc9a8d308262706fe248efe759f8b63511efb1e7f3926e9af2dcb", "firstIndex": 4697614758},
+{"blockNumber": 8149130, "blockHash": "0x655ea638fd9e35cc25f4332f260d7bf98f4f6fa9a72e1bff861209f18659e94c", "firstIndex": 4764727744},
+{"blockNumber": 8208672, "blockHash": "0xb5847a670dc3b6181f9e2e40e4218548048366d237a0d12e938b9879bc8cf800", "firstIndex": 4831837882},
+{"blockNumber": 8271345, "blockHash": "0x96797214946f29093883b877ccb0f2a9f771a9a3db3794a642b5dcb781c4d194", "firstIndex": 4898942160},
+{"blockNumber": 8302858, "blockHash": "0x6a5977b3382ca69a9e0412333f97b911c1f69f857d8f31dd0fc930980e24f2fc", "firstIndex": 4966054626},
+{"blockNumber": 8333618, "blockHash": "0x2547294aa23b67c42adbdddfcf424b17a95c4ff0f352a6a2442c529cfb0c892a", "firstIndex": 5033163605},
+{"blockNumber": 8360582, "blockHash": "0xf34f5dceb0ef22e0f782b56c12790472acc675997b9c45075bd4e18a9dacd03c", "firstIndex": 5100273631},
+{"blockNumber": 8387230, "blockHash": "0x0fbea42e87620b5beeb76b67febc173847c54333d7dce9fa2f8f2a3fa9c8c22a", "firstIndex": 5167381673},
+{"blockNumber": 8414795, "blockHash": "0x6c9c000cf5e35da3a7e9e1cd56147c8ce9b43a76d6de945675efd9dc03b628c9", "firstIndex": 5234477010},
+{"blockNumber": 8444749, "blockHash": "0xba85f8c9abaddc34e2113eb49385667ba4b008168ae701f46aa7a7ce78c633a1", "firstIndex": 5301598562},
+{"blockNumber": 8474551, "blockHash": "0x720866a40242f087dd25b6f0dd79224884f435b114a39e60c5669f5c942c78c1", "firstIndex": 5368707262},
+{"blockNumber": 8501310, "blockHash": "0x2b6da233532c701202fb5ac67e005f7d3eb71f88a9fac10c25d24dd11ada05e5", "firstIndex": 5435803858},
+{"blockNumber": 8526970, "blockHash": "0x005f9bbad0a10234129d09894d7fcf04bf1398d326510eedb4195808c282802d", "firstIndex": 5502926509},
+{"blockNumber": 8550412, "blockHash": "0x37c9f3efc9f33cf62f590087c8c9ac70011883f75e648647a6fd0fec00ca627c", "firstIndex": 5570034950},
+{"blockNumber": 8573540, "blockHash": "0x81cfb46a07be7c70bb8a0f76b03a4cd502f92032bea68ad7ba10e26351673000", "firstIndex": 5637137662},
+{"blockNumber": 8590416, "blockHash": "0x5c223d58ef22d7b0dd8c498e8498da4787b5dc706681c2bc83849441f5d0922d", "firstIndex": 5704252906},
+{"blockNumber": 8616793, "blockHash": "0x9043ce02742fb5ec43a696602867b7ce6003a95b36cd28a37eeb9785a46ad49f", "firstIndex": 5771357264},
+{"blockNumber": 8647290, "blockHash": "0xd90115193764b0a33f3f2a719381b3ddbce2532607c72fb287a864eb391eeada", "firstIndex": 5838466144},
+{"blockNumber": 8673192, "blockHash": "0x9bc92d340cccaf4c8c03372efc24eb92c5159106729de8d2e9e064f5568d082b", "firstIndex": 5905577457},
+{"blockNumber": 8700694, "blockHash": "0xb3d656a173b962bc6825198e94a4974289db06a8998060bd0f5ee2044a7a7deb", "firstIndex": 5972679345},
+{"blockNumber": 8724533, "blockHash": "0x253ffc6d77b88fe18736e4c313e9930341c444bc87b2ee22b26cfe8d9d0b178d", "firstIndex": 6039795829},
+{"blockNumber": 8743948, "blockHash": "0x04eb66d0261705d31e629193148d0685058d7759ba5f95d2d38e412dbadb8256", "firstIndex": 6106901747},
+{"blockNumber": 8758378, "blockHash": "0x64adf54e662d11db716610157da672c3d8b45f001dbce40a269871b86a84d026", "firstIndex": 6174011544},
+{"blockNumber": 8777722, "blockHash": "0x0a7f9a956024b404c915e70b42221aa027b2dd715b0697f099dccefae0b9af97", "firstIndex": 6241124215},
+{"blockNumber": 8800154, "blockHash": "0x411f90dc18f2bca31fa63615c2866c907bbac1fae8c06782cabfaf788efba665", "firstIndex": 6308233216},
+{"blockNumber": 8829725, "blockHash": "0x5686f3a5eec1b070d0113c588f8f4a560d57ad96b8045cedb5c08bbadaa0273e", "firstIndex": 6375340033},
+{"blockNumber": 8858036, "blockHash": "0x4f9b5d9fac9c6f6e2224f613cda12e8ab95d636774ce87489dce8a9f805ee2e5", "firstIndex": 6442450330},
+{"blockNumber": 8884811, "blockHash": "0x9cf74f978872683802c065e72b5a5326fdad95f19733c34d927b575cd85fd0bd", "firstIndex": 6509559380}
]
diff --git a/core/filtermaps/filtermaps.go b/core/filtermaps/filtermaps.go
deleted file mode 100644
index fede54df57..0000000000
--- a/core/filtermaps/filtermaps.go
+++ /dev/null
@@ -1,889 +0,0 @@
-// Copyright 2024 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see .
-
-package filtermaps
-
-import (
- "errors"
- "fmt"
- "os"
- "slices"
- "sync"
- "time"
-
- "github.com/ethereum/go-ethereum/common"
- "github.com/ethereum/go-ethereum/common/lru"
- "github.com/ethereum/go-ethereum/core/rawdb"
- "github.com/ethereum/go-ethereum/core/types"
- "github.com/ethereum/go-ethereum/ethdb"
- "github.com/ethereum/go-ethereum/log"
- "github.com/ethereum/go-ethereum/metrics"
-)
-
-var (
- mapCountGauge = metrics.NewRegisteredGauge("filtermaps/maps/count", nil) // actual number of rendered maps
- mapLogValueMeter = metrics.NewRegisteredMeter("filtermaps/maps/logvalues", nil) // number of log values processed
- mapBlockMeter = metrics.NewRegisteredMeter("filtermaps/maps/blocks", nil) // number of block delimiters processed
- mapRenderTimer = metrics.NewRegisteredTimer("filtermaps/maps/rendertime", nil) // time elapsed while rendering a single map
- mapWriteTimer = metrics.NewRegisteredTimer("filtermaps/maps/writetime", nil) // time elapsed while writing a batch of finished maps to db
- matchRequestTimer = metrics.NewRegisteredTimer("filtermaps/match/requesttime", nil) // processing time a matching request in a single epoch
- matchEpochTimer = metrics.NewRegisteredTimer("filtermaps/match/epochtime", nil) // total processing time a matching request
- matchBaseRowAccessMeter = metrics.NewRegisteredMeter("filtermaps/match/baserowaccess", nil) // number of accessed rows on layer 0
- matchBaseRowSizeMeter = metrics.NewRegisteredMeter("filtermaps/match/baserowsize", nil) // size of accessed rows on layer 0
- matchExtRowAccessMeter = metrics.NewRegisteredMeter("filtermaps/match/extrowaccess", nil) // number of accessed rows on higher layers
- matchExtRowSizeMeter = metrics.NewRegisteredMeter("filtermaps/match/extrowsize", nil) // size of accessed rows on higher layers
- matchLogLookup = metrics.NewRegisteredMeter("filtermaps/match/loglookup", nil) // number of log lookups based on potential matches
- matchAllMeter = metrics.NewRegisteredMeter("filtermaps/match/matchall", nil) // number of requests returned with ErrMatchAll
-)
-
-const (
- databaseVersion = 2 // reindexed if database version does not match
- cachedLastBlocks = 1000 // last block of map pointers
- cachedLvPointers = 1000 // first log value pointer of block pointers
- cachedFilterMaps = 3 // complete filter maps (cached by map renderer)
- cachedRenderSnapshots = 8 // saved map renderer data at block boundaries
-)
-
-// FilterMaps is the in-memory representation of the log index structure that is
-// responsible for building and updating the index according to the canonical
-// chain.
-//
-// Note that FilterMaps implements the same data structure as proposed in EIP-7745
-// without the tree hashing and consensus changes:
-// https://eips.ethereum.org/EIPS/eip-7745
-type FilterMaps struct {
- // If disabled is set, log indexing is fully disabled.
- // This is configured by the --history.logs.disable Geth flag.
- // We chose to implement disabling this way because it requires less special
- // case logic in eth/filters.
- disabled bool
- disabledCh chan struct{} // closed by indexer if disabled
-
- closeCh chan struct{}
- closeWg sync.WaitGroup
- history uint64
- hashScheme bool // use hashdb-safe delete range method
- exportFileName string
- Params
-
- db ethdb.KeyValueStore
-
- // fields written by the indexer and read by matcher backend. Indexer can
- // read them without a lock and write them under indexLock write lock.
- // Matcher backend can read them under indexLock read lock.
- indexLock sync.RWMutex
- indexedRange filterMapsRange
- indexedView *ChainView // always consistent with the log index
- hasTempRange bool
-
- // cleanedEpochsBefore indicates that all unindexed data before this point
- // has been cleaned.
- //
- // This field is only accessed and modified within tryUnindexTail, so no
- // explicit locking is required.
- cleanedEpochsBefore uint32
-
- // also accessed by indexer and matcher backend but no locking needed.
- filterMapCache *lru.Cache[uint32, filterMap]
- lastBlockCache *lru.Cache[uint32, lastBlockOfMap]
- lvPointerCache *lru.Cache[uint64, uint64]
-
- // the matchers set and the fields of FilterMapsMatcherBackend instances are
- // read and written both by exported functions and the indexer.
- // Note that if both indexLock and matchersLock needs to be locked then
- // indexLock should be locked first.
- matchersLock sync.Mutex
- matchers map[*FilterMapsMatcherBackend]struct{}
-
- // fields only accessed by the indexer (no mutex required).
- renderSnapshots *lru.Cache[uint64, *renderedMap]
- startedHeadIndex, startedTailIndex, startedTailUnindex bool
- startedHeadIndexAt, startedTailIndexAt, startedTailUnindexAt time.Time
- loggedHeadIndex, loggedTailIndex bool
- lastLogHeadIndex, lastLogTailIndex time.Time
- ptrHeadIndex, ptrTailIndex, ptrTailUnindexBlock uint64
- ptrTailUnindexMap uint32
-
- targetView *ChainView
- matcherSyncRequests []*FilterMapsMatcherBackend
- historyCutoff uint64
- finalBlock, lastFinal uint64
- lastFinalEpoch uint32
- stop bool
- targetCh chan targetUpdate
- blockProcessingCh chan bool
- blockProcessing bool
- matcherSyncCh chan *FilterMapsMatcherBackend
- waitIdleCh chan chan bool
- tailRenderer *mapRenderer
-
- // test hooks
- testDisableSnapshots, testSnapshotUsed bool
- testProcessEventsHook func()
-}
-
-// filterMap is a full or partial in-memory representation of a filter map where
-// rows are allowed to have a nil value meaning the row is not stored in the
-// structure. Note that therefore a known empty row should be represented with
-// a zero-length slice.
-// It can be used as a memory cache or an overlay while preparing a batch of
-// changes to the structure. In either case a nil value should be interpreted
-// as transparent (uncached/unchanged).
-type filterMap []FilterRow
-
-// fastCopy returns a copy of the given filter map. Note that the row slices are
-// copied but their contents are not. This permits appending to the rows further
-// (which happens during map rendering) without affecting the validity of
-// copies made for snapshots during rendering.
-// Appending to the rows of both the original map and the fast copy, or two fast
-// copies of the same map would result in data corruption, therefore a fast copy
-// should always be used in a read only way.
-func (fm filterMap) fastCopy() filterMap {
- return slices.Clone(fm)
-}
-
-// fullCopy returns a copy of the given filter map, also making a copy of each
-// individual filter row, ensuring that a modification to either one will never
-// affect the other.
-func (fm filterMap) fullCopy() filterMap {
- c := make(filterMap, len(fm))
- for i, row := range fm {
- c[i] = slices.Clone(row)
- }
- return c
-}
-
-// FilterRow encodes a single row of a filter map as a list of column indices.
-// Note that the values are always stored in the same order as they were added
-// and if the same column index is added twice, it is also stored twice.
-// Order of column indices and potential duplications do not matter when searching
-// for a value but leaving the original order makes reverting to a previous state
-// simpler.
-type FilterRow []uint32
-
-// Equal returns true if the given filter rows are equivalent.
-func (a FilterRow) Equal(b FilterRow) bool {
- return slices.Equal(a, b)
-}
-
-// filterMapsRange describes the rendered range of filter maps and the range
-// of fully rendered blocks.
-type filterMapsRange struct {
- initialized bool
- headIndexed bool
- headDelimiter uint64 // zero if headIndexed is false
-
- // if initialized then all maps are rendered in the maps range
- maps common.Range[uint32]
-
- // if tailPartialEpoch > 0 then maps between firstRenderedMap-mapsPerEpoch and
- // firstRenderedMap-mapsPerEpoch+tailPartialEpoch-1 are rendered
- tailPartialEpoch uint32
-
- // if initialized then all log values in the blocks range are fully
- // rendered
- // blockLvPointers are available in the blocks range
- blocks common.Range[uint64]
-}
-
-// hasIndexedBlocks returns true if the range has at least one fully indexed block.
-func (fmr *filterMapsRange) hasIndexedBlocks() bool {
- return fmr.initialized && !fmr.blocks.IsEmpty() && !fmr.maps.IsEmpty()
-}
-
-// lastBlockOfMap is used for caching the (number, id) pairs belonging to the
-// last block of each map.
-type lastBlockOfMap struct {
- number uint64
- id common.Hash
-}
-
-// Config contains the configuration options for NewFilterMaps.
-type Config struct {
- History uint64 // number of historical blocks to index
- Disabled bool // disables indexing completely
-
- // This option enables the checkpoint JSON file generator.
- // If set, the given file will be updated with checkpoint information.
- ExportFileName string
-
- // expect trie nodes of hash based state scheme in the filtermaps key range;
- // use safe iterator based implementation of DeleteRange that skips them
- HashScheme bool
-}
-
-// NewFilterMaps creates a new FilterMaps and starts the indexer.
-func NewFilterMaps(db ethdb.KeyValueStore, initView *ChainView, historyCutoff, finalBlock uint64, params Params, config Config) (*FilterMaps, error) {
- rs, initialized, err := rawdb.ReadFilterMapsRange(db)
- if err != nil || (initialized && rs.Version != databaseVersion) {
- rs, initialized = rawdb.FilterMapsRange{}, false
- log.Warn("Invalid log index database version; resetting log index")
- }
- if err := params.sanitize(); err != nil {
- return nil, err
- }
- f := &FilterMaps{
- db: db,
- closeCh: make(chan struct{}),
- waitIdleCh: make(chan chan bool),
- targetCh: make(chan targetUpdate, 1),
- blockProcessingCh: make(chan bool, 1),
- history: config.History,
- disabled: config.Disabled,
- hashScheme: config.HashScheme,
- disabledCh: make(chan struct{}),
- exportFileName: config.ExportFileName,
- Params: params,
- targetView: initView,
- indexedView: initView,
- indexedRange: filterMapsRange{
- initialized: initialized,
- headIndexed: rs.HeadIndexed,
- headDelimiter: rs.HeadDelimiter,
- blocks: common.NewRange(rs.BlocksFirst, rs.BlocksAfterLast-rs.BlocksFirst),
- maps: common.NewRange(rs.MapsFirst, rs.MapsAfterLast-rs.MapsFirst),
- tailPartialEpoch: rs.TailPartialEpoch,
- },
- // deleting last unindexed epoch might have been interrupted by shutdown
- cleanedEpochsBefore: max(rs.MapsFirst>>params.logMapsPerEpoch, 1) - 1,
- historyCutoff: historyCutoff,
- finalBlock: finalBlock,
- matcherSyncCh: make(chan *FilterMapsMatcherBackend),
- matchers: make(map[*FilterMapsMatcherBackend]struct{}),
- filterMapCache: lru.NewCache[uint32, filterMap](cachedFilterMaps),
- lastBlockCache: lru.NewCache[uint32, lastBlockOfMap](cachedLastBlocks),
- lvPointerCache: lru.NewCache[uint64, uint64](cachedLvPointers),
- renderSnapshots: lru.NewCache[uint64, *renderedMap](cachedRenderSnapshots),
- }
- f.checkRevertRange() // revert maps that are inconsistent with the current chain view
-
- if f.indexedRange.hasIndexedBlocks() {
- log.Info("Initialized log indexer",
- "firstblock", f.indexedRange.blocks.First(), "lastblock", f.indexedRange.blocks.Last(),
- "firstmap", f.indexedRange.maps.First(), "lastmap", f.indexedRange.maps.Last(),
- "headindexed", f.indexedRange.headIndexed)
- }
- return f, nil
-}
-
-// Start starts the indexer.
-func (f *FilterMaps) Start() {
- if !f.testDisableSnapshots && f.indexedRange.hasIndexedBlocks() && f.indexedRange.headIndexed {
- // previous target head rendered; load last map as snapshot
- if err := f.loadHeadSnapshot(); err != nil {
- log.Error("Could not load head filter map snapshot", "error", err)
- }
- }
- f.closeWg.Add(2)
- go f.removeBloomBits()
- go f.indexerLoop()
-}
-
-// Stop ensures that the indexer is fully stopped before returning.
-func (f *FilterMaps) Stop() {
- close(f.closeCh)
- f.closeWg.Wait()
-}
-
-// checkRevertRange checks whether the existing index is consistent with the
-// current indexed view and reverts inconsistent maps if necessary.
-func (f *FilterMaps) checkRevertRange() {
- if f.indexedRange.maps.Count() == 0 {
- return
- }
- lastMap := f.indexedRange.maps.Last()
- lastBlockNumber, lastBlockId, err := f.getLastBlockOfMap(lastMap)
- if err != nil {
- log.Error("Error initializing log index database; resetting log index", "error", err)
- f.reset()
- return
- }
- for lastBlockNumber > f.indexedView.HeadNumber() || f.indexedView.BlockId(lastBlockNumber) != lastBlockId {
- // revert last map
- if f.indexedRange.maps.Count() == 1 {
- f.reset() // reset database if no rendered maps remained
- return
- }
- lastMap--
- newRange := f.indexedRange
- newRange.maps.SetLast(lastMap)
- lastBlockNumber, lastBlockId, err = f.getLastBlockOfMap(lastMap)
- if err != nil {
- log.Error("Error initializing log index database; resetting log index", "error", err)
- f.reset()
- return
- }
- newRange.blocks.SetAfterLast(lastBlockNumber) // lastBlockNumber is probably partially indexed
- newRange.headIndexed = false
- newRange.headDelimiter = 0
- // only shorten range and leave map data; next head render will overwrite it
- f.setRange(f.db, f.indexedView, newRange, false)
- }
-}
-
-// reset un-initializes the FilterMaps structure and removes all related data from
-// the database.
-// Note that in case of leveldb database the fallback implementation of DeleteRange
-// might take a long time to finish and deleting the entire database may be
-// interrupted by a shutdown. Deleting the filterMapsRange entry first does
-// guarantee though that the next init() will not return successfully until the
-// entire database has been cleaned.
-func (f *FilterMaps) reset() {
- f.indexLock.Lock()
- f.indexedRange = filterMapsRange{}
- f.indexedView = nil
- f.filterMapCache.Purge()
- f.renderSnapshots.Purge()
- f.lastBlockCache.Purge()
- f.lvPointerCache.Purge()
- f.indexLock.Unlock()
- // deleting the range first ensures that resetDb will be called again at next
- // startup and any leftover data will be removed even if it cannot finish now.
- rawdb.DeleteFilterMapsRange(f.db)
- f.safeDeleteWithLogs(rawdb.DeleteFilterMapsDb, "Resetting log index database", f.isShuttingDown)
-}
-
-// isShuttingDown returns true if FilterMaps is shutting down.
-func (f *FilterMaps) isShuttingDown() bool {
- select {
- case <-f.closeCh:
- return true
- default:
- return false
- }
-}
-
-// init initializes an empty log index according to the current targetView.
-func (f *FilterMaps) init() error {
- // ensure that there is no remaining data in the filter maps key range
- if err := f.safeDeleteWithLogs(rawdb.DeleteFilterMapsDb, "Resetting log index database", f.isShuttingDown); err != nil {
- return err
- }
-
- f.indexLock.Lock()
- defer f.indexLock.Unlock()
-
- var bestIdx, bestLen int
- for idx, checkpointList := range checkpoints {
- // binary search for the last matching epoch head
- min, max := 0, len(checkpointList)
- for min < max {
- mid := (min + max + 1) / 2
- cp := checkpointList[mid-1]
- if cp.BlockNumber <= f.targetView.HeadNumber() && f.targetView.BlockId(cp.BlockNumber) == cp.BlockId {
- min = mid
- } else {
- max = mid - 1
- }
- }
- if max > bestLen {
- bestIdx, bestLen = idx, max
- }
- }
- var initBlockNumber uint64
- if bestLen > 0 {
- initBlockNumber = checkpoints[bestIdx][bestLen-1].BlockNumber
- }
- if initBlockNumber < f.historyCutoff {
- return errors.New("cannot start indexing before history cutoff point")
- }
- batch := f.db.NewBatch()
- for epoch := range bestLen {
- cp := checkpoints[bestIdx][epoch]
- f.storeLastBlockOfMap(batch, f.lastEpochMap(uint32(epoch)), cp.BlockNumber, cp.BlockId)
- f.storeBlockLvPointer(batch, cp.BlockNumber, cp.FirstIndex)
- }
- fmr := filterMapsRange{
- initialized: true,
- }
- if bestLen > 0 {
- cp := checkpoints[bestIdx][bestLen-1]
- fmr.blocks = common.NewRange(cp.BlockNumber+1, 0)
- fmr.maps = common.NewRange(f.firstEpochMap(uint32(bestLen)), 0)
- }
- f.setRange(batch, f.targetView, fmr, false)
- return batch.Write()
-}
-
-// removeBloomBits removes old bloom bits data from the database.
-func (f *FilterMaps) removeBloomBits() {
- f.safeDeleteWithLogs(rawdb.DeleteBloomBitsDb, "Removing old bloom bits database", f.isShuttingDown)
- f.closeWg.Done()
-}
-
-// safeDeleteWithLogs is a wrapper for a function that performs a safe range
-// delete operation using rawdb.SafeDeleteRange. It emits log messages if the
-// process takes long enough to call the stop callback.
-func (f *FilterMaps) safeDeleteWithLogs(deleteFn func(db ethdb.KeyValueStore, hashScheme bool, stopCb func(bool) bool) error, action string, stopCb func() bool) error {
- var (
- start = time.Now()
- logPrinted bool
- lastLogPrinted = start
- )
- switch err := deleteFn(f.db, f.hashScheme, func(deleted bool) bool {
- if deleted && !logPrinted || time.Since(lastLogPrinted) > time.Second*10 {
- log.Info(action+" in progress...", "elapsed", common.PrettyDuration(time.Since(start)))
- logPrinted, lastLogPrinted = true, time.Now()
- }
- return stopCb()
- }); {
- case err == nil:
- if logPrinted {
- log.Info(action+" finished", "elapsed", common.PrettyDuration(time.Since(start)))
- }
- return nil
- case errors.Is(err, rawdb.ErrDeleteRangeInterrupted):
- log.Warn(action+" interrupted", "elapsed", common.PrettyDuration(time.Since(start)))
- return err
- default:
- log.Error(action+" failed", "error", err)
- return err
- }
-}
-
-// setRange updates the indexed chain view and covered range and also adds the
-// changes to the given batch.
-//
-// Note that this function assumes that the index write lock is being held.
-func (f *FilterMaps) setRange(batch ethdb.KeyValueWriter, newView *ChainView, newRange filterMapsRange, isTempRange bool) {
- f.indexedView = newView
- f.indexedRange = newRange
- f.hasTempRange = isTempRange
- f.updateMatchersValidRange()
- if newRange.initialized {
- rs := rawdb.FilterMapsRange{
- Version: databaseVersion,
- HeadIndexed: newRange.headIndexed,
- HeadDelimiter: newRange.headDelimiter,
- BlocksFirst: newRange.blocks.First(),
- BlocksAfterLast: newRange.blocks.AfterLast(),
- MapsFirst: newRange.maps.First(),
- MapsAfterLast: newRange.maps.AfterLast(),
- TailPartialEpoch: newRange.tailPartialEpoch,
- }
- rawdb.WriteFilterMapsRange(batch, rs)
- if !isTempRange {
- mapCountGauge.Update(int64(newRange.maps.Count() + newRange.tailPartialEpoch))
- }
- } else {
- rawdb.DeleteFilterMapsRange(batch)
- mapCountGauge.Update(0)
- }
-}
-
-// getLogByLvIndex returns the log at the given log value index. If the index does
-// not point to the first log value entry of a log then no log and no error are
-// returned as this can happen when the log value index was a false positive.
-// Note that this function assumes that the log index structure is consistent
-// with the canonical chain at the point where the given log value index points.
-// If this is not the case then an invalid result or an error may be returned.
-//
-// Note that this function assumes that the indexer read lock is being held when
-// called from outside the indexerLoop goroutine.
-func (f *FilterMaps) getLogByLvIndex(lvIndex uint64) (*types.Log, error) {
- mapIndex := uint32(lvIndex >> f.logValuesPerMap)
- if !f.indexedRange.maps.Includes(mapIndex) {
- return nil, nil
- }
- // find possible block range based on map to block pointers
- lastBlockNumber, _, err := f.getLastBlockOfMap(mapIndex)
- if err != nil {
- return nil, fmt.Errorf("failed to retrieve last block of map %d containing searched log value index %d: %v", mapIndex, lvIndex, err)
- }
- var firstBlockNumber uint64
- if mapIndex > 0 {
- firstBlockNumber, _, err = f.getLastBlockOfMap(mapIndex - 1)
- if err != nil {
- return nil, fmt.Errorf("failed to retrieve last block of map %d before searched log value index %d: %v", mapIndex, lvIndex, err)
- }
- }
- if firstBlockNumber < f.indexedRange.blocks.First() {
- firstBlockNumber = f.indexedRange.blocks.First()
- }
- // find block with binary search based on block to log value index pointers
- for firstBlockNumber < lastBlockNumber {
- midBlockNumber := (firstBlockNumber + lastBlockNumber + 1) / 2
- midLvPointer, err := f.getBlockLvPointer(midBlockNumber)
- if err != nil {
- return nil, fmt.Errorf("failed to retrieve log value pointer of block %d while binary searching log value index %d: %v", midBlockNumber, lvIndex, err)
- }
- if lvIndex < midLvPointer {
- lastBlockNumber = midBlockNumber - 1
- } else {
- firstBlockNumber = midBlockNumber
- }
- }
- // get block receipts
- receipts := f.indexedView.Receipts(firstBlockNumber)
- if receipts == nil {
- return nil, fmt.Errorf("failed to retrieve receipts for block %d containing searched log value index %d: %v", firstBlockNumber, lvIndex, err)
- }
- lvPointer, err := f.getBlockLvPointer(firstBlockNumber)
- if err != nil {
- return nil, fmt.Errorf("failed to retrieve log value pointer of block %d containing searched log value index %d: %v", firstBlockNumber, lvIndex, err)
- }
- // iterate through receipts to find the exact log starting at lvIndex
- for _, receipt := range receipts {
- for _, log := range receipt.Logs {
- l := uint64(len(log.Topics) + 1)
- r := f.valuesPerMap - lvPointer%f.valuesPerMap
- if l > r {
- lvPointer += r // skip to map boundary
- }
- if lvPointer > lvIndex {
- // lvIndex does not point to the first log value (address value)
- // generated by a log as true matches should always do, so it
- // is considered a false positive (no log and no error returned).
- return nil, nil
- }
- if lvPointer == lvIndex {
- return log, nil // potential match
- }
- lvPointer += l
- }
- }
- return nil, nil
-}
-
-// getFilterMap fetches an entire filter map from the database.
-func (f *FilterMaps) getFilterMap(mapIndex uint32) (filterMap, error) {
- if fm, ok := f.filterMapCache.Get(mapIndex); ok {
- return fm, nil
- }
- fm := make(filterMap, f.mapHeight)
- for rowIndex := range fm {
- rows, err := f.getFilterMapRows([]uint32{mapIndex}, uint32(rowIndex), false)
- if err != nil {
- return nil, fmt.Errorf("failed to load filter map %d from database: %v", mapIndex, err)
- }
- fm[rowIndex] = rows[0]
- }
- f.filterMapCache.Add(mapIndex, fm)
- return fm, nil
-}
-
-// getFilterMapRows fetches a set of filter map rows at the corresponding map
-// indices and a shared row index. If baseLayerOnly is true then only the first
-// baseRowLength entries are returned.
-func (f *FilterMaps) getFilterMapRows(mapIndices []uint32, rowIndex uint32, baseLayerOnly bool) ([]FilterRow, error) {
- rows := make([]FilterRow, len(mapIndices))
- var ptr int
- for len(mapIndices) > ptr {
- var (
- groupIndex = f.mapGroupIndex(mapIndices[ptr])
- groupLength = 1
- )
- for ptr+groupLength < len(mapIndices) && f.mapGroupIndex(mapIndices[ptr+groupLength]) == groupIndex {
- groupLength++
- }
- if err := f.getFilterMapRowsOfGroup(rows[ptr:ptr+groupLength], mapIndices[ptr:ptr+groupLength], rowIndex, baseLayerOnly); err != nil {
- return nil, err
- }
- ptr += groupLength
- }
- return rows, nil
-}
-
-// getFilterMapRowsOfGroup fetches a set of filter map rows at map indices
-// belonging to the same base row group.
-func (f *FilterMaps) getFilterMapRowsOfGroup(target []FilterRow, mapIndices []uint32, rowIndex uint32, baseLayerOnly bool) error {
- var (
- groupIndex = f.mapGroupIndex(mapIndices[0])
- mapRowIndex = f.mapRowIndex(groupIndex, rowIndex)
- )
- baseRows, err := rawdb.ReadFilterMapBaseRows(f.db, mapRowIndex, f.baseRowGroupSize, f.logMapWidth)
- if err != nil {
- return fmt.Errorf("failed to retrieve base row group %d of row %d: %v", groupIndex, rowIndex, err)
- }
- for i, mapIndex := range mapIndices {
- if f.mapGroupIndex(mapIndex) != groupIndex {
- return fmt.Errorf("maps are not in the same base row group, index: %d, group: %d", mapIndex, groupIndex)
- }
- row := baseRows[f.mapGroupOffset(mapIndex)]
- if !baseLayerOnly {
- extRow, err := rawdb.ReadFilterMapExtRow(f.db, f.mapRowIndex(mapIndex, rowIndex), f.logMapWidth)
- if err != nil {
- return fmt.Errorf("failed to retrieve filter map %d extended row %d: %v", mapIndex, rowIndex, err)
- }
- row = append(row, extRow...)
- }
- target[i] = row
- }
- return nil
-}
-
-// storeFilterMapRows stores a set of filter map rows at the corresponding map
-// indices and a shared row index.
-func (f *FilterMaps) storeFilterMapRows(batch ethdb.Batch, mapIndices []uint32, rowIndex uint32, rows []FilterRow) error {
- for len(mapIndices) > 0 {
- var (
- pos = 1
- groupIndex = f.mapGroupIndex(mapIndices[0])
- )
- for pos < len(mapIndices) && f.mapGroupIndex(mapIndices[pos]) == groupIndex {
- pos++
- }
- if err := f.storeFilterMapRowsOfGroup(batch, mapIndices[:pos], rowIndex, rows[:pos]); err != nil {
- return err
- }
- mapIndices, rows = mapIndices[pos:], rows[pos:]
- }
- return nil
-}
-
-// storeFilterMapRowsOfGroup stores a set of filter map rows at map indices
-// belonging to the same base row group.
-func (f *FilterMaps) storeFilterMapRowsOfGroup(batch ethdb.Batch, mapIndices []uint32, rowIndex uint32, rows []FilterRow) error {
- var (
- baseRows [][]uint32
- groupIndex = f.mapGroupIndex(mapIndices[0])
- mapRowIndex = f.mapRowIndex(groupIndex, rowIndex)
- )
- if uint32(len(mapIndices)) != f.baseRowGroupSize { // skip base rows read if all rows are replaced
- var err error
- baseRows, err = rawdb.ReadFilterMapBaseRows(f.db, mapRowIndex, f.baseRowGroupSize, f.logMapWidth)
- if err != nil {
- return fmt.Errorf("failed to retrieve filter map %d base rows %d for modification: %v", groupIndex, rowIndex, err)
- }
- } else {
- baseRows = make([][]uint32, f.baseRowGroupSize)
- }
- for i, mapIndex := range mapIndices {
- if f.mapGroupIndex(mapIndex) != groupIndex {
- return fmt.Errorf("maps are not in the same base row group, index: %d, group: %d", mapIndex, groupIndex)
- }
- baseRow := []uint32(rows[i])
- var extRow FilterRow
- if uint32(len(rows[i])) > f.baseRowLength {
- extRow = baseRow[f.baseRowLength:]
- baseRow = baseRow[:f.baseRowLength]
- }
- baseRows[f.mapGroupOffset(mapIndex)] = baseRow
- rawdb.WriteFilterMapExtRow(batch, f.mapRowIndex(mapIndex, rowIndex), extRow, f.logMapWidth)
- }
- rawdb.WriteFilterMapBaseRows(batch, mapRowIndex, baseRows, f.logMapWidth)
- return nil
-}
-
-// mapRowIndex calculates the unified storage index where the given row of the
-// given map is stored. Note that this indexing scheme is the same as the one
-// proposed in EIP-7745 for tree-hashing the filter map structure and for the
-// same data proximity reasons it is also suitable for database representation.
-// See also:
-// https://eips.ethereum.org/EIPS/eip-7745#hash-tree-structure
-func (f *FilterMaps) mapRowIndex(mapIndex, rowIndex uint32) uint64 {
- epochIndex, mapSubIndex := mapIndex>>f.logMapsPerEpoch, mapIndex&(f.mapsPerEpoch-1)
- return (uint64(epochIndex)< 0 {
- firstBlock, _, err = f.getLastBlockOfMap(firstMap - 1)
- if err != nil {
- return false, fmt.Errorf("failed to retrieve last block before deleted epoch %d: %v", epoch, err)
- }
- firstBlock++
- }
- // update rendered range if necessary
- var (
- fmr = f.indexedRange
- firstEpoch = f.mapEpoch(f.indexedRange.maps.First())
- afterLastEpoch = f.mapEpoch(f.indexedRange.maps.AfterLast() + f.mapsPerEpoch - 1)
- )
- if f.indexedRange.tailPartialEpoch != 0 && firstEpoch > 0 {
- firstEpoch--
- }
- switch {
- case epoch < firstEpoch:
- // cleanup of already unindexed epoch; range not affected
- case epoch == firstEpoch && epoch+1 < afterLastEpoch:
- // first fully or partially rendered epoch and there is at least one
- // rendered map in the next epoch; remove from indexed range
- fmr.tailPartialEpoch = 0
- fmr.maps.SetFirst(f.firstEpochMap(epoch + 1))
- fmr.blocks.SetFirst(lastBlock + 1)
- f.setRange(f.db, f.indexedView, fmr, false)
- default:
- // cannot be cleaned or unindexed; return with error
- return false, errors.New("invalid tail epoch number")
- }
- // remove index data
- deleteFn := func(db ethdb.KeyValueStore, hashScheme bool, stopCb func(bool) bool) error {
- first := f.mapRowIndex(firstMap, 0)
- count := f.mapRowIndex(firstMap+f.mapsPerEpoch, 0) - first
- if err := rawdb.DeleteFilterMapRows(f.db, common.NewRange(first, count), hashScheme, stopCb); err != nil {
- return err
- }
- for mapIndex := firstMap; mapIndex < firstMap+f.mapsPerEpoch; mapIndex++ {
- f.filterMapCache.Remove(mapIndex)
- }
- delMapRange := common.NewRange(firstMap, f.mapsPerEpoch-1) // keep last entry
- if err := rawdb.DeleteFilterMapLastBlocks(f.db, delMapRange, hashScheme, stopCb); err != nil {
- return err
- }
- for mapIndex := firstMap; mapIndex < firstMap+f.mapsPerEpoch-1; mapIndex++ {
- f.lastBlockCache.Remove(mapIndex)
- }
- delBlockRange := common.NewRange(firstBlock, lastBlock-firstBlock) // keep last entry
- if err := rawdb.DeleteBlockLvPointers(f.db, delBlockRange, hashScheme, stopCb); err != nil {
- return err
- }
- for blockNumber := firstBlock; blockNumber < lastBlock; blockNumber++ {
- f.lvPointerCache.Remove(blockNumber)
- }
- return nil
- }
- action := fmt.Sprintf("Deleting tail epoch #%d", epoch)
- stopFn := func() bool {
- f.processEvents()
- return f.stop || !f.targetHeadIndexed()
- }
- if err := f.safeDeleteWithLogs(deleteFn, action, stopFn); err == nil {
- // everything removed; mark as cleaned and report success
- if f.cleanedEpochsBefore == epoch {
- f.cleanedEpochsBefore = epoch + 1
- }
- return true, nil
- } else {
- // more data left in epoch range; mark as dirty and report unfinished
- if f.cleanedEpochsBefore > epoch {
- f.cleanedEpochsBefore = epoch
- }
- if errors.Is(err, rawdb.ErrDeleteRangeInterrupted) {
- return false, nil
- }
- return false, err
- }
-}
-
-// exportCheckpoints exports epoch checkpoints in the format used by checkpoints.go.
-//
-// Note: acquiring the indexLock read lock is unnecessary here, as this function
-// is always called within the indexLoop.
-func (f *FilterMaps) exportCheckpoints() {
- finalLvPtr, err := f.getBlockLvPointer(f.finalBlock + 1)
- if err != nil {
- log.Error("Error fetching log value pointer of finalized block", "block", f.finalBlock, "error", err)
- return
- }
- epochCount := uint32(finalLvPtr >> (f.logValuesPerMap + f.logMapsPerEpoch))
- if epochCount == f.lastFinalEpoch {
- return
- }
- w, err := os.Create(f.exportFileName)
- if err != nil {
- log.Error("Error creating checkpoint export file", "name", f.exportFileName, "error", err)
- return
- }
- defer w.Close()
-
- log.Info("Exporting log index checkpoints", "epochs", epochCount, "file", f.exportFileName)
- w.WriteString("[\n")
- comma := ","
- for epoch := uint32(0); epoch < epochCount; epoch++ {
- lastBlock, lastBlockId, err := f.getLastBlockOfMap(f.lastEpochMap(epoch))
- if err != nil {
- log.Error("Error fetching last block of epoch", "epoch", epoch, "error", err)
- return
- }
- lvPtr, err := f.getBlockLvPointer(lastBlock)
- if err != nil {
- log.Error("Error fetching log value pointer of last block", "block", lastBlock, "error", err)
- return
- }
- if epoch == epochCount-1 {
- comma = ""
- }
- w.WriteString(fmt.Sprintf("{\"blockNumber\": %d, \"blockId\": \"0x%064x\", \"firstIndex\": %d}%s\n", lastBlock, lastBlockId, lvPtr, comma))
- }
- w.WriteString("]\n")
- f.lastFinalEpoch = epochCount
-}
diff --git a/core/filtermaps/index_view.go b/core/filtermaps/index_view.go
new file mode 100644
index 0000000000..92999028e5
--- /dev/null
+++ b/core/filtermaps/index_view.go
@@ -0,0 +1,274 @@
+// Copyright 2025 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package filtermaps
+
+import (
+ "errors"
+ "sync/atomic"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/core/types"
+)
+
+var (
+ ErrInvalidView = errors.New("index view already invalidated")
+)
+
+type IndexView struct {
+ // if invalid <= 0 then the view is considered invalid and will return error
+ // on any read operation (storage maps before firstMemoryMap might have changed
+ // since its creation)
+ refCount, invalid int32
+ storage *mapStorage
+
+ tailEpoch uint32 //TODO apply to read functions
+ blockRange common.Range[uint64] //TODO apply to read functions
+ headBlockHash common.Hash
+ headLvPointer uint64 // points after head block delimiter
+
+ firstMemoryMap uint32
+ firstMemoryBlock uint64
+ finishedMaps []*finishedMap
+ headMapIndex uint32
+ headMap *memoryMap
+}
+
+func (iv *IndexView) Release() {
+ iv.addRefCount(-1)
+}
+
+func (iv *IndexView) addRefCount(add int32) bool {
+ return atomic.AddInt32(&iv.refCount, add) <= 0
+}
+
+func (iv *IndexView) checkReleased() bool {
+ return atomic.LoadInt32(&iv.refCount) <= 0
+}
+
+func (iv *IndexView) invalidate() {
+ atomic.StoreInt32(&iv.invalid, 1)
+}
+
+func (iv *IndexView) checkInvalid() bool {
+ return atomic.LoadInt32(&iv.invalid) != 0
+}
+
+func (iv *IndexView) GetParams() *Params {
+ return iv.storage.params
+}
+
+func (iv *IndexView) BlockRange() common.Range[uint64] {
+ return iv.blockRange
+}
+
+// getBlockLvPointer returns the starting log value index where the log values
+// generated by the given block are located.
+// Note that the function returns a valid result (the next free log value index)
+// for headNumber + 1.
+func (iv *IndexView) GetBlockLvPointer(blockNumber uint64) (uint64, error) {
+ if iv.checkInvalid() {
+ return 0, ErrInvalidView
+ }
+
+ if blockNumber < iv.firstMemoryBlock {
+ lvPtr, err := iv.storage.getBlockLvPointer(blockNumber)
+ if iv.checkInvalid() {
+ return 0, ErrInvalidView
+ }
+ return lvPtr, err
+ }
+ if blockNumber == iv.blockRange.AfterLast() {
+ return iv.headLvPointer, nil
+ }
+ if blockNumber > iv.blockRange.AfterLast() {
+ return 0, ErrOutOfRange
+ }
+ for _, fm := range iv.finishedMaps {
+ if blockNumber >= fm.firstBlock() && blockNumber <= fm.lastBlock.number {
+ return fm.blockPtrs[blockNumber-fm.firstBlock()], nil
+ }
+ }
+ if blockNumber >= iv.headMap.firstBlock() && blockNumber <= iv.headMap.lastBlock.number {
+ return iv.headMap.blockPtrs[blockNumber-iv.headMap.firstBlock()], nil
+ }
+ panic("IndexView.GetBlockLvPointer: gap in blockLvPtrs")
+}
+
+// GetLastBlockOfMap returns the number and hash of the block that generated the
+// last log value entry of the given map.
+func (iv *IndexView) GetLastBlockOfMap(mapIndex uint32) (uint64, common.Hash, error) {
+ if iv.checkInvalid() {
+ return 0, common.Hash{}, ErrInvalidView
+ }
+
+ if mapIndex < iv.firstMemoryMap {
+ lastNumber, lastHash, err := iv.storage.getLastBlockOfMap(mapIndex)
+ if iv.checkInvalid() {
+ return 0, common.Hash{}, ErrInvalidView
+ }
+ return lastNumber, lastHash, err
+ }
+ if mapIndex > iv.headMapIndex {
+ return 0, common.Hash{}, ErrOutOfRange
+ }
+ if mapIndex == iv.headMapIndex {
+ return iv.headMap.lastBlock.number, iv.headMap.lastBlock.hash, nil
+ }
+ fm := iv.finishedMaps[mapIndex-iv.firstMemoryMap]
+ return fm.lastBlock.number, fm.lastBlock.hash, nil
+}
+
+// GetFilterMapRows returns a batch of filter maps rows from the same row index,
+// each truncated to the length limit of the specified mapping layer.
+// The function assumes that the map indices are in strictly ascending order.
+func (iv *IndexView) GetFilterMapRows(mapIndices []uint32, rowIndex, layerIndex uint32) (rows []FilterRow, err error) {
+ if iv.checkInvalid() {
+ return nil, ErrInvalidView
+ }
+
+ dbIndices := len(mapIndices)
+ for dbIndices > 0 && mapIndices[dbIndices-1] >= iv.firstMemoryMap {
+ dbIndices--
+ }
+ if dbIndices > 0 {
+ rows, err = iv.storage.getFilterMapRows(mapIndices[:dbIndices], rowIndex, layerIndex)
+ if iv.checkInvalid() {
+ return nil, ErrInvalidView
+ }
+ if err != nil {
+ return nil, err
+ }
+ }
+ for i := dbIndices; i < len(mapIndices); i++ {
+ mapIndex := mapIndices[i]
+ var row FilterRow
+ if mapIndex == iv.headMapIndex {
+ row = iv.headMap.getRow(rowIndex, iv.storage.params.getMaxRowLength(layerIndex+1))
+ }
+ if mapIndex < iv.headMapIndex {
+ row = iv.finishedMaps[mapIndex-iv.firstMemoryMap].getRow(rowIndex, iv.storage.params.getMaxRowLength(layerIndex+1))
+ }
+ rows = append(rows, row)
+ }
+ return rows, nil
+}
+
+type renderState struct {
+ params *Params
+ renderRange common.Range[uint32]
+ lvPointer uint64
+ mapIndex uint32
+ currentMap *memoryMap
+ finishedMaps []*finishedMap
+ nextBlock uint64
+ partialBlock bool
+ partialBlockHash common.Hash
+}
+
+func (rs *renderState) checkNextHash(hash common.Hash) bool {
+ if rs.partialBlock && rs.partialBlockHash != hash {
+ return false
+ }
+ rs.partialBlock = false
+ return true
+}
+
+func (rs *renderState) addReceipts(receipts types.Receipts) {
+ if rs.partialBlock {
+ panic("checkNextHash has to be called before adding partially rendered block")
+ }
+ if rs.currentMap != nil {
+ rs.currentMap.blockPtrs = append(rs.currentMap.blockPtrs, rs.lvPointer)
+ rs.currentMap.lastBlock = lastBlockOfMap{number: rs.nextBlock} // hash will be set by addHeader
+ }
+ for _, receipt := range receipts {
+ //TODO add tx delimiter
+ for _, log := range receipt.Logs {
+ mapRemaining := rs.params.valuesPerMap - rs.lvPointer%rs.params.valuesPerMap
+ if mapRemaining <= uint64(len(log.Topics)) {
+ rs.advance(mapRemaining)
+ }
+ if rs.currentMap != nil {
+ rs.addValue(addressValue(log.Address))
+ for _, topic := range log.Topics {
+ rs.addValue(topicValue(topic))
+ }
+ } else {
+ rs.advance(uint64(len(log.Topics) + 1))
+ }
+ if rs.finished() {
+ return
+ }
+ }
+ }
+}
+
+func (rs *renderState) addHeader(header *types.Header) (uint32, []*finishedMap) {
+ if rs.partialBlock {
+ panic("checkNextHash has to be called before adding partially rendered block")
+ }
+ if rs.nextBlock != header.Number.Uint64() {
+ panic("wrong block number")
+ }
+ if !rs.finished() {
+ rs.advance(1) //TODO blockValue
+ }
+ rs.nextBlock++
+ lastBlock := lastBlockOfMap{number: header.Number.Uint64(), hash: header.Hash()}
+ if rs.currentMap != nil {
+ rs.currentMap.lastBlock = lastBlock
+ }
+ for _, fm := range rs.finishedMaps {
+ fm.lastBlock = lastBlock
+ }
+ firstMemoryMap, finishedMaps := rs.mapIndex-uint32(len(rs.finishedMaps)), rs.finishedMaps
+ rs.finishedMaps = nil
+ return firstMemoryMap, finishedMaps
+}
+
+// assumes currentMap != nil
+func (rs *renderState) addValue(logValue common.Hash) {
+ if rs.renderRange.Includes(rs.mapIndex) {
+ for layerIndex := uint32(0); ; layerIndex++ {
+ rowIndex := rs.params.rowIndex(rs.mapIndex, layerIndex, logValue)
+ if rs.currentMap.rowLength(rowIndex) < rs.params.getMaxRowLength(layerIndex) {
+ rs.currentMap.addToRow(rowIndex, rs.params.columnIndex(rs.lvPointer, &logValue))
+ break
+ }
+ }
+ }
+ rs.advance(1)
+}
+
+func (rs *renderState) advance(count uint64) {
+ rs.lvPointer += count
+ if uint32(rs.lvPointer>>rs.params.logValuesPerMap) > rs.mapIndex {
+ if rs.currentMap != nil {
+ rs.finishedMaps = append(rs.finishedMaps, rs.currentMap.finished())
+ }
+ rs.mapIndex++
+ if rs.renderRange.Includes(rs.mapIndex) {
+ rs.currentMap = rs.params.newMemoryMap()
+ } else {
+ rs.currentMap = nil
+ }
+ }
+}
+
+func (rs *renderState) finished() bool {
+ return rs.mapIndex >= rs.renderRange.AfterLast()
+}
diff --git a/core/filtermaps/indexer.go b/core/filtermaps/indexer.go
index ca50fb466c..675ce3a6b5 100644
--- a/core/filtermaps/indexer.go
+++ b/core/filtermaps/indexer.go
@@ -1,4 +1,4 @@
-// Copyright 2024 The go-ethereum Authors
+// Copyright 2025 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
@@ -17,442 +17,645 @@
package filtermaps
import (
- "errors"
+ "fmt"
"math"
- "time"
+ "os"
+ "sync"
"github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/common/lru"
+ "github.com/ethereum/go-ethereum/core/types"
+ "github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/log"
+ "github.com/ethereum/go-ethereum/metrics"
)
const (
- logFrequency = time.Second * 20 // log info frequency during long indexing/unindexing process
- headLogDelay = time.Second // head indexing log info delay (do not log if finished faster)
+ maxCanonicalSnapshots = 4
+ maxRecentSnapshots = 4
+ maxIndexViewMaps = 2
)
-// indexerLoop initializes and updates the log index structure according to the
-// current targetView.
-func (f *FilterMaps) indexerLoop() {
- defer f.closeWg.Done()
+var (
+ mapCountGauge = metrics.NewRegisteredGauge("filtermaps/maps/count", nil) // actual number of rendered maps
+ mapLogValueMeter = metrics.NewRegisteredMeter("filtermaps/maps/logvalues", nil) // number of log values processed
+ mapBlockMeter = metrics.NewRegisteredMeter("filtermaps/maps/blocks", nil) // number of block delimiters processed
+ mapRenderTimer = metrics.NewRegisteredTimer("filtermaps/maps/rendertime", nil) // time elapsed while rendering a single map
+ mapWriteTimer = metrics.NewRegisteredTimer("filtermaps/maps/writetime", nil) // time elapsed while writing a batch of finished maps to db
+)
- if f.disabled {
- f.reset()
- close(f.disabledCh)
- return
+// Indexer maintains a search data structure based on a parent blockchain that
+// is intended to make log event search more efficient. Once indexed up to the
+// chain head, it provides IndexView objects for recent chain heads.
+// Indexer implements core.Indexer.
+type Indexer struct {
+ config Config
+ storage *mapStorage
+ checkpoints []checkpointList
+ headRenderer, tailRenderer *renderState
+ historyCutoff, finalized uint64
+ tailRenderLast, headNumber uint64
+ lastCanonical uint64
+ tailEpoch, targetTailEpoch, activeViewTailEpoch uint32
+ canonicalHashes []common.Hash // last one belongs to lastCanonical
+ recentHashes []common.Hash // last one is the most recently saved
+ snapshotsLock sync.RWMutex
+ snapshots map[common.Hash]*IndexView
+ headMapsCache *lru.Cache[uint32, *finishedMap]
+ lastFinalEpoch uint32
+}
+
+// Config contains the configuration options for Indexer.
+type Config struct {
+ History uint64 // number of historical blocks to index
+ Disabled bool // disables indexing completely
+
+ // This option enables the checkpoint JSON file generator.
+ // If set, the given file will be updated with checkpoint information.
+ ExportFileName string
+
+ // expect trie nodes of hash based state scheme in the filtermaps key range;
+ // use safe iterator based implementation of DeleteRange that skips them
+ HashScheme bool
+}
+
+// NewIndexer creates a new Indexer.
+func NewIndexer(db ethdb.KeyValueStore, params Params, config Config) *Indexer {
+ params.sanitize()
+ mapDb := newMapDatabase(¶ms, db, config.HashScheme)
+ ix := &Indexer{
+ config: config,
+ storage: newMapStorage(¶ms, mapDb, nil),
+ checkpoints: checkpoints,
+ snapshots: make(map[common.Hash]*IndexView),
+ headMapsCache: lru.NewCache[uint32, *finishedMap](maxIndexViewMaps),
+ lastFinalEpoch: math.MaxUint32,
}
- log.Info("Started log indexer")
+ if config.Disabled {
+ ix.storage.deleteMaps(common.NewRange[uint32](0, math.MaxUint32))
+ return ix
+ }
+ ix.headRenderer = ix.initMapBoundary(ix.storage.lastBoundaryBefore(math.MaxUint32), math.MaxUint32)
+ ix.updateTailEpoch()
+ ix.updateActiveViewTailEpoch()
+ ix.updateTailState()
+ return ix
+}
- for !f.stop {
- // Note: acquiring the indexLock read lock is unnecessary here,
- // as the `indexedRange` is accessed within the indexerLoop.
- if !f.indexedRange.initialized {
- if f.targetView.HeadNumber() == 0 {
- // initialize when chain head is available
- f.processSingleEvent(true)
- continue
- }
- if err := f.init(); err != nil {
- f.disableForError("initialization", err)
- f.reset() // remove broken index from DB
- return
- }
- }
- if !f.targetHeadIndexed() {
- if err := f.tryIndexHead(); err != nil && err != errChainUpdate {
- f.disableForError("head rendering", err)
- return
+// GetIndexView returns an immutable IndexView corresponding to the given head
+// block hash if available. Note that each returned IndexView has to be released
+// after use with the Release funcion in order to avoid memory leakage.
+func (ix *Indexer) GetIndexView(headBlockHash common.Hash) *IndexView {
+ if ix.config.Disabled {
+ return nil
+ }
+ ix.snapshotsLock.RLock()
+ iv := ix.snapshots[headBlockHash]
+ ix.snapshotsLock.RUnlock()
+ if iv == nil || iv.checkReleased() {
+ return nil
+ }
+ iv.addRefCount(1)
+ return iv
+}
+
+// Status returns the current indexer status. The ready flag indicates whether
+// the indexer is ready to process new block data. The needBlocks range, if not
+// empty, indicates that the indexer requests past blocks in order to complete
+// the index. These blocks should be delivered in strictly ascending order.
+// Note that if ready is false then needBlocks might still be non-empty, in
+// which case the blocks are not expected to be delivered yet but the index
+// server might already start pre-fetching them.
+// Status implements core.Indexer.
+func (ix *Indexer) Status() (bool, common.Range[uint64]) {
+ if ix.config.Disabled {
+ return false, common.Range[uint64]{}
+ }
+ return ix.storage.isReady(), ix.needBlocks()
+}
+
+// AddBlockData delivers block data for new heads and requested historical range.
+// It returns the indexer status. Unwanted data is silently ignored. If the
+// indexer is not ready to process then the received data is also ignored and
+// then requested through the needBlocks response either in the current response
+// or later.
+// Note that this function also resumes the storage layer background process if
+// it was previously suspended.
+// AddBlockData implements core.Indexer.
+func (ix *Indexer) AddBlockData(header *types.Header, receipts types.Receipts) (ready bool, needBlocks common.Range[uint64]) {
+ if ix.config.Disabled {
+ return false, common.Range[uint64]{}
+ }
+ ix.storage.suspendOrResume(false)
+ if !ix.storage.isReady() {
+ return false, ix.needBlocks()
+ }
+ ix.headNumber = max(ix.headNumber, header.Number.Uint64())
+ number, hash := header.Number.Uint64(), header.Hash()
+ if number > ix.headRenderer.nextBlock {
+ ix.tryCheckpointInit(number, hash)
+ }
+ if number == ix.headRenderer.nextBlock {
+ if ix.headRenderer.checkNextHash(hash) {
+ ix.headRenderer.addReceipts(receipts)
+ firstMapIndex, finishedMaps := ix.headRenderer.addHeader(header)
+ ix.storeFinishedMaps(firstMapIndex, finishedMaps, true, true)
+ if number+maxCanonicalSnapshots > ix.headNumber {
+ ix.storeHeadIndexView(number, hash)
}
} else {
- if f.finalBlock != f.lastFinal {
- if f.exportFileName != "" {
- f.exportCheckpoints()
+ ix.headRenderer = ix.initMapBoundary(max(ix.headRenderer.renderRange.First(), 1)-1, math.MaxUint32)
+ }
+ ix.updateTailEpoch()
+ ix.updateTailState()
+ }
+ if ix.tailRenderer != nil && number == ix.tailRenderer.nextBlock {
+ if ix.tailRenderer.checkNextHash(hash) {
+ ix.tailRenderer.addReceipts(receipts)
+ firstMapIndex, finishedMaps := ix.tailRenderer.addHeader(header)
+ ix.storeFinishedMaps(firstMapIndex, finishedMaps, false, false)
+ if ix.tailRenderer.finished() {
+ ix.tailEpoch--
+ ix.tailRenderer = nil
+ ix.updateTailState()
+ }
+ } else {
+ // Note that if there is a canonical hash mismatch at the tail epoch then we need to revert the head renderer before this point.
+ ix.headRenderer = ix.initMapBoundary(max(ix.tailRenderer.renderRange.First(), 1)-1, math.MaxUint32)
+ ix.tailRenderer = nil
+ }
+ }
+ return ix.storage.isReady(), ix.needBlocks()
+}
+
+// Revert resets the index head to the given block number. Note that the indexer
+// might have to discard more data if a snapshot is not available for the given
+// block number. In this case it will request previously delivered but discarded
+// block data through the needBlocks status response.
+// Note that Revert works even if the indexer is in a "not ready" status, thereby
+// guaranteeing that all index data is always consistent with the canonical chain.
+// Revert implements core.Indexer.
+func (ix *Indexer) Revert(blockNumber uint64) {
+ if ix.config.Disabled {
+ return
+ }
+ firstCanonical := ix.lastCanonical + 1 - uint64(len(ix.canonicalHashes))
+ if blockNumber >= firstCanonical && blockNumber <= ix.lastCanonical {
+ blockHash := ix.canonicalHashes[blockNumber-firstCanonical]
+ if snapshot, ok := ix.snapshots[blockHash]; ok {
+ ix.headRenderer = ix.initSnapshot(snapshot)
+ if ix.headRenderer != nil {
+ return
+ }
+ }
+ }
+ mapIndex := uint32(math.MaxUint32)
+ for mapIndex > 0 {
+ mapIndex = ix.storage.lastBoundaryBefore(mapIndex - 1)
+ if mapIndex == 0 {
+ break
+ }
+ lastNumber, _, err := ix.storage.getLastBlockOfMap(mapIndex - 1)
+ if err != nil {
+ log.Error("Last block of map not found, reverting database", "mapIndex", mapIndex)
+ mapIndex--
+ continue
+ }
+ if lastNumber < blockNumber {
+ break
+ }
+ }
+ ix.revertMaps(mapIndex)
+ ix.headRenderer = ix.initMapBoundary(mapIndex, math.MaxUint32)
+ ix.headNumber = blockNumber
+ ix.updateTailEpoch()
+}
+
+// SetFinalized notifies the indexer about the latest finalized block number.
+// SetFinalized implements core.Indexer.
+func (ix *Indexer) SetFinalized(blockNumber uint64) {
+ if ix.finalized == blockNumber {
+ return
+ }
+ ix.finalized = blockNumber
+ ix.exportCheckpoints()
+}
+
+// SetHistoryCutoff notifies the indexer about the latest historical cutoff point.
+// The indexer will not request block data earlier than this point.
+// SetHistoryCutoff implements core.Indexer.
+func (ix *Indexer) SetHistoryCutoff(blockNumber uint64) {
+ ix.historyCutoff = blockNumber
+}
+
+// Suspended suspends the asynchronous storage layer background process during
+// block processing. The next AddBlockData call will resume this process.
+// Suspended implements core.Indexer.
+func (ix *Indexer) Suspended() {
+ if ix.config.Disabled {
+ return
+ }
+ ix.storage.suspendOrResume(true)
+}
+
+// initMapBoundary initializes a new map renderer at the last suitable map
+// boundary before startMap. If this boundary is not right before startMap then
+// startMap is lowered to right after the boundary. The returned renderState
+// will render maps in the startMap..limitMap-1 range.
+// Note that the first requested block typically still starts in the previous
+// map and in case of tail renderers with an upper map limit, the last requested
+// block typically ends after the upper limit. In this case the maps outside the
+// rendered range are not modified, the log values outside the range are ignored.
+func (ix *Indexer) initMapBoundary(startMap, limitMap uint32) *renderState {
+ rs := &renderState{
+ params: ix.storage.params,
+ }
+ for {
+ startMap = ix.storage.lastBoundaryBefore(startMap)
+ if startMap == 0 {
+ break
+ }
+ lastNumber, lastHash, err := ix.storage.getLastBlockOfMap(startMap - 1)
+ if err != nil {
+ log.Error("Last block of map not found, reverting database", "mapIndex", startMap-1)
+ startMap = ix.storage.lastBoundaryBefore(startMap - 1)
+ ix.revertMaps(startMap)
+ continue
+ }
+ lvPointer, err := ix.storage.getBlockLvPointer(lastNumber)
+ if err != nil {
+ log.Error("Block pointer of last block of map not found, reverting database", "mapIndex", startMap-1, "blockNumber", lastNumber)
+ startMap = ix.storage.lastBoundaryBefore(startMap - 1)
+ ix.revertMaps(startMap)
+ continue
+ }
+ rs.lvPointer = lvPointer
+ rs.mapIndex = uint32(lvPointer >> ix.storage.params.logValuesPerMap)
+ rs.nextBlock = lastNumber
+ rs.partialBlock = true
+ rs.partialBlockHash = lastHash
+ break
+ }
+ rs.renderRange = common.NewRange[uint32](startMap, limitMap-startMap)
+ if rs.renderRange.Includes(rs.mapIndex) {
+ rs.currentMap = rs.params.newMemoryMap()
+ }
+ return rs
+}
+
+// initSnapshot initializes a new map renderer based on a snapshot. Since this
+// method is only used to initialize head renderers, a snapshot initialized
+// renderState always has an upper render limit of MaxUint32-1.
+func (ix *Indexer) initSnapshot(snapshot *IndexView) *renderState {
+ mapIndex := ix.storage.lastBoundaryBefore(snapshot.firstMemoryMap)
+ ix.revertMaps(mapIndex)
+ if snapshot.checkInvalid() {
+ log.Error("Failed to revert to invalidated snapshot", "blockNumber", snapshot.blockRange.Last())
+ return nil
+ }
+
+ return &renderState{
+ params: ix.storage.params,
+ renderRange: common.NewRange[uint32](snapshot.headMapIndex, math.MaxUint32-snapshot.headMapIndex),
+ currentMap: snapshot.headMap.clone(),
+ mapIndex: snapshot.headMapIndex,
+ lvPointer: snapshot.headLvPointer,
+ }
+}
+
+// revertMaps removes all rendered maps starting from mapIndex. It also removes
+// active renderers and snapshots invalidated by the revert and purges the map
+// cache.
+// Note that while headRenderer generally always exists, revertMaps might be
+// called while headRenderer is nil and might set headRenderer to nil.
+func (ix *Indexer) revertMaps(mapIndex uint32) {
+ if mapIndex < ix.storage.lastBoundaryBefore(math.MaxUint32) {
+ for hash, iv := range ix.snapshots {
+ if iv.firstMemoryMap > mapIndex {
+ iv.invalidate()
+ ix.snapshotsLock.Lock()
+ delete(ix.snapshots, hash)
+ ix.snapshotsLock.Unlock()
+ }
+ }
+ ix.storage.deleteMaps(common.NewRange[uint32](mapIndex, math.MaxUint32-mapIndex))
+ ix.headMapsCache.Purge() // invalidate all maps cached by index
+ }
+ if ix.headRenderer != nil && mapIndex <= ix.headRenderer.mapIndex {
+ ix.headRenderer = nil
+ }
+ if ix.tailRenderer != nil && mapIndex <= ix.tailRenderer.mapIndex {
+ ix.tailRenderer = nil
+ }
+}
+
+// updateTailEpoch recalculates the current tailEpoch and the targetTailEpoch.
+func (ix *Indexer) updateTailEpoch() {
+ ix.tailEpoch = ix.storage.tailEpoch()
+ if ix.config.History == 0 {
+ ix.targetTailEpoch = 0
+ return
+ }
+ headEpoch := ix.storage.params.mapEpoch(ix.headRenderer.mapIndex)
+ for ix.targetTailEpoch < headEpoch {
+ nextTailNumber, err := ix.storage.tailNumberOfEpoch(ix.targetTailEpoch + 1)
+ if err != nil {
+ log.Error("Could not get tail block number of epoch", "epoch", ix.targetTailEpoch+1, "error", err)
+ return
+ }
+ if nextTailNumber+ix.config.History > ix.headRenderer.nextBlock {
+ break
+ }
+ ix.targetTailEpoch++
+ }
+ for ix.targetTailEpoch > 0 {
+ prevTailEpoch := ix.storage.params.mapEpoch(ix.storage.lastBoundaryBefore(ix.storage.params.firstEpochMap(ix.targetTailEpoch - 1)))
+ prevTailNumber, err := ix.storage.tailNumberOfEpoch(prevTailEpoch)
+ if err != nil {
+ log.Error("Could not get tail block number of epoch", "epoch", prevTailEpoch, "error", err)
+ return
+ }
+ if prevTailNumber+ix.config.History <= ix.headRenderer.nextBlock {
+ break
+ }
+ ix.targetTailEpoch = prevTailEpoch
+ }
+}
+
+// updateActiveViewTailEpoch recalculates activeViewTailEpoch which is the earliest
+// tail epoch required by an active IndexView. Tail unindexing is only allowed
+// if min(targetTailEpoch, activeViewTailEpoch) > tailEpoch.
+func (ix *Indexer) updateActiveViewTailEpoch() {
+ ix.snapshotsLock.RLock()
+ defer ix.snapshotsLock.RUnlock()
+
+ ix.activeViewTailEpoch = math.MaxUint32
+ for _, iv := range ix.snapshots {
+ ix.activeViewTailEpoch = min(ix.activeViewTailEpoch, iv.tailEpoch)
+ }
+}
+
+// updateTailState performs tail unindexing or initializes a new tailRenderer to
+// render a new tail epoch if necessary.
+func (ix *Indexer) updateTailState() {
+ epoch := min(ix.targetTailEpoch, ix.activeViewTailEpoch)
+ if epoch >= ix.tailEpoch && ix.tailRenderer != nil {
+ ix.tailRenderer = nil
+ ix.storage.deleteMaps(common.NewRange[uint32](ix.storage.params.firstEpochMap(ix.tailEpoch-1), ix.storage.params.mapsPerEpoch))
+ }
+ if epoch > ix.tailEpoch {
+ ix.storage.deleteMaps(common.NewRange[uint32](ix.storage.params.firstEpochMap(ix.tailEpoch), ix.storage.params.mapsPerEpoch*(epoch-ix.tailEpoch)))
+ if epoch == ix.tailEpoch+1 {
+ log.Info("Unindexed tail epoch #%d", ix.tailEpoch)
+ } else {
+ log.Info("Unindexed tail epochs #%d to #%d", ix.tailEpoch, epoch-1)
+ }
+ ix.tailEpoch = epoch
+ }
+ if epoch < ix.tailEpoch && ix.tailRenderer == nil {
+ if lastBlock, _, err := ix.storage.getLastBlockOfMap(ix.storage.params.lastEpochMap(ix.tailEpoch - 1)); err == nil {
+ ix.tailRenderer = ix.initMapBoundary(ix.storage.lastBoundaryBefore(ix.storage.params.lastEpochMap(ix.tailEpoch-1)), ix.storage.params.firstEpochMap(ix.tailEpoch))
+ ix.tailRenderLast = lastBlock
+ } else {
+ log.Error("Could not get last block of new tail epoch", "epoch", ix.tailEpoch-1, "error", err)
+ }
+ }
+}
+
+// epochsUntilBlock returns the numer of epochs in the checkpoint list whose
+// last block number is less than or equal to the specified number.
+func (cpList checkpointList) epochsUntilBlock(number uint64) uint32 {
+ first, last := uint32(0), uint32(len(cpList))
+ for first < last {
+ mid := (first + last) / 2
+ if cpList[mid].BlockNumber > number {
+ last = mid
+ } else {
+ first = mid + 1
+ }
+ }
+ return first
+}
+
+func (ix *Indexer) tryCheckpointInit(number uint64, hash common.Hash) {
+ var ci int
+ for ci < len(ix.checkpoints) {
+ cpList := ix.checkpoints[ci]
+ epochs := cpList.epochsUntilBlock(number)
+ if epochs == 0 || cpList[epochs-1].BlockNumber != number {
+ // block number does not match, skip list (a relevant block might match later)
+ ci++
+ continue
+ }
+ if cpList[epochs-1].BlockHash == hash {
+ // apply matching checkpoint, discard other lists
+ if err := ix.storage.addKnownEpochs(cpList[:epochs]); err == nil {
+ ix.checkpoints = []checkpointList{cpList}
+ ix.headRenderer = ix.initMapBoundary(ix.storage.params.firstEpochMap(epochs), math.MaxUint32)
+ return
+ } else {
+ log.Error("Error initializing epoch boundaries", "error", err)
+ }
+ }
+ // checkpoint does not match, discard list
+ ix.checkpoints[ci] = ix.checkpoints[len(ix.checkpoints)-1]
+ ix.checkpoints = ix.checkpoints[:len(ix.checkpoints)-1]
+ }
+}
+
+func (ix *Indexer) needBlocks() common.Range[uint64] {
+ if ix.finalized > ix.headRenderer.nextBlock {
+ // request potential checkpoint in this range if available
+ for _, cpList := range ix.checkpoints {
+ if epochs := cpList.epochsUntilBlock(ix.headNumber); epochs > 0 {
+ blockNumber := cpList[epochs-1].BlockNumber
+ if ix.storage.lastBoundaryBefore(math.MaxUint32) >= ix.storage.params.firstEpochMap(epochs) ||
+ blockNumber <= ix.headRenderer.nextBlock || blockNumber < ix.historyCutoff {
+ continue
}
- f.lastFinal = f.finalBlock
+ return common.NewRange[uint64](blockNumber, 1)
}
- // always attempt unindexing before indexing the tail in order to
- // ensure that a potentially dirty previously unindexed epoch is
- // always cleaned up before any new maps are rendered.
- if done, err := f.tryUnindexTail(); err != nil {
- f.disableForError("tail unindexing", err)
- return
- } else if !done {
- continue
- }
- if done, err := f.tryIndexTail(); err != nil {
- f.disableForError("tail rendering", err)
- return
- } else if !done {
- continue
- }
- // tail indexing/unindexing is done; if head is also indexed then
- // wait here until there is a new head
- f.waitForNewHead()
+ }
+ }
+ if ix.headRenderer.nextBlock <= ix.headNumber && ix.headRenderer.nextBlock >= ix.historyCutoff {
+ return common.NewRange[uint64](ix.headRenderer.nextBlock, ix.headNumber+1-ix.headRenderer.nextBlock)
+ }
+ if ix.tailRenderer != nil &&
+ ix.tailRenderer.nextBlock <= ix.tailRenderLast && ix.tailRenderer.nextBlock >= ix.historyCutoff {
+ return common.NewRange[uint64](ix.tailRenderer.nextBlock, ix.tailRenderLast+1-ix.tailRenderer.nextBlock)
+ }
+ return common.Range[uint64]{}
+}
+
+func (ix *Indexer) Stop() {
+ ix.storage.stop()
+}
+
+func (ix *Indexer) releaseView(hash common.Hash) {
+ iv := ix.snapshots[hash]
+ if iv == nil {
+ return
+ }
+ if iv.addRefCount(-1) {
+ iv.invalidate()
+ ix.snapshotsLock.Lock()
+ delete(ix.snapshots, hash)
+ ix.snapshotsLock.Unlock()
+ }
+}
+
+func (ix *Indexer) storeFinishedMaps(firstMapIndex uint32, maps []*finishedMap, forceCommit, cacheHeadMaps bool) {
+ if len(maps) == 0 {
+ return
+ }
+ for i, fm := range maps {
+ ix.storage.addMap(firstMapIndex+uint32(i), fm, forceCommit && i == len(maps)-1)
+ if cacheHeadMaps {
+ ix.headMapsCache.Add(firstMapIndex+uint32(i), fm)
}
}
}
-// disableForError is called when the indexer encounters a database error, for example a
-// missing receipt. We can't continue operating when the database is broken, so the
-// indexer goes into disabled state.
-// Note that the partial index is left in disk; maybe a client update can fix the
-// issue without reindexing.
-func (f *FilterMaps) disableForError(op string, err error) {
- log.Error("Log index "+op+" failed, reverting to unindexed mode", "error", err)
- f.disabled = true
- close(f.disabledCh)
-}
-
-type targetUpdate struct {
- targetView *ChainView
- historyCutoff, finalBlock uint64
-}
-
-// SetTarget sets a new target chain view for the indexer to render.
-// Note that SetTargetView never blocks.
-func (f *FilterMaps) SetTarget(targetView *ChainView, historyCutoff, finalBlock uint64) {
- if targetView == nil {
- panic("nil targetView")
+func (ix *Indexer) getFilterMap(mapIndex uint32) (*finishedMap, error) {
+ if fm, ok := ix.headMapsCache.Get(mapIndex); ok {
+ return fm, nil
}
- for {
- select {
- case <-f.targetCh:
- case f.targetCh <- targetUpdate{
- targetView: targetView,
- historyCutoff: historyCutoff,
- finalBlock: finalBlock,
- }:
- return
+ fm, err := ix.storage.getFilterMap(mapIndex)
+ if err != nil {
+ return nil, err
+ }
+ ix.headMapsCache.Add(mapIndex, fm)
+ return fm, nil
+}
+
+func (ix *Indexer) checkReleasedViews() {
+ var deleted bool
+ for hash, iv := range ix.snapshots {
+ if iv.checkReleased() {
+ iv.invalidate()
+ ix.snapshotsLock.Lock()
+ delete(ix.snapshots, hash)
+ ix.snapshotsLock.Unlock()
+ deleted = true
}
}
+ if deleted {
+ ix.updateActiveViewTailEpoch()
+ ix.updateTailState()
+ }
}
-// SetBlockProcessing sets the block processing flag that temporarily suspends
-// log index rendering.
-// Note that SetBlockProcessing never blocks.
-func (f *FilterMaps) SetBlockProcessing(blockProcessing bool) {
- for {
- select {
- case <-f.blockProcessingCh:
- case f.blockProcessingCh <- blockProcessing:
- return
+func (ix *Indexer) storeHeadIndexView(number uint64, hash common.Hash) {
+ if ix.headRenderer.currentMap == nil {
+ return
+ }
+ ix.checkReleasedViews()
+ firstMemoryMap := max(ix.headRenderer.mapIndex, maxIndexViewMaps) - maxIndexViewMaps
+ finishedMaps := make([]*finishedMap, 0, ix.headRenderer.mapIndex-firstMemoryMap)
+ for mapIndex := firstMemoryMap; mapIndex < ix.headRenderer.mapIndex; mapIndex++ {
+ fm, err := ix.getFilterMap(mapIndex)
+ if err != nil {
+ log.Error("Error loading recent filter map", "mapIndex", mapIndex, "error", err)
+ }
+ if fm != nil && err == nil {
+ finishedMaps = append(finishedMaps, fm)
+ } else {
+ finishedMaps = finishedMaps[:0]
+ firstMemoryMap = mapIndex + 1
}
}
-}
-
-// WaitIdle blocks until the indexer is in an idle state while synced up to the
-// latest targetView.
-func (f *FilterMaps) WaitIdle() {
- for {
- ch := make(chan bool)
- select {
- case f.waitIdleCh <- ch:
- if <-ch {
- return
- }
- case <-f.disabledCh:
- f.closeWg.Wait()
- return
- }
+ var firstMemoryBlock uint64
+ if len(finishedMaps) > 0 {
+ firstMemoryBlock = finishedMaps[0].firstBlock()
+ } else {
+ firstMemoryBlock = ix.headRenderer.currentMap.firstBlock()
}
-}
-
-// waitForNewHead blocks until there is a new target head to index and block
-// processing has been finished.
-func (f *FilterMaps) waitForNewHead() {
- for !f.stop && (f.blockProcessing || f.targetHeadIndexed()) {
- f.processSingleEvent(true)
+ tailEpoch := max(ix.tailEpoch, ix.targetTailEpoch)
+ tailNumber, err := ix.storage.tailNumberOfEpoch(tailEpoch)
+ if err != nil {
+ log.Error("Could not get tail block number of epoch", "epoch", tailEpoch, "error", err)
+ return
}
-}
-
-// processEvents processes all events, blocking only if a block processing is
-// happening and indexing should be suspended.
-func (f *FilterMaps) processEvents() {
- if f.testProcessEventsHook != nil {
- f.testProcessEventsHook()
+ ix.snapshotsLock.Lock()
+ ix.snapshots[hash] = &IndexView{
+ refCount: 2,
+ storage: ix.storage,
+ tailEpoch: tailEpoch,
+ blockRange: common.NewRange(tailNumber, number+1-tailNumber),
+ headBlockHash: hash,
+ headLvPointer: ix.headRenderer.lvPointer,
+ headMap: ix.headRenderer.currentMap.clone(),
+ headMapIndex: ix.headRenderer.mapIndex,
+ firstMemoryMap: firstMemoryMap,
+ firstMemoryBlock: firstMemoryBlock,
+ finishedMaps: finishedMaps,
}
- for f.processSingleEvent(f.blockProcessing) {
- }
-}
-
-// processSingleEvent processes a single event either in a blocking or
-// non-blocking manner. It returns true if it did process an event.
-func (f *FilterMaps) processSingleEvent(blocking bool) bool {
- if f.stop {
- return false
- }
- // Note: acquiring the indexLock read lock is unnecessary here,
- // as this function is always called within the indexLoop.
- if !f.hasTempRange {
- for _, mb := range f.matcherSyncRequests {
- mb.synced()
- }
- f.matcherSyncRequests = nil
- }
- if blocking {
- select {
- case target := <-f.targetCh:
- f.setTarget(target)
- case mb := <-f.matcherSyncCh:
- f.matcherSyncRequests = append(f.matcherSyncRequests, mb)
- case f.blockProcessing = <-f.blockProcessingCh:
- case <-f.closeCh:
- f.stop = true
- case ch := <-f.waitIdleCh:
- select {
- case target := <-f.targetCh:
- f.setTarget(target)
- default:
- }
- ch <- !f.blockProcessing && f.targetHeadIndexed()
+ ix.snapshotsLock.Unlock()
+ if number == ix.lastCanonical+1 {
+ if len(ix.canonicalHashes) == maxCanonicalSnapshots {
+ ix.releaseView(ix.canonicalHashes[0])
+ copy(ix.canonicalHashes[0:maxCanonicalSnapshots-1], ix.canonicalHashes[1:maxCanonicalSnapshots])
+ ix.canonicalHashes[maxCanonicalSnapshots-1] = hash
+ } else {
+ ix.canonicalHashes = append(ix.canonicalHashes, hash)
}
} else {
- select {
- case target := <-f.targetCh:
- f.setTarget(target)
- case mb := <-f.matcherSyncCh:
- f.matcherSyncRequests = append(f.matcherSyncRequests, mb)
- case f.blockProcessing = <-f.blockProcessingCh:
- case <-f.closeCh:
- f.stop = true
- default:
- return false
+ for _, oldHash := range ix.canonicalHashes {
+ ix.releaseView(oldHash)
}
+ ix.canonicalHashes = []common.Hash{hash}
}
- return true
+ ix.lastCanonical = number
+ if len(ix.recentHashes) == maxRecentSnapshots {
+ ix.releaseView(ix.recentHashes[0])
+ copy(ix.recentHashes[0:maxRecentSnapshots-1], ix.recentHashes[1:maxRecentSnapshots])
+ ix.recentHashes[maxRecentSnapshots-1] = hash
+ } else {
+ ix.recentHashes = append(ix.recentHashes, hash)
+ }
+ ix.updateActiveViewTailEpoch()
+ ix.updateTailState()
}
-// setTarget updates the target chain view of the iterator.
-func (f *FilterMaps) setTarget(target targetUpdate) {
- f.targetView = target.targetView
- f.historyCutoff = target.historyCutoff
- f.finalBlock = target.finalBlock
-}
-
-// tryIndexHead tries to render head maps according to the current targetView.
-// Should be called when targetHeadIndexed returns false. If this function
-// returns no error then either stop is true or head indexing is finished.
-func (f *FilterMaps) tryIndexHead() error {
- headRenderer, err := f.renderMapsBefore(math.MaxUint32)
+// exportCheckpoints exports epoch checkpoints in the format used by checkpoints.go.
+func (ix *Indexer) exportCheckpoints() {
+ finalLvPtr, err := ix.storage.getBlockLvPointer(ix.finalized + 1)
if err != nil {
- return err
- }
- if headRenderer == nil {
- return errors.New("head indexer has nothing to do") // tryIndexHead should be called when head is not indexed
- }
- if !f.startedHeadIndex {
- f.lastLogHeadIndex = time.Now()
- f.startedHeadIndexAt = f.lastLogHeadIndex
- f.startedHeadIndex = true
- f.ptrHeadIndex = f.indexedRange.blocks.AfterLast()
- }
- if _, err := headRenderer.run(func() bool {
- f.processEvents()
- return f.stop
- }, func() {
- f.tryUnindexTail()
- if f.indexedRange.hasIndexedBlocks() && f.indexedRange.blocks.AfterLast() >= f.ptrHeadIndex &&
- ((!f.loggedHeadIndex && time.Since(f.startedHeadIndexAt) > headLogDelay) ||
- time.Since(f.lastLogHeadIndex) > logFrequency) {
- log.Info("Log index head rendering in progress",
- "firstblock", f.indexedRange.blocks.First(), "lastblock", f.indexedRange.blocks.Last(),
- "processed", f.indexedRange.blocks.AfterLast()-f.ptrHeadIndex,
- "remaining", f.indexedView.HeadNumber()-f.indexedRange.blocks.Last(),
- "elapsed", common.PrettyDuration(time.Since(f.startedHeadIndexAt)))
- f.loggedHeadIndex = true
- f.lastLogHeadIndex = time.Now()
+ if err != ErrOutOfRange {
+ log.Error("Error fetching log value pointer of finalized block", "block", ix.finalized, "error", err)
}
- }); err != nil {
- return err
+ return
}
- if f.loggedHeadIndex && f.indexedRange.hasIndexedBlocks() {
- log.Info("Log index head rendering finished",
- "firstblock", f.indexedRange.blocks.First(), "lastblock", f.indexedRange.blocks.Last(),
- "processed", f.indexedRange.blocks.AfterLast()-f.ptrHeadIndex,
- "elapsed", common.PrettyDuration(time.Since(f.startedHeadIndexAt)))
+ epochCount := ix.storage.params.mapEpoch(uint32(finalLvPtr >> ix.storage.params.logValuesPerMap))
+ if epochCount == ix.lastFinalEpoch {
+ return
}
- f.loggedHeadIndex, f.startedHeadIndex = false, false
- return nil
-}
-
-// tryIndexTail tries to render tail epochs until the tail target block is
-// indexed and returns true if successful.
-// Note that tail indexing is only started if the log index head is fully
-// rendered according to targetView and is suspended as soon as the targetView
-// is changed.
-func (f *FilterMaps) tryIndexTail() (bool, error) {
- for {
- firstEpoch := f.mapEpoch(f.indexedRange.maps.First())
- if firstEpoch == 0 || !f.needTailEpoch(firstEpoch-1) {
- break
- }
- f.processEvents()
- if f.stop || !f.targetHeadIndexed() {
- return false, nil
- }
- // resume process if tail rendering was interrupted because of head rendering
- tailRenderer := f.tailRenderer
- f.tailRenderer = nil
- if tailRenderer != nil && tailRenderer.renderBefore != f.indexedRange.maps.First() {
- tailRenderer = nil
- }
- if tailRenderer == nil {
- var err error
- tailRenderer, err = f.renderMapsBefore(f.indexedRange.maps.First())
- if err != nil {
- return false, err
- }
- }
- if tailRenderer == nil {
- break
- }
- if !f.startedTailIndex {
- f.lastLogTailIndex = time.Now()
- f.startedTailIndexAt = f.lastLogTailIndex
- f.startedTailIndex = true
- f.ptrTailIndex = f.indexedRange.blocks.First() - f.tailPartialBlocks()
- }
- done, err := tailRenderer.run(func() bool {
- f.processEvents()
- return f.stop || !f.targetHeadIndexed()
- }, func() {
- tpb, ttb := f.tailPartialBlocks(), f.tailTargetBlock()
- remaining := uint64(1)
- if f.indexedRange.blocks.First() > ttb+tpb {
- remaining = f.indexedRange.blocks.First() - ttb - tpb
- }
- if f.indexedRange.hasIndexedBlocks() && f.ptrTailIndex >= f.indexedRange.blocks.First() &&
- (!f.loggedTailIndex || time.Since(f.lastLogTailIndex) > logFrequency) {
- log.Info("Log index tail rendering in progress",
- "firstblock", f.indexedRange.blocks.First(), "last block", f.indexedRange.blocks.Last(),
- "processed", f.ptrTailIndex-f.indexedRange.blocks.First()+tpb,
- "remaining", remaining,
- "next tail epoch percentage", f.indexedRange.tailPartialEpoch*100/f.mapsPerEpoch,
- "elapsed", common.PrettyDuration(time.Since(f.startedTailIndexAt)))
- f.loggedTailIndex = true
- f.lastLogTailIndex = time.Now()
- }
- })
- if err != nil && !f.needTailEpoch(firstEpoch-1) {
- // stop silently if cutoff point has move beyond epoch boundary while rendering
- return true, nil
- }
- if err != nil {
- return false, err
- }
- if !done {
- f.tailRenderer = tailRenderer // only keep tail renderer if interrupted by stopCb
- return false, nil
- }
- }
- if f.loggedTailIndex && f.indexedRange.hasIndexedBlocks() {
- log.Info("Log index tail rendering finished",
- "firstblock", f.indexedRange.blocks.First(), "lastblock", f.indexedRange.blocks.Last(),
- "processed", f.ptrTailIndex-f.indexedRange.blocks.First(),
- "elapsed", common.PrettyDuration(time.Since(f.startedTailIndexAt)))
- f.loggedTailIndex = false
- }
- return true, nil
-}
-
-// tryUnindexTail removes entire epochs of log index data as long as the first
-// fully indexed block is at least as old as the tail target.
-// Note that unindexing is very quick as it only removes continuous ranges of
-// data from the database and is also called while running head indexing.
-func (f *FilterMaps) tryUnindexTail() (bool, error) {
- firstEpoch := f.mapEpoch(f.indexedRange.maps.First())
- if f.indexedRange.tailPartialEpoch > 0 && firstEpoch > 0 {
- firstEpoch--
- }
- for epoch := min(firstEpoch, f.cleanedEpochsBefore); !f.needTailEpoch(epoch); epoch++ {
- if !f.startedTailUnindex {
- f.startedTailUnindexAt = time.Now()
- f.startedTailUnindex = true
- f.ptrTailUnindexMap = f.indexedRange.maps.First() - f.indexedRange.tailPartialEpoch
- f.ptrTailUnindexBlock = f.indexedRange.blocks.First() - f.tailPartialBlocks()
- }
- if done, err := f.deleteTailEpoch(epoch); !done {
- return false, err
- }
- f.processEvents()
- if f.stop || !f.targetHeadIndexed() {
- return false, nil
- }
- }
- if f.startedTailUnindex && f.indexedRange.hasIndexedBlocks() {
- log.Info("Log index tail unindexing finished",
- "firstblock", f.indexedRange.blocks.First(), "lastblock", f.indexedRange.blocks.Last(),
- "removedmaps", f.indexedRange.maps.First()-f.ptrTailUnindexMap,
- "removedblocks", f.indexedRange.blocks.First()-f.tailPartialBlocks()-f.ptrTailUnindexBlock,
- "elapsed", common.PrettyDuration(time.Since(f.startedTailUnindexAt)))
- f.startedTailUnindex = false
- }
- return true, nil
-}
-
-// needTailEpoch returns true if the given tail epoch needs to be kept
-// according to the current tail target, false if it can be removed.
-func (f *FilterMaps) needTailEpoch(epoch uint32) bool {
- firstEpoch := f.mapEpoch(f.indexedRange.maps.First())
- if epoch > firstEpoch {
- return true
- }
- if f.firstEpochMap(epoch+1) >= f.indexedRange.maps.AfterLast() {
- return true
- }
- if epoch+1 < firstEpoch {
- return false
- }
- var lastBlockOfPrevEpoch uint64
- if epoch > 0 {
- var err error
- lastBlockOfPrevEpoch, _, err = f.getLastBlockOfMap(f.lastEpochMap(epoch - 1))
- if err != nil {
- log.Error("Could not get last block of previous epoch", "epoch", epoch-1, "error", err)
- return epoch >= firstEpoch
- }
- }
- if f.historyCutoff > lastBlockOfPrevEpoch {
- return false
- }
- lastBlockOfEpoch, _, err := f.getLastBlockOfMap(f.lastEpochMap(epoch))
+ w, err := os.Create(ix.config.ExportFileName)
if err != nil {
- log.Error("Could not get last block of epoch", "epoch", epoch, "error", err)
- return epoch >= firstEpoch
+ log.Error("Error creating checkpoint export file", "name", ix.config.ExportFileName, "error", err)
+ return
}
- return f.tailTargetBlock() <= lastBlockOfEpoch
-}
+ defer w.Close()
-// tailTargetBlock returns the target value for the tail block number according
-// to the log history parameter and the current index head.
-func (f *FilterMaps) tailTargetBlock() uint64 {
- if f.history == 0 || f.indexedView.HeadNumber() < f.history {
- return 0
- }
- return f.indexedView.HeadNumber() + 1 - f.history
-}
-
-// tailPartialBlocks returns the number of rendered blocks in the partially
-// rendered next tail epoch.
-func (f *FilterMaps) tailPartialBlocks() uint64 {
- if f.indexedRange.tailPartialEpoch == 0 {
- return 0
- }
- end, _, err := f.getLastBlockOfMap(f.indexedRange.maps.First() - f.mapsPerEpoch + f.indexedRange.tailPartialEpoch - 1)
- if err != nil {
- log.Error("Error fetching last block of map", "mapIndex", f.indexedRange.maps.First()-f.mapsPerEpoch+f.indexedRange.tailPartialEpoch-1, "error", err)
- }
- var start uint64
- if f.indexedRange.maps.First()-f.mapsPerEpoch > 0 {
- start, _, err = f.getLastBlockOfMap(f.indexedRange.maps.First() - f.mapsPerEpoch - 1)
+ log.Info("Exporting log index checkpoints", "epochs", epochCount, "file", ix.config.ExportFileName)
+ w.WriteString("[\n")
+ comma := ","
+ for epoch := uint32(0); epoch < epochCount; epoch++ {
+ lastBlock, lastBlockId, err := ix.storage.getLastBlockOfMap(ix.storage.params.lastEpochMap(epoch))
if err != nil {
- log.Error("Error fetching last block of map", "mapIndex", f.indexedRange.maps.First()-f.mapsPerEpoch-1, "error", err)
+ log.Error("Error fetching last block of epoch", "epoch", epoch, "error", err)
+ return
}
+ lvPtr, err := ix.storage.getBlockLvPointer(lastBlock)
+ if err != nil {
+ log.Error("Error fetching log value pointer of last block", "block", lastBlock, "error", err)
+ return
+ }
+ if epoch == epochCount-1 {
+ comma = ""
+ }
+ w.WriteString(fmt.Sprintf("{\"blockNumber\": %d, \"blockId\": \"0x%064x\", \"firstIndex\": %d}%s\n", lastBlock, lastBlockId, lvPtr, comma))
}
- return end - start
-}
-
-// targetHeadIndexed returns true if the current log index is consistent with
-// targetView with its head block fully rendered.
-func (f *FilterMaps) targetHeadIndexed() bool {
- return equalViews(f.targetView, f.indexedView) && f.indexedRange.headIndexed
+ w.WriteString("]\n")
+ ix.lastFinalEpoch = epochCount
}
diff --git a/core/filtermaps/indexer_test.go b/core/filtermaps/indexer_test.go
deleted file mode 100644
index 35441905b7..0000000000
--- a/core/filtermaps/indexer_test.go
+++ /dev/null
@@ -1,637 +0,0 @@
-// Copyright 2024 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see .
-
-package filtermaps
-
-import (
- "context"
- crand "crypto/rand"
- "crypto/sha256"
- "encoding/binary"
- "math/big"
- "math/rand"
- "sync"
- "testing"
-
- "github.com/ethereum/go-ethereum/common"
- "github.com/ethereum/go-ethereum/consensus/ethash"
- "github.com/ethereum/go-ethereum/core"
- "github.com/ethereum/go-ethereum/core/rawdb"
- "github.com/ethereum/go-ethereum/core/types"
- "github.com/ethereum/go-ethereum/ethdb"
- "github.com/ethereum/go-ethereum/params"
- "github.com/ethereum/go-ethereum/rlp"
-)
-
-var testParams = Params{
- logMapHeight: 2,
- logMapWidth: 24,
- logMapsPerEpoch: 4,
- logValuesPerMap: 4,
- baseRowGroupSize: 4,
- baseRowLengthRatio: 2,
- logLayerDiff: 2,
-}
-
-func TestIndexerRandomRange(t *testing.T) {
- ts := newTestSetup(t)
- defer ts.close()
-
- forks := make([][]common.Hash, 10)
- ts.chain.addBlocks(1000, 5, 2, 4, false)
- for i := range forks {
- if i != 0 {
- forkBlock := rand.Intn(1000)
- ts.chain.setHead(forkBlock)
- ts.chain.addBlocks(1000-forkBlock, 5, 2, 4, false)
- }
- forks[i] = ts.chain.getCanonicalChain()
- }
- expspos := func(block uint64) uint64 { // expected position of block start
- if block == 0 {
- return 0
- }
- logCount := (block - 1) * 5 * 2
- mapIndex := logCount / 3
- spos := mapIndex*16 + (logCount%3)*5
- if mapIndex == 0 || logCount%3 != 0 {
- spos++
- }
- return spos
- }
- expdpos := func(block uint64) uint64 { // expected position of delimiter
- if block == 0 {
- return 0
- }
- logCount := block * 5 * 2
- mapIndex := (logCount - 1) / 3
- return mapIndex*16 + (logCount-mapIndex*3)*5
- }
- ts.setHistory(0, false)
- var (
- history int
- noHistory bool
- fork, head = len(forks) - 1, 1000
- checkSnapshot bool
- )
- ts.fm.WaitIdle()
- for i := 0; i < 200; i++ {
- switch rand.Intn(3) {
- case 0:
- // change history settings
- switch rand.Intn(10) {
- case 0:
- history, noHistory = 0, false
- case 1:
- history, noHistory = 0, true
- default:
- history, noHistory = rand.Intn(1000)+1, false
- }
- ts.testDisableSnapshots = rand.Intn(2) == 0
- ts.setHistory(uint64(history), noHistory)
- case 1:
- // change head to random position of random fork
- fork, head = rand.Intn(len(forks)), rand.Intn(1001)
- ts.chain.setCanonicalChain(forks[fork][:head+1])
- case 2:
- checkSnapshot = false
- if head < 1000 {
- checkSnapshot = !noHistory && head != 0 // no snapshot generated for block 0
- // add blocks after the current head
- head += rand.Intn(1000-head) + 1
- ts.fm.testSnapshotUsed = false
- ts.chain.setCanonicalChain(forks[fork][:head+1])
- }
- }
- ts.fm.WaitIdle()
- if checkSnapshot {
- if ts.fm.testSnapshotUsed == ts.fm.testDisableSnapshots {
- ts.t.Fatalf("Invalid snapshot used state after head extension (used: %v, disabled: %v)", ts.fm.testSnapshotUsed, ts.fm.testDisableSnapshots)
- }
- checkSnapshot = false
- }
- if noHistory {
- if ts.fm.indexedRange.initialized {
- t.Fatalf("filterMapsRange initialized while indexing is disabled")
- }
- continue
- }
- if !ts.fm.indexedRange.initialized {
- t.Fatalf("filterMapsRange not initialized while indexing is enabled")
- }
- var tailBlock uint64
- if history > 0 && history <= head {
- tailBlock = uint64(head + 1 - history)
- }
- var tailEpoch uint32
- if tailBlock > 0 {
- tailLvPtr := expspos(tailBlock) - 1
- tailEpoch = uint32(tailLvPtr >> (testParams.logValuesPerMap + testParams.logMapsPerEpoch))
- }
- tailLvPtr := uint64(tailEpoch) << (testParams.logValuesPerMap + testParams.logMapsPerEpoch) // first available lv ptr
- var expTailBlock uint64
- if tailEpoch > 0 {
- for expspos(expTailBlock) <= tailLvPtr {
- expTailBlock++
- }
- }
- if ts.fm.indexedRange.blocks.Last() != uint64(head) {
- ts.t.Fatalf("Invalid index head (expected #%d, got #%d)", head, ts.fm.indexedRange.blocks.Last())
- }
- expHeadDelimiter := expdpos(uint64(head))
- if ts.fm.indexedRange.headDelimiter != expHeadDelimiter {
- ts.t.Fatalf("Invalid index head delimiter pointer (expected %d, got %d)", expHeadDelimiter, ts.fm.indexedRange.headDelimiter)
- }
-
- if ts.fm.indexedRange.blocks.First() != expTailBlock {
- ts.t.Fatalf("Invalid index tail block (expected #%d, got #%d)", expTailBlock, ts.fm.indexedRange.blocks.First())
- }
- }
-}
-
-func TestIndexerMatcherView(t *testing.T) {
- testIndexerMatcherView(t, false)
-}
-
-func TestIndexerMatcherViewWithConcurrentRead(t *testing.T) {
- testIndexerMatcherView(t, true)
-}
-
-func testIndexerMatcherView(t *testing.T, concurrentRead bool) {
- ts := newTestSetup(t)
- defer ts.close()
-
- forks := make([][]common.Hash, 20)
- hashes := make([]common.Hash, 20)
- ts.chain.addBlocks(100, 5, 2, 4, true)
- ts.setHistory(0, false)
- for i := range forks {
- if i != 0 {
- ts.chain.setHead(100 - i)
- ts.chain.addBlocks(i, 5, 2, 4, true)
- }
- ts.fm.WaitIdle()
- forks[i] = ts.chain.getCanonicalChain()
- hashes[i] = ts.matcherViewHash()
- }
- fork := len(forks) - 1
- for i := 0; i < 5000; i++ {
- oldFork := fork
- fork = rand.Intn(len(forks))
- stopCh := make(chan chan struct{})
- if concurrentRead {
- go func() {
- for {
- ts.matcherViewHash()
- select {
- case ch := <-stopCh:
- close(ch)
- return
- default:
- }
- }
- }()
- }
- ts.chain.setCanonicalChain(forks[fork])
- ts.fm.WaitIdle()
- if concurrentRead {
- ch := make(chan struct{})
- stopCh <- ch
- <-ch
- }
- hash := ts.matcherViewHash()
- if hash != hashes[fork] {
- t.Fatalf("Matcher view hash mismatch when switching from for %d to %d", oldFork, fork)
- }
- }
-}
-
-func TestLogsByIndex(t *testing.T) {
- ts := newTestSetup(t)
- defer func() {
- ts.fm.testProcessEventsHook = nil
- ts.close()
- }()
-
- ts.chain.addBlocks(1000, 10, 3, 4, true)
- ts.setHistory(0, false)
- ts.fm.WaitIdle()
- firstLog := make([]uint64, 1001) // first valid log position per block
- lastLog := make([]uint64, 1001) // last valid log position per block
- for i := uint64(0); i <= ts.fm.indexedRange.headDelimiter; i++ {
- log, err := ts.fm.getLogByLvIndex(i)
- if err != nil {
- t.Fatalf("Error getting log by index %d: %v", i, err)
- }
- if log != nil {
- if firstLog[log.BlockNumber] == 0 {
- firstLog[log.BlockNumber] = i
- }
- lastLog[log.BlockNumber] = i
- }
- }
- var failed bool
- ts.fm.testProcessEventsHook = func() {
- if ts.fm.indexedRange.blocks.IsEmpty() {
- return
- }
- if lvi := firstLog[ts.fm.indexedRange.blocks.First()]; lvi != 0 {
- log, err := ts.fm.getLogByLvIndex(lvi)
- if log == nil || err != nil {
- t.Errorf("Error getting first log of indexed block range: %v", err)
- failed = true
- }
- }
- if lvi := lastLog[ts.fm.indexedRange.blocks.Last()]; lvi != 0 {
- log, err := ts.fm.getLogByLvIndex(lvi)
- if log == nil || err != nil {
- t.Errorf("Error getting last log of indexed block range: %v", err)
- failed = true
- }
- }
- }
- chain := ts.chain.getCanonicalChain()
- for i := 0; i < 1000 && !failed; i++ {
- head := rand.Intn(len(chain))
- ts.chain.setCanonicalChain(chain[:head+1])
- ts.fm.WaitIdle()
- }
-}
-
-func TestIndexerCompareDb(t *testing.T) {
- ts := newTestSetup(t)
- defer ts.close()
-
- ts.chain.addBlocks(500, 10, 3, 4, true)
- ts.setHistory(0, false)
- ts.fm.WaitIdle()
- // revert points are stored after block 500
- ts.chain.addBlocks(500, 10, 3, 4, true)
- ts.fm.WaitIdle()
- chain1 := ts.chain.getCanonicalChain()
- ts.storeDbHash("chain 1 [0, 1000]")
-
- ts.chain.setHead(600)
- ts.fm.WaitIdle()
- ts.storeDbHash("chain 1/2 [0, 600]")
-
- ts.chain.addBlocks(600, 10, 3, 4, true)
- ts.fm.WaitIdle()
- chain2 := ts.chain.getCanonicalChain()
- ts.storeDbHash("chain 2 [0, 1200]")
-
- ts.chain.setHead(600)
- ts.fm.WaitIdle()
- ts.checkDbHash("chain 1/2 [0, 600]")
-
- ts.setHistory(800, false)
- ts.chain.setCanonicalChain(chain1)
- ts.fm.WaitIdle()
- ts.storeDbHash("chain 1 [201, 1000]")
-
- ts.setHistory(0, false)
- ts.fm.WaitIdle()
- ts.checkDbHash("chain 1 [0, 1000]")
-
- ts.setHistory(800, false)
- ts.chain.setCanonicalChain(chain2)
- ts.fm.WaitIdle()
- ts.storeDbHash("chain 2 [401, 1200]")
-
- ts.setHistory(0, true)
- ts.fm.WaitIdle()
- ts.storeDbHash("no index")
-
- ts.chain.setCanonicalChain(chain2[:501])
- ts.setHistory(0, false)
- ts.fm.WaitIdle()
- ts.chain.setCanonicalChain(chain2)
- ts.fm.WaitIdle()
- ts.checkDbHash("chain 2 [0, 1200]")
-
- ts.chain.setCanonicalChain(chain1)
- ts.fm.WaitIdle()
- ts.setHistory(800, false)
- ts.fm.WaitIdle()
- ts.checkDbHash("chain 1 [201, 1000]")
-
- ts.chain.setCanonicalChain(chain2)
- ts.fm.WaitIdle()
- ts.checkDbHash("chain 2 [401, 1200]")
-
- ts.setHistory(0, true)
- ts.fm.WaitIdle()
- ts.checkDbHash("no index")
-}
-
-type testSetup struct {
- t *testing.T
- fm *FilterMaps
- db ethdb.Database
- chain *testChain
- params Params
- dbHashes map[string]common.Hash
- testDisableSnapshots bool
-}
-
-func newTestSetup(t *testing.T) *testSetup {
- params := testParams
- params.deriveFields()
- ts := &testSetup{
- t: t,
- db: rawdb.NewMemoryDatabase(),
- params: params,
- dbHashes: make(map[string]common.Hash),
- }
- ts.chain = ts.newTestChain()
- return ts
-}
-
-func (ts *testSetup) setHistory(history uint64, noHistory bool) {
- if ts.fm != nil {
- ts.fm.Stop()
- }
- head := ts.chain.CurrentBlock()
- view := NewChainView(ts.chain, head.Number.Uint64(), head.Hash())
- config := Config{
- History: history,
- Disabled: noHistory,
- }
- ts.fm, _ = NewFilterMaps(ts.db, view, 0, 0, ts.params, config)
- ts.fm.testDisableSnapshots = ts.testDisableSnapshots
- ts.fm.Start()
-}
-
-func (ts *testSetup) storeDbHash(id string) {
- dbHash := ts.fmDbHash()
- for otherId, otherHash := range ts.dbHashes {
- if otherHash == dbHash {
- ts.t.Fatalf("Unexpected equal database hashes `%s` and `%s`", id, otherId)
- }
- }
- ts.dbHashes[id] = dbHash
-}
-
-func (ts *testSetup) checkDbHash(id string) {
- if ts.fmDbHash() != ts.dbHashes[id] {
- ts.t.Fatalf("Database `%s` hash mismatch", id)
- }
-}
-
-func (ts *testSetup) fmDbHash() common.Hash {
- hasher := sha256.New()
- it := ts.db.NewIterator(nil, nil)
- for it.Next() {
- hasher.Write(it.Key())
- hasher.Write(it.Value())
- }
- it.Release()
- var result common.Hash
- hasher.Sum(result[:0])
- return result
-}
-
-func (ts *testSetup) matcherViewHash() common.Hash {
- mb := ts.fm.NewMatcherBackend()
- defer mb.Close()
-
- ctx := context.Background()
- params := mb.GetParams()
- hasher := sha256.New()
- var headPtr uint64
- for b := uint64(0); ; b++ {
- lvptr, err := mb.GetBlockLvPointer(ctx, b)
- if err != nil || (b > 0 && lvptr == headPtr) {
- break
- }
- var enc [8]byte
- binary.LittleEndian.PutUint64(enc[:], lvptr)
- hasher.Write(enc[:])
- headPtr = lvptr
- }
- headMap := uint32(headPtr >> params.logValuesPerMap)
- var enc [12]byte
- for r := uint32(0); r < params.mapHeight; r++ {
- binary.LittleEndian.PutUint32(enc[:4], r)
- for m := uint32(0); m <= headMap; m++ {
- binary.LittleEndian.PutUint32(enc[4:8], m)
- rows, _ := mb.GetFilterMapRows(ctx, []uint32{m}, r, false)
- for _, row := range rows {
- for _, v := range row {
- binary.LittleEndian.PutUint32(enc[8:], v)
- hasher.Write(enc[:])
- }
- }
- }
- }
- var hash common.Hash
- hasher.Sum(hash[:0])
- for i := 0; i < 50; i++ {
- hasher.Reset()
- hasher.Write(hash[:])
- lvptr := binary.LittleEndian.Uint64(hash[:8]) % headPtr
- if log, _ := mb.GetLogByLvIndex(ctx, lvptr); log != nil {
- enc, err := rlp.EncodeToBytes(log)
- if err != nil {
- panic(err)
- }
- hasher.Write(enc)
- }
- hasher.Sum(hash[:0])
- }
- return hash
-}
-
-func (ts *testSetup) close() {
- if ts.fm != nil {
- ts.fm.Stop()
- }
- ts.db.Close()
- ts.chain.db.Close()
-}
-
-type testChain struct {
- ts *testSetup
- db ethdb.Database
- lock sync.RWMutex
- canonical []common.Hash
- blocks map[common.Hash]*types.Block
- receipts map[common.Hash]types.Receipts
-}
-
-func (ts *testSetup) newTestChain() *testChain {
- return &testChain{
- ts: ts,
- blocks: make(map[common.Hash]*types.Block),
- receipts: make(map[common.Hash]types.Receipts),
- }
-}
-
-func (tc *testChain) CurrentBlock() *types.Header {
- tc.lock.RLock()
- defer tc.lock.RUnlock()
-
- if len(tc.canonical) == 0 {
- return nil
- }
- return tc.blocks[tc.canonical[len(tc.canonical)-1]].Header()
-}
-
-func (tc *testChain) GetHeader(hash common.Hash, number uint64) *types.Header {
- tc.lock.RLock()
- defer tc.lock.RUnlock()
-
- if block := tc.blocks[hash]; block != nil {
- return block.Header()
- }
- return nil
-}
-
-func (tc *testChain) GetCanonicalHash(number uint64) common.Hash {
- tc.lock.RLock()
- defer tc.lock.RUnlock()
-
- if uint64(len(tc.canonical)) <= number {
- return common.Hash{}
- }
- return tc.canonical[number]
-}
-
-func (tc *testChain) GetReceiptsByHash(hash common.Hash) types.Receipts {
- tc.lock.RLock()
- defer tc.lock.RUnlock()
-
- return tc.receipts[hash]
-}
-
-func (tc *testChain) GetRawReceipts(hash common.Hash, number uint64) types.Receipts {
- tc.lock.RLock()
- defer tc.lock.RUnlock()
-
- return tc.receipts[hash]
-}
-
-func (tc *testChain) addBlocks(count, maxTxPerBlock, maxLogsPerReceipt, maxTopicsPerLog int, random bool) {
- tc.lock.Lock()
- blockGen := func(i int, gen *core.BlockGen) {
- var txCount int
- if random {
- txCount = rand.Intn(maxTxPerBlock + 1)
- } else {
- txCount = maxTxPerBlock
- }
- for k := txCount; k > 0; k-- {
- receipt := types.NewReceipt(nil, false, 0)
- var logCount int
- if random {
- logCount = rand.Intn(maxLogsPerReceipt + 1)
- } else {
- logCount = maxLogsPerReceipt
- }
- receipt.Logs = make([]*types.Log, logCount)
- for i := range receipt.Logs {
- log := &types.Log{}
- receipt.Logs[i] = log
- crand.Read(log.Address[:])
- var topicCount int
- if random {
- topicCount = rand.Intn(maxTopicsPerLog + 1)
- } else {
- topicCount = maxTopicsPerLog
- }
- log.Topics = make([]common.Hash, topicCount)
- for j := range log.Topics {
- crand.Read(log.Topics[j][:])
- }
- }
- gen.AddUncheckedReceipt(receipt)
- gen.AddUncheckedTx(types.NewTransaction(999, common.HexToAddress("0x999"), big.NewInt(999), 999, gen.BaseFee(), nil))
- }
- }
-
- var (
- blocks []*types.Block
- receipts []types.Receipts
- engine = ethash.NewFaker()
- )
-
- if len(tc.canonical) == 0 {
- gspec := &core.Genesis{
- Alloc: types.GenesisAlloc{},
- BaseFee: big.NewInt(params.InitialBaseFee),
- Config: params.TestChainConfig,
- }
- tc.db, blocks, receipts = core.GenerateChainWithGenesis(gspec, engine, count, blockGen)
- gblock := gspec.ToBlock()
- ghash := gblock.Hash()
- tc.canonical = []common.Hash{ghash}
- tc.blocks[ghash] = gblock
- tc.receipts[ghash] = types.Receipts{}
- } else {
- blocks, receipts = core.GenerateChain(params.TestChainConfig, tc.blocks[tc.canonical[len(tc.canonical)-1]], engine, tc.db, count, blockGen)
- }
-
- for i, block := range blocks {
- num, hash := int(block.NumberU64()), block.Hash()
- if len(tc.canonical) != num {
- panic("canonical chain length mismatch")
- }
- tc.canonical = append(tc.canonical, hash)
- tc.blocks[hash] = block
- if receipts[i] != nil {
- tc.receipts[hash] = receipts[i]
- } else {
- tc.receipts[hash] = types.Receipts{}
- }
- }
- tc.lock.Unlock()
- tc.setTargetHead()
-}
-
-func (tc *testChain) setHead(headNum int) {
- tc.lock.Lock()
- tc.canonical = tc.canonical[:headNum+1]
- tc.lock.Unlock()
- tc.setTargetHead()
-}
-
-func (tc *testChain) setTargetHead() {
- head := tc.CurrentBlock()
- if tc.ts.fm != nil {
- if !tc.ts.fm.disabled {
- //tc.ts.fm.targetViewCh <- NewChainView(tc, head.Number.Uint64(), head.Hash())
- tc.ts.fm.SetTarget(NewChainView(tc, head.Number.Uint64(), head.Hash()), 0, 0)
- }
- }
-}
-
-func (tc *testChain) getCanonicalChain() []common.Hash {
- tc.lock.RLock()
- defer tc.lock.RUnlock()
-
- cc := make([]common.Hash, len(tc.canonical))
- copy(cc, tc.canonical)
- return cc
-}
-
-// restore an earlier state of the chain
-func (tc *testChain) setCanonicalChain(cc []common.Hash) {
- tc.lock.Lock()
- tc.canonical = make([]common.Hash, len(cc))
- copy(tc.canonical, cc)
- tc.lock.Unlock()
- tc.setTargetHead()
-}
diff --git a/core/filtermaps/map_database.go b/core/filtermaps/map_database.go
new file mode 100644
index 0000000000..a16a1fda90
--- /dev/null
+++ b/core/filtermaps/map_database.go
@@ -0,0 +1,575 @@
+// Copyright 2025 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package filtermaps
+
+import (
+ "errors"
+ "fmt"
+ "math"
+ "sort"
+ "time"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/common/lru"
+ "github.com/ethereum/go-ethereum/core/rawdb"
+ "github.com/ethereum/go-ethereum/ethdb"
+ "github.com/ethereum/go-ethereum/log"
+)
+
+const (
+ databaseVersion = 4 // reindexed if database version does not match
+ cachedLastBlocks = 1000 // last block of map pointers
+ cachedLvPointers = 1000 // first log value pointer of block pointers
+ maxWritesPerBatch = 1000000
+)
+
+// mapDatabase implements the database storage layer for filter maps and the
+// belonging pointers.
+type mapDatabase struct {
+ params *Params
+ db ethdb.KeyValueStore
+ hashScheme bool
+
+ lastBlockCache *lru.Cache[uint32, lastBlockOfMap]
+ lvPointerCache *lru.Cache[uint64, uint64]
+
+ testReadRows bool
+}
+
+// lastBlockOfMap is used for storing and caching the (number, hash) pairs
+// belonging to the last block of each map.
+type lastBlockOfMap struct {
+ number uint64
+ hash common.Hash
+}
+
+// newMapDatabase created a new mapDatabase layer.
+func newMapDatabase(params *Params, db ethdb.KeyValueStore, hashScheme bool) *mapDatabase {
+ return &mapDatabase{
+ params: params,
+ db: db,
+ hashScheme: hashScheme,
+ lastBlockCache: lru.NewCache[uint32, lastBlockOfMap](cachedLastBlocks),
+ lvPointerCache: lru.NewCache[uint64, uint64](cachedLvPointers),
+ }
+}
+
+// loadMapRange loads the valid and dirty map ranges and known epoch boundaries
+// from the database.
+func (m *mapDatabase) loadMapRange() (valid, dirty rangeSet[uint32], knownEpochs uint32, found bool) {
+ fmr, ok, err := rawdb.ReadFilterMapsRange(m.db)
+ if !ok || err != nil || fmr.Version != databaseVersion {
+ return
+ }
+ return decodeRangeSet32(fmr.ValidMaps), decodeRangeSet32(fmr.DirtyMaps), fmr.KnownEpochs, true
+}
+
+// storeMapRange stores the valid and dirty map ranges and known epoch boundaries
+// into the database.
+func (m *mapDatabase) storeMapRange(valid, dirty rangeSet[uint32], knownEpochs uint32) {
+ rawdb.WriteFilterMapsRange(m.db, rawdb.FilterMapsRange{
+ Version: databaseVersion,
+ ValidMaps: encodeRangeSet32(valid),
+ DirtyMaps: encodeRangeSet32(dirty),
+ KnownEpochs: knownEpochs,
+ })
+}
+
+// deleteMapRange removes the map range entry from the database, signaling a
+// revert to the non-initialized state.
+func (m *mapDatabase) deleteMapRange() {
+ rawdb.DeleteFilterMapsRange(m.db)
+}
+
+func decodeRangeSet32(enc []uint32) rangeSet[uint32] {
+ rs := make(rangeSet[uint32], len(enc)/2)
+ for i := range rs {
+ rs[i] = common.NewRange(enc[i*2], enc[i*2+1])
+ }
+ return rs
+}
+
+func encodeRangeSet32(rs rangeSet[uint32]) []uint32 {
+ enc := make([]uint32, len(rs)*2)
+ for i, r := range rs {
+ enc[i*2] = r.First()
+ enc[i*2+1] = r.Count()
+ }
+ return enc
+}
+
+// getBlockLvPointer returns the starting log value index where the log values
+// generated by the given block are located.
+func (m *mapDatabase) getBlockLvPointer(blockNumber uint64) (uint64, error) {
+ if lvPointer, ok := m.lvPointerCache.Get(blockNumber); ok {
+ return lvPointer, nil
+ }
+ lvPointer, err := rawdb.ReadBlockLvPointer(m.db, blockNumber)
+ if err != nil {
+ return 0, fmt.Errorf("failed to retrieve log value pointer of block %d: %v", blockNumber, err)
+ }
+ m.lvPointerCache.Add(blockNumber, lvPointer)
+ return lvPointer, nil
+}
+
+// storeBlockLvPointer stores the starting log value index where the log values
+// generated by the given block are located.
+func (m *mapDatabase) storeBlockLvPointer(db ethdb.KeyValueWriter, blockNumber, lvPointer uint64) {
+ m.lvPointerCache.Add(blockNumber, lvPointer)
+ rawdb.WriteBlockLvPointer(db, blockNumber, lvPointer)
+}
+
+// getLastBlockOfMap returns the number and hash of the block that generated the
+// last log value entry of the given map.
+func (m *mapDatabase) getLastBlockOfMap(mapIndex uint32) (uint64, common.Hash, error) {
+ if lastBlock, ok := m.lastBlockCache.Get(mapIndex); ok {
+ return lastBlock.number, lastBlock.hash, nil
+ }
+ number, hash, err := rawdb.ReadFilterMapLastBlock(m.db, mapIndex)
+ if err != nil {
+ return 0, common.Hash{}, fmt.Errorf("failed to retrieve last block of map %d: %v", mapIndex, err)
+ }
+ m.lastBlockCache.Add(mapIndex, lastBlockOfMap{number: number, hash: hash})
+ return number, hash, nil
+}
+
+// storeLastBlockOfMap stores the number of the block that generated the last
+// log value entry of the given map.
+func (m *mapDatabase) storeLastBlockOfMap(db ethdb.KeyValueWriter, mapIndex uint32, number uint64, hash common.Hash) {
+ m.lastBlockCache.Add(mapIndex, lastBlockOfMap{number: number, hash: hash})
+ rawdb.WriteFilterMapLastBlock(db, mapIndex, number, hash)
+}
+
+// deleteEpochRows deletes the map rows from an entire epoch.
+func (m *mapDatabase) deleteEpochRows(epoch uint32, stopCallback func() bool) (bool, error) {
+ deleteFn := func(db ethdb.KeyValueStore, hashScheme bool, stopCb func(bool) bool) error {
+ first := m.params.mapRowIndex(m.params.firstEpochMap(epoch), 0)
+ afterLast := m.params.mapRowIndex(m.params.firstEpochMap(epoch+1), 0)
+ return rawdb.DeleteFilterMapRows(db, common.NewRange(first, afterLast-first), hashScheme, stopCb)
+ }
+ action := fmt.Sprintf("Deleting epoch #%d", epoch)
+ switch err := m.safeDeleteWithLogs(deleteFn, action, stopCallback); err {
+ case nil:
+ return true, nil
+ case rawdb.ErrDeleteRangeInterrupted:
+ return false, nil
+ default:
+ return false, err
+ }
+}
+
+// reset deletes the entire log index database. Note that deleteMapRange should
+// be used first in order to prevent trying to use a partially deleted database
+// later.
+func (m *mapDatabase) reset(stopCallback func() bool) (bool, error) {
+ rawdb.DeleteFilterMapsRange(m.db)
+ if err := m.safeDeleteWithLogs(rawdb.DeleteFilterMapsDb, "Resetting log index database", stopCallback); err != rawdb.ErrDeleteRangeInterrupted {
+ return err == nil, err
+ }
+ return false, nil
+}
+
+// deletePointers removes the last block of map and block log value pointers
+// belonging to the specified map range.
+// Note that in order to determine the relevant block range, the last block
+// pointer of the map before the specified map range has to be present unless
+// the range starts at map 0. The last block pointer of the last map in the range
+// also has to be present unless the range ends at MaxUint32-1.
+func (m *mapDatabase) deletePointers(deleteMaps common.Range[uint32], stopCallback func() bool) error {
+ var firstBlock, afterLastBlock uint64
+ if deleteMaps.First() > 0 {
+ lb, _, err := m.getLastBlockOfMap(deleteMaps.First() - 1)
+ if err != nil {
+ return err
+ }
+ firstBlock = lb + 1
+ }
+ if deleteMaps.AfterLast() < math.MaxUint32 {
+ lb, _, err := m.getLastBlockOfMap(deleteMaps.Last())
+ if err != nil {
+ return err
+ }
+ afterLastBlock = lb
+ } else {
+ afterLastBlock = math.MaxUint64
+ }
+ if afterLastBlock > firstBlock {
+ m.lvPointerCache.Purge()
+ blockRange := common.NewRange[uint64](firstBlock, afterLastBlock-firstBlock) // keep last pointer
+ if err := rawdb.DeleteBlockLvPointers(m.db, blockRange, m.hashScheme, func(bool) bool { return stopCallback() }); err != nil {
+ return err
+ }
+ }
+ if deleteMaps.Count() > 1 {
+ m.lastBlockCache.Purge()
+ mapRange := common.NewRange[uint32](deleteMaps.First(), deleteMaps.Count()-1) // keep last pointer
+ if err := rawdb.DeleteFilterMapLastBlocks(m.db, mapRange, m.hashScheme, func(bool) bool { return stopCallback() }); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+// writePointers writes the pointers belonging to the give maps to the database.
+func (m *mapDatabase) writePointers(writeRange common.Range[uint32], maps []*finishedMap, stopCallback func() bool) (bool, error) {
+ batch := m.db.NewBatch()
+ var (
+ batchCnt uint32
+ writeErr error
+ )
+ checkStopOrCommit := func() bool {
+ batchCnt++
+ if batchCnt >= maxWritesPerBatch {
+ if writeErr = batch.Write(); writeErr != nil {
+ return true
+ }
+ if stopCallback() {
+ return true
+ }
+ batch = m.db.NewBatch()
+ batchCnt = 0
+ }
+ return false
+ }
+ for mapIndex := range writeRange.Iter() {
+ fm := maps[mapIndex-writeRange.First()]
+ m.storeLastBlockOfMap(batch, mapIndex, fm.lastBlock.number, fm.lastBlock.hash)
+ if checkStopOrCommit() {
+ return false, writeErr
+ }
+ for i, lvPtr := range fm.blockPtrs {
+ m.storeBlockLvPointer(batch, fm.firstBlock()+uint64(i), lvPtr)
+ if checkStopOrCommit() {
+ return false, writeErr
+ }
+ }
+ }
+ if err := batch.Write(); err != nil {
+ return false, err
+ }
+ return true, nil
+}
+
+// getFilterMapRows returns a batch of filter maps rows from the same row index,
+// each truncated to the length limit of the specified mapping layer.
+// The function assumes that the map indices are in strictly ascending order.
+// Note that the function modifies the mapIndices slice.
+func (m *mapDatabase) getFilterMapRows(mapIndices []uint32, rowIndex, layerIndex uint32) ([]FilterRow, error) {
+ rows := make([]FilterRow, len(mapIndices))
+ readRows := make([]*FilterRow, len(mapIndices))
+ for i := range rows {
+ readRows[i] = &rows[i]
+ }
+ for dbLayer := range min(layerIndex+1, uint32(len(m.params.rowGroupSize))) {
+ if err := m.getFilterMapRowsOfDbLayer(mapIndices, rowIndex, dbLayer, readRows); err != nil {
+ return nil, err
+ }
+ j := 0
+ maxRowLength := m.params.maxRowLength[dbLayer]
+ for i, row := range readRows {
+ if uint32(len(*row)) == maxRowLength {
+ if j != i {
+ mapIndices[j] = mapIndices[i]
+ readRows[j] = readRows[i]
+ }
+ j++
+ }
+ }
+ if j == 0 {
+ break
+ }
+ mapIndices = mapIndices[:j]
+ readRows = readRows[:j]
+ }
+ return rows, nil
+}
+
+// getFilterMapRowsOfDbLayer loads the part of the given filter rows belonging to
+// the specified database layer and appends it to a slice of already existing
+// rows. If dbLayer > 0 then it is assumed that the data from all previous database
+// layers have been appended to the filter rows.
+func (m *mapDatabase) getFilterMapRowsOfDbLayer(mapIndices []uint32, rowIndex, dbLayer uint32, appendTo []*FilterRow) error {
+ var ptr int
+ for ptr < len(mapIndices) {
+ var (
+ groupIndex = m.params.mapGroupIndex(mapIndices[ptr], dbLayer)
+ groupLength = 1
+ )
+ for ptr+groupLength < len(mapIndices) && m.params.mapGroupIndex(mapIndices[ptr+groupLength], dbLayer) == groupIndex {
+ groupLength++
+ }
+ if err := m.getFilterMapRowsOfDbLayerGroup(mapIndices[ptr:ptr+groupLength], rowIndex, dbLayer, appendTo[ptr:ptr+groupLength]); err != nil {
+ return err
+ }
+ ptr += groupLength
+ }
+ return nil
+}
+
+// getFilterMapRowsOfDbLayerGroup loads the part of the given filter rows belonging to
+// the specified database layer and appends it to a slice of already existing
+// rows. The map indices should be located in the same row group according to the
+// group size at the given database layer.
+func (m *mapDatabase) getFilterMapRowsOfDbLayerGroup(mapIndices []uint32, rowIndex, dbLayer uint32, appendTo []*FilterRow) error {
+ var (
+ groupSize = m.params.rowGroupSize[dbLayer]
+ groupIndex = m.params.mapGroupIndex(mapIndices[0], dbLayer)
+ mapRowIndex = m.params.mapRowIndex(groupIndex, rowIndex)
+ )
+ if groupSize == 1 {
+ row, err := rawdb.ReadFilterMapSingleRow(m.db, mapRowIndex, dbLayer, m.params.logMapWidth)
+ if err != nil {
+ return fmt.Errorf("failed to retrieve row %d of row %d layer %d: %v", groupIndex, rowIndex, dbLayer, err)
+ }
+ *appendTo[0] = append(*appendTo[0], FilterRow(row)...)
+ return nil
+ }
+ rows, err := rawdb.ReadFilterMapRowGroup(m.db, mapRowIndex, dbLayer, groupSize, m.params.logMapWidth)
+ if err != nil {
+ return fmt.Errorf("failed to retrieve row group %d of row %d layer %d: %v", groupIndex, rowIndex, dbLayer, err)
+ }
+ for i, mapIndex := range mapIndices {
+ if m.params.mapGroupIndex(mapIndex, dbLayer) != groupIndex {
+ return fmt.Errorf("maps are not in the same base row group, index: %d, group: %d", mapIndex, groupIndex)
+ }
+ *appendTo[i] = append(*appendTo[i], FilterRow(rows[m.params.mapGroupOffset(mapIndex, dbLayer)])...)
+ }
+ return nil
+}
+
+// getFilterMap returns the filter map at the specified index.
+func (m *mapDatabase) getFilterMap(mapIndex uint32) (*finishedMap, error) {
+ fm := new(finishedMap)
+ lastBlock, lbHash, err := m.getLastBlockOfMap(mapIndex)
+ if err != nil {
+ return nil, fmt.Errorf("failed to retrieve last block of map %d: %v", mapIndex, err)
+ }
+ fm.lastBlock = lastBlockOfMap{number: lastBlock, hash: lbHash}
+ var firstBlock uint64
+ if mapIndex > 0 {
+ plb, _, err := m.getLastBlockOfMap(mapIndex - 1)
+ if err != nil {
+ return nil, fmt.Errorf("failed to retrieve last block of map %d: %v", mapIndex-1, err)
+ }
+ firstBlock = plb + 1
+ }
+ fm.blockPtrs = make([]uint64, lastBlock+1-firstBlock)
+ for i := range fm.blockPtrs {
+ blockPtr, err := m.getBlockLvPointer(firstBlock + uint64(i))
+ if err != nil {
+ return nil, fmt.Errorf("failed to retrieve block pointer %d: %v", firstBlock+uint64(i), err)
+ }
+ fm.blockPtrs[i] = blockPtr
+ }
+ for rowIndex := range m.params.mapHeight {
+ var mi [1]uint32
+ mi[0] = mapIndex
+ row, err := m.getFilterMapRows(mi[:], rowIndex, uint32(len(m.params.rowGroupSize)-1))
+ if err != nil {
+ return nil, fmt.Errorf("failed to retrieve row %d of map %d: %v", rowIndex, mapIndex, err)
+ }
+ fm.rowData = append(fm.rowData, row[0]...)
+ fm.rowPtrs = append(fm.rowPtrs, uint16(len(fm.rowData)))
+ }
+ return fm, nil
+}
+
+// writeMapRows updates a section of the filter map range, writing a continuous
+// range of new map data and/or removing dirty data. writeRange and deleteRange
+// can and should overlap if dirty data is being overwritten with new map data.
+// writeRange alone should only be specified where the database is known to be
+// empty.
+// keepEmptyRange is optional, it signals that a certain range is known to be
+// empty in the database and should also stay so. It should not overlap with the
+// other two ranges. If these three ranges together cover an entire row group
+// then the entire group can be updated without reading its previous value,
+// thereby speeding up the write process significantly.
+// Note that writeMapRows only updates the filter map rows and does not touch
+// the belonging pointers.
+func (m *mapDatabase) writeMapRows(writeRange, deleteRange, keepEmptyRange common.Range[uint32], maps []*finishedMap, stopCallback func() bool) (bool, error) {
+ if !writeRange.Intersection(keepEmptyRange).IsEmpty() || !deleteRange.Intersection(keepEmptyRange).IsEmpty() {
+ panic("invalid writeMapRows map ranges")
+ }
+ writePattern := m.makeWritePattern(writeRange, deleteRange, keepEmptyRange)
+ batch := m.db.NewBatch()
+ rowsPerBatch := uint32(max(maxWritesPerBatch/len(writePattern), 1))
+ var (
+ batchCnt uint32
+ writeErr error
+ )
+ checkStopOrCommit := func() bool {
+ batchCnt++
+ if batchCnt >= rowsPerBatch {
+ if writeErr = batch.Write(); writeErr != nil {
+ return true
+ }
+ if stopCallback() {
+ return true
+ }
+ batch = m.db.NewBatch()
+ batchCnt = 0
+ }
+ return false
+ }
+
+ for rowIndex := range m.params.mapHeight {
+ if err := m.writeRowUpdates(batch, writePattern, writeRange, maps, rowIndex); err != nil {
+ return false, err
+ }
+ if checkStopOrCommit() {
+ return false, writeErr
+ }
+ }
+ if err := batch.Write(); err != nil {
+ return false, err
+ }
+ return true, nil
+}
+
+type writePatterItem struct {
+ mapIndex, dbLayer uint32
+ writeRange, deleteRange, keepEmptyRange common.Range[uint32]
+}
+
+// makeWritePattern pre-generates the order of operations that need to be repeated
+// for every row.
+func (m *mapDatabase) makeWritePattern(writeRange, deleteRange, keepEmptyRange common.Range[uint32]) (writePattern []writePatterItem) {
+ for dbLayer, groupSize := range m.params.rowGroupSize {
+ updateRange := deleteRange.Union(writeRange)
+ firstGroup, lastGroup := updateRange.First()/groupSize, updateRange.Last()/groupSize
+ for i := firstGroup; i <= lastGroup; i++ {
+ groupRange := common.NewRange[uint32](i*groupSize, groupSize)
+ writePattern = append(writePattern, writePatterItem{
+ mapIndex: i * groupSize,
+ dbLayer: uint32(dbLayer),
+ writeRange: writeRange.Intersection(groupRange),
+ deleteRange: deleteRange.Intersection(groupRange),
+ keepEmptyRange: keepEmptyRange.Intersection(groupRange),
+ })
+ }
+ }
+ sort.Slice(writePattern, func(i, j int) bool {
+ return writePattern[i].mapIndex < writePattern[j].mapIndex ||
+ (writePattern[i].mapIndex == writePattern[j].mapIndex && writePattern[i].dbLayer < writePattern[j].dbLayer)
+ })
+ return
+}
+
+// writeRowUpdates performs the filter row update operation on a single row.
+func (m *mapDatabase) writeRowUpdates(batch ethdb.Batch, writePattern []writePatterItem, writeRange common.Range[uint32], maps []*finishedMap, rowIndex uint32) error {
+ for _, w := range writePattern {
+ if groupSize := m.params.rowGroupSize[w.dbLayer]; groupSize == 1 {
+ var row FilterRow
+ if writeRange.Includes(w.mapIndex) {
+ row = maps[w.mapIndex-writeRange.First()].getRow(rowIndex, m.params.maxRowLength[w.dbLayer])
+ }
+ var from uint32
+ if w.dbLayer > 0 {
+ from = m.params.maxRowLength[w.dbLayer-1]
+ }
+ if uint32(len(row)) > from {
+ row = row[from:]
+ } else {
+ row = nil
+ }
+ if len(row) > 0 || !w.deleteRange.IsEmpty() {
+ rawdb.WriteFilterMapSingleRow(batch, m.params.mapRowIndex(w.mapIndex, rowIndex), w.dbLayer, row, m.params.logMapWidth)
+ }
+ } else {
+ rows := make([][]uint32, groupSize)
+ writeGroup := !w.deleteRange.IsEmpty()
+ if w.writeRange.Count()+w.deleteRange.Count()-w.writeRange.Intersection(w.deleteRange).Count()+w.keepEmptyRange.Count() < groupSize {
+ oldRows, err := rawdb.ReadFilterMapRowGroup(m.db, m.params.mapRowIndex(w.mapIndex, rowIndex), w.dbLayer, groupSize, m.params.logMapWidth)
+ m.testReadRows = true
+ if err != nil {
+ return err
+ }
+ for i := range groupSize {
+ if mapIndex := w.mapIndex + i; !w.writeRange.Includes(mapIndex) && !w.deleteRange.Includes(mapIndex) && !w.keepEmptyRange.Includes(mapIndex) {
+ rows[i] = oldRows[i]
+ }
+ }
+ }
+ var from uint32
+ if w.dbLayer > 0 {
+ from = m.params.maxRowLength[w.dbLayer-1]
+ }
+ to := m.params.maxRowLength[w.dbLayer]
+ for i := range groupSize {
+ if writeRange.Includes(w.mapIndex + i) {
+ if row := maps[w.mapIndex+i-writeRange.First()].getRow(rowIndex, to); uint32(len(row)) > from {
+ rows[i] = row[from:]
+ writeGroup = true
+ }
+ }
+ }
+ if writeGroup {
+ rawdb.WriteFilterMapRowGroup(batch, m.params.mapRowIndex(w.mapIndex, rowIndex), w.dbLayer, rows, m.params.logMapWidth)
+ }
+ }
+ }
+ return nil
+}
+
+// safeDeleteWithLogs is a wrapper for a function that performs a safe range
+// delete operation using rawdb.SafeDeleteRange. It emits log messages if the
+// process takes long enough to call the stop callback.
+func (m *mapDatabase) safeDeleteWithLogs(deleteFn func(db ethdb.KeyValueStore, hashScheme bool, stopCb func(bool) bool) error, action string, stopCb func() bool) error {
+ var (
+ start = time.Now()
+ logPrinted bool
+ lastLogPrinted = start
+ )
+ switch err := deleteFn(m.db, m.hashScheme, func(deleted bool) bool {
+ if deleted && !logPrinted || time.Since(lastLogPrinted) > time.Second*10 {
+ log.Info(action+" in progress...", "elapsed", common.PrettyDuration(time.Since(start)))
+ logPrinted, lastLogPrinted = true, time.Now()
+ }
+ return stopCb()
+ }); {
+ case err == nil:
+ if logPrinted {
+ log.Info(action+" finished", "elapsed", common.PrettyDuration(time.Since(start)))
+ }
+ return nil
+ case errors.Is(err, rawdb.ErrDeleteRangeInterrupted):
+ log.Warn(action+" interrupted", "elapsed", common.PrettyDuration(time.Since(start)))
+ return err
+ default:
+ log.Error(action+" failed", "error", err)
+ return err
+ }
+}
+
+// removeBloomBits removes old bloom bits data from the database.
+func (m *mapDatabase) removeBloomBits(stopCb func() bool) {
+ m.safeDeleteWithLogs(rawdb.DeleteBloomBitsDb, "Removing old bloom bits database", stopCb)
+}
+
+// storeEpochCheckpoint writes a single checkpoint (known epoch boundary) to the
+// database.
+func (m *mapDatabase) storeEpochCheckpoint(epoch uint32, cp epochCheckpoint) {
+ m.storeLastBlockOfMap(m.db, m.params.lastEpochMap(epoch), cp.BlockNumber, cp.BlockHash)
+ m.storeBlockLvPointer(m.db, cp.BlockNumber, cp.FirstIndex)
+}
+
+// storeCheckpointList writes a list of checkpoints to the database.
+func (m *mapDatabase) storeCheckpointList(firstEpoch uint32, cpList checkpointList) {
+ for i, cp := range cpList {
+ m.storeEpochCheckpoint(firstEpoch+uint32(i), cp)
+ }
+}
diff --git a/core/filtermaps/map_database_test.go b/core/filtermaps/map_database_test.go
new file mode 100644
index 0000000000..e0ce431769
--- /dev/null
+++ b/core/filtermaps/map_database_test.go
@@ -0,0 +1,79 @@
+// Copyright 2025 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package filtermaps
+
+import (
+ "math"
+ "testing"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/ethdb/memorydb"
+)
+
+var testParams = Params{
+ logMapHeight: 8,
+ logMapWidth: 24,
+ logMapsPerEpoch: 6,
+ logValuesPerMap: 8,
+ logMappingFrequency: []uint{6, 4, 2, 0},
+ maxRowLength: []uint32{4, 16, 64, 256},
+ rowGroupSize: []uint32{16, 4, 1, 1},
+}
+
+func testStop() bool { return false }
+
+func TestMapDatabase(t *testing.T) {
+ params := testParams
+ params.sanitize()
+ db := memorydb.New()
+ mapDb := newMapDatabase(¶ms, db, false)
+ reader := mapReader{
+ getFilterMapRows: mapDb.getFilterMapRows,
+ getFilterMap: mapDb.getFilterMap,
+ getBlockLvPointer: mapDb.getBlockLvPointer,
+ getLastBlockOfMap: mapDb.getLastBlockOfMap,
+ }
+ // initialize database with checkpoints
+ maps := generateTestMaps(¶ms, nil, 0x200)
+ cpList := generateTestCheckpoints(¶ms, maps)
+ mapDb.storeCheckpointList(0, cpList)
+ // add new maps to the head
+ maps = generateTestMaps(¶ms, maps, 0x50)
+ mapDb.writeMapRows(common.NewRange[uint32](0x200, 0x50), common.Range[uint32]{}, common.NewRange[uint32](0x250, math.MaxUint32-0x250), maps[0x200:], testStop)
+ mapDb.writePointers(common.NewRange[uint32](0x200, 0x50), maps[0x200:], testStop)
+ testMapReader(t, "mapDatabase test #1", ¶ms, reader, cpList, maps[0x200:])
+ // backfill previous epoch with a single write
+ mapDb.writeMapRows(common.NewRange[uint32](0x1c0, 0x40), common.Range[uint32]{}, common.Range[uint32]{}, maps[0x1c0:0x200], testStop)
+ mapDb.writePointers(common.NewRange[uint32](0x1c0, 0x40), maps[0x1c0:0x200], testStop)
+ testMapReader(t, "mapDatabase test #2", ¶ms, reader, cpList[:7], maps[0x1c0:])
+ // backfill previous epoch in two steps
+ mapDb.writeMapRows(common.NewRange[uint32](0x180, 0x10), common.Range[uint32]{}, common.NewRange[uint32](0x190, 0x30), maps[0x180:0x190], testStop)
+ mapDb.writePointers(common.NewRange[uint32](0x180, 0x10), maps[0x180:0x190], testStop)
+ mapDb.writeMapRows(common.NewRange[uint32](0x190, 0x30), common.Range[uint32]{}, common.Range[uint32]{}, maps[0x190:0x1c0], testStop)
+ mapDb.writePointers(common.NewRange[uint32](0x190, 0x30), maps[0x190:0x1c0], testStop)
+ testMapReader(t, "mapDatabase test #3", ¶ms, reader, cpList[:6], maps[0x180:])
+ // add new maps while reorging some existing ones
+ maps = generateTestMaps(¶ms, maps[:0x230], 0x30)
+ mapDb.writeMapRows(common.NewRange[uint32](0x230, 0x30), common.NewRange[uint32](0x230, 0x30), common.NewRange[uint32](0x260, math.MaxUint32-0x260), maps[0x230:], testStop)
+ mapDb.deletePointers(common.NewRange[uint32](0x230, math.MaxUint32-0x230), testStop)
+ mapDb.writePointers(common.NewRange[uint32](0x230, 0x30), maps[0x230:], testStop)
+ testMapReader(t, "mapDatabase test #4", ¶ms, reader, cpList[:6], maps[0x180:])
+ // unindex tail epoch
+ mapDb.writeMapRows(common.Range[uint32]{}, common.NewRange[uint32](0x180, 0x40), common.Range[uint32]{}, nil, testStop)
+ mapDb.deletePointers(common.NewRange[uint32](0x180, 0x40), testStop)
+ testMapReader(t, "mapDatabase test #5", ¶ms, reader, cpList[:7], maps[0x1c0:])
+}
diff --git a/core/filtermaps/map_renderer.go b/core/filtermaps/map_renderer.go
deleted file mode 100644
index e1284a3829..0000000000
--- a/core/filtermaps/map_renderer.go
+++ /dev/null
@@ -1,815 +0,0 @@
-// Copyright 2024 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see .
-
-package filtermaps
-
-import (
- "errors"
- "fmt"
- "math"
- "slices"
- "sort"
- "time"
-
- "github.com/ethereum/go-ethereum/common"
- "github.com/ethereum/go-ethereum/common/lru"
- "github.com/ethereum/go-ethereum/core/types"
- "github.com/ethereum/go-ethereum/log"
-)
-
-const (
- maxMapsPerBatch = 32 // maximum number of maps rendered in memory
- valuesPerCallback = 1024 // log values processed per event process callback
- cachedRowMappings = 10000 // log value to row mappings cached during rendering
-
- // Number of rows written to db in a single batch.
- // The map renderer splits up writes like this to ensure that regular
- // block processing latency is not affected by large batch writes.
- rowsPerBatch = 1024
-)
-
-var (
- errChainUpdate = errors.New("rendered section of chain updated")
-)
-
-// mapRenderer represents a process that renders filter maps in a specified
-// range according to the actual targetView.
-type mapRenderer struct {
- f *FilterMaps
- renderBefore uint32
- currentMap *renderedMap
- finishedMaps map[uint32]*renderedMap
- finished common.Range[uint32]
- iterator *logIterator
-}
-
-// renderedMap represents a single filter map that is being rendered in memory.
-type renderedMap struct {
- filterMap filterMap
- mapIndex uint32
- lastBlock uint64
- lastBlockId common.Hash
- blockLvPtrs []uint64 // start pointers of blocks starting in this map; last one is lastBlock
- finished bool // iterator finished; all values rendered
- headDelimiter uint64 // if finished then points to the future block delimiter of the head block
-}
-
-// firstBlock returns the first block number that starts in the given map.
-func (r *renderedMap) firstBlock() uint64 {
- return r.lastBlock + 1 - uint64(len(r.blockLvPtrs))
-}
-
-// renderMapsBefore creates a mapRenderer that renders the log index until the
-// specified map index boundary, starting from the latest available starting
-// point that is consistent with the current targetView.
-// The renderer ensures that filterMapsRange, indexedView and the actual map
-// data are always consistent with each other. If renderBefore is greater than
-// the latest existing rendered map then indexedView is updated to targetView,
-// otherwise it is checked that the rendered range is consistent with both
-// views.
-func (f *FilterMaps) renderMapsBefore(renderBefore uint32) (*mapRenderer, error) {
- nextMap, startBlock, startLvPtr, err := f.lastCanonicalMapBoundaryBefore(renderBefore)
- if err != nil {
- return nil, err
- }
- if snapshot := f.lastCanonicalSnapshotOfMap(nextMap); snapshot != nil {
- return f.renderMapsFromSnapshot(snapshot)
- }
- if nextMap >= renderBefore {
- return nil, nil
- }
- return f.renderMapsFromMapBoundary(nextMap, renderBefore, startBlock, startLvPtr)
-}
-
-// renderMapsFromSnapshot creates a mapRenderer that starts rendering from a
-// snapshot made at a block boundary.
-func (f *FilterMaps) renderMapsFromSnapshot(cp *renderedMap) (*mapRenderer, error) {
- f.testSnapshotUsed = true
- iter, err := f.newLogIteratorFromBlockDelimiter(cp.lastBlock, cp.headDelimiter)
- if err != nil {
- return nil, fmt.Errorf("failed to create log iterator from block delimiter %d: %v", cp.lastBlock, err)
- }
- return &mapRenderer{
- f: f,
- currentMap: &renderedMap{
- filterMap: cp.filterMap.fullCopy(),
- mapIndex: cp.mapIndex,
- lastBlock: cp.lastBlock,
- blockLvPtrs: slices.Clone(cp.blockLvPtrs),
- },
- finishedMaps: make(map[uint32]*renderedMap),
- finished: common.NewRange(cp.mapIndex, 0),
- renderBefore: math.MaxUint32,
- iterator: iter,
- }, nil
-}
-
-// renderMapsFromMapBoundary creates a mapRenderer that starts rendering at a
-// map boundary.
-func (f *FilterMaps) renderMapsFromMapBoundary(firstMap, renderBefore uint32, startBlock, startLvPtr uint64) (*mapRenderer, error) {
- iter, err := f.newLogIteratorFromMapBoundary(firstMap, startBlock, startLvPtr)
- if err != nil {
- return nil, fmt.Errorf("failed to create log iterator from map boundary %d: %v", firstMap, err)
- }
- return &mapRenderer{
- f: f,
- currentMap: &renderedMap{
- filterMap: f.emptyFilterMap(),
- mapIndex: firstMap,
- lastBlock: iter.blockNumber,
- },
- finishedMaps: make(map[uint32]*renderedMap),
- finished: common.NewRange(firstMap, 0),
- renderBefore: renderBefore,
- iterator: iter,
- }, nil
-}
-
-// lastCanonicalSnapshotOfMap returns the latest cached snapshot of the given map
-// that is also consistent with the current targetView.
-func (f *FilterMaps) lastCanonicalSnapshotOfMap(mapIndex uint32) *renderedMap {
- var best *renderedMap
- for _, blockNumber := range f.renderSnapshots.Keys() {
- if cp, _ := f.renderSnapshots.Get(blockNumber); cp != nil && blockNumber < f.indexedRange.blocks.AfterLast() &&
- blockNumber <= f.indexedView.HeadNumber() && f.indexedView.BlockId(blockNumber) == cp.lastBlockId &&
- blockNumber <= f.targetView.HeadNumber() && f.targetView.BlockId(blockNumber) == cp.lastBlockId &&
- cp.mapIndex == mapIndex && (best == nil || blockNumber > best.lastBlock) {
- best = cp
- }
- }
- return best
-}
-
-// lastCanonicalMapBoundaryBefore returns the latest map boundary before the
-// specified map index that matches the current targetView. This can either
-// be a checkpoint (hardcoded or left from a previously unindexed tail epoch)
-// or the boundary of a currently rendered map.
-// Along with the next map index where the rendering can be started, the number
-// and starting log value pointer of the last block is also returned.
-func (f *FilterMaps) lastCanonicalMapBoundaryBefore(renderBefore uint32) (nextMap uint32, startBlock, startLvPtr uint64, err error) {
- if !f.indexedRange.initialized {
- return 0, 0, 0, nil
- }
- mapIndex := renderBefore
- for {
- var ok bool
- if mapIndex, ok = f.lastMapBoundaryBefore(mapIndex); !ok {
- return 0, 0, 0, nil
- }
- lastBlock, lastBlockId, err := f.getLastBlockOfMap(mapIndex)
- if err != nil {
- return 0, 0, 0, fmt.Errorf("failed to retrieve last block of reverse iterated map %d: %v", mapIndex, err)
- }
- if (f.indexedRange.headIndexed && mapIndex >= f.indexedRange.maps.Last()) ||
- lastBlock >= f.targetView.HeadNumber() || lastBlockId != f.targetView.BlockId(lastBlock) {
- continue // map is not full or inconsistent with targetView; roll back
- }
- lvPtr, err := f.getBlockLvPointer(lastBlock)
- if err != nil {
- return 0, 0, 0, fmt.Errorf("failed to retrieve log value pointer of last canonical boundary block %d: %v", lastBlock, err)
- }
- return mapIndex + 1, lastBlock, lvPtr, nil
- }
-}
-
-// lastMapBoundaryBefore returns the latest map boundary before the specified
-// map index.
-func (f *FilterMaps) lastMapBoundaryBefore(renderBefore uint32) (uint32, bool) {
- if !f.indexedRange.initialized || f.indexedRange.maps.AfterLast() == 0 || renderBefore == 0 {
- return 0, false
- }
- afterLastFullMap := f.indexedRange.maps.AfterLast()
- if afterLastFullMap > 0 && f.indexedRange.headIndexed {
- afterLastFullMap-- // last map is not full
- }
- firstRendered := min(renderBefore-1, afterLastFullMap)
- if firstRendered == 0 {
- return 0, false
- }
- if firstRendered >= f.indexedRange.maps.First() {
- return firstRendered - 1, true
- }
- if firstRendered+f.mapsPerEpoch > f.indexedRange.maps.First() {
- firstRendered = min(firstRendered, f.indexedRange.maps.First()-f.mapsPerEpoch+f.indexedRange.tailPartialEpoch)
- } else {
- firstRendered = (firstRendered >> f.logMapsPerEpoch) << f.logMapsPerEpoch
- }
- if firstRendered == 0 {
- return 0, false
- }
- return firstRendered - 1, true
-}
-
-// emptyFilterMap returns an empty filter map.
-func (f *FilterMaps) emptyFilterMap() filterMap {
- return make(filterMap, f.mapHeight)
-}
-
-// loadHeadSnapshot loads the last rendered map from the database and creates
-// a snapshot.
-func (f *FilterMaps) loadHeadSnapshot() error {
- fm, err := f.getFilterMap(f.indexedRange.maps.Last())
- if err != nil {
- return fmt.Errorf("failed to load head snapshot map %d: %v", f.indexedRange.maps.Last(), err)
- }
- lastBlock, _, err := f.getLastBlockOfMap(f.indexedRange.maps.Last())
- if err != nil {
- return fmt.Errorf("failed to retrieve last block of head snapshot map %d: %v", f.indexedRange.maps.Last(), err)
- }
- var firstBlock uint64
- if f.indexedRange.maps.AfterLast() > 1 {
- prevLastBlock, _, err := f.getLastBlockOfMap(f.indexedRange.maps.Last() - 1)
- if err != nil {
- return fmt.Errorf("failed to retrieve last block of map %d before head snapshot: %v", f.indexedRange.maps.Last()-1, err)
- }
- firstBlock = prevLastBlock + 1
- }
- lvPtrs := make([]uint64, lastBlock+1-firstBlock)
- for i := range lvPtrs {
- lvPtrs[i], err = f.getBlockLvPointer(firstBlock + uint64(i))
- if err != nil {
- return fmt.Errorf("failed to retrieve log value pointer of head snapshot block %d: %v", firstBlock+uint64(i), err)
- }
- }
- f.renderSnapshots.Add(f.indexedRange.blocks.Last(), &renderedMap{
- filterMap: fm.fullCopy(),
- mapIndex: f.indexedRange.maps.Last(),
- lastBlock: f.indexedRange.blocks.Last(),
- lastBlockId: f.indexedView.BlockId(f.indexedRange.blocks.Last()),
- blockLvPtrs: lvPtrs,
- finished: true,
- headDelimiter: f.indexedRange.headDelimiter,
- })
- return nil
-}
-
-// makeSnapshot creates a snapshot of the current state of the rendered map.
-func (r *mapRenderer) makeSnapshot() {
- if r.iterator.blockNumber != r.currentMap.lastBlock || r.iterator.chainView != r.f.targetView {
- panic("iterator state inconsistent with current rendered map")
- }
- r.f.renderSnapshots.Add(r.currentMap.lastBlock, &renderedMap{
- filterMap: r.currentMap.filterMap.fastCopy(),
- mapIndex: r.currentMap.mapIndex,
- lastBlock: r.currentMap.lastBlock,
- lastBlockId: r.iterator.chainView.BlockId(r.currentMap.lastBlock),
- blockLvPtrs: r.currentMap.blockLvPtrs,
- finished: true,
- headDelimiter: r.iterator.lvIndex,
- })
-}
-
-// run does the actual map rendering. It periodically calls the stopCb callback
-// and if it returns true the process is interrupted an can be resumed later
-// by calling run again. The writeCb callback is called after new maps have
-// been written to disk and the index range has been updated accordingly.
-func (r *mapRenderer) run(stopCb func() bool, writeCb func()) (bool, error) {
- for {
- if done, err := r.renderCurrentMap(stopCb); !done {
- return done, err // stopped or failed
- }
- // map finished
- r.finishedMaps[r.currentMap.mapIndex] = r.currentMap
- r.finished.SetLast(r.finished.AfterLast())
- if len(r.finishedMaps) >= maxMapsPerBatch || r.f.mapGroupOffset(r.finished.AfterLast()) == 0 {
- if err := r.writeFinishedMaps(stopCb); err != nil {
- return false, err
- }
- writeCb()
- }
- if r.finished.AfterLast() == r.renderBefore || r.iterator.finished {
- if err := r.writeFinishedMaps(stopCb); err != nil {
- return false, err
- }
- writeCb()
- return true, nil
- }
- r.currentMap = &renderedMap{
- filterMap: r.f.emptyFilterMap(),
- mapIndex: r.finished.AfterLast(),
- }
- }
-}
-
-// renderCurrentMap renders a single map.
-func (r *mapRenderer) renderCurrentMap(stopCb func() bool) (bool, error) {
- var (
- totalTime time.Duration
- logValuesProcessed, blocksProcessed int64
- )
- start := time.Now()
- if !r.iterator.updateChainView(r.f.targetView) {
- return false, errChainUpdate
- }
- var waitCnt int
-
- if r.iterator.lvIndex == 0 {
- r.currentMap.blockLvPtrs = []uint64{0}
- }
- type lvPos struct{ rowIndex, layerIndex uint32 }
- rowMappingCache := lru.NewCache[common.Hash, lvPos](cachedRowMappings)
- defer rowMappingCache.Purge()
-
- for r.iterator.lvIndex < uint64(r.currentMap.mapIndex+1)<= valuesPerCallback {
- totalTime += time.Since(start)
- if stopCb() {
- return false, nil
- }
- start = time.Now()
- if !r.iterator.updateChainView(r.f.targetView) {
- return false, errChainUpdate
- }
- waitCnt = 0
- }
- if logValue := r.iterator.getValueHash(); logValue != (common.Hash{}) {
- lvp, cached := rowMappingCache.Get(logValue)
- if !cached {
- lvp = lvPos{rowIndex: r.f.rowIndex(r.currentMap.mapIndex, 0, logValue)}
- }
- for uint32(len(r.currentMap.filterMap[lvp.rowIndex])) >= r.f.maxRowLength(lvp.layerIndex) {
- lvp.layerIndex++
- lvp.rowIndex = r.f.rowIndex(r.currentMap.mapIndex, lvp.layerIndex, logValue)
- cached = false
- }
- r.currentMap.filterMap[lvp.rowIndex] = append(r.currentMap.filterMap[lvp.rowIndex], r.f.columnIndex(r.iterator.lvIndex, &logValue))
- if !cached {
- rowMappingCache.Add(logValue, lvp)
- }
- }
- if err := r.iterator.next(); err != nil {
- return false, fmt.Errorf("failed to advance log iterator at %d while rendering map %d: %v", r.iterator.lvIndex, r.currentMap.mapIndex, err)
- }
- if !r.iterator.skipToBoundary {
- logValuesProcessed++
- r.currentMap.lastBlock = r.iterator.blockNumber
- if r.iterator.blockStart {
- blocksProcessed++
- r.currentMap.blockLvPtrs = append(r.currentMap.blockLvPtrs, r.iterator.lvIndex)
- }
- if !r.f.testDisableSnapshots && r.renderBefore >= r.f.indexedRange.maps.AfterLast() &&
- (r.iterator.delimiter || r.iterator.finished) {
- r.makeSnapshot()
- }
- }
- }
- if r.iterator.finished {
- r.currentMap.finished = true
- r.currentMap.headDelimiter = r.iterator.lvIndex
- }
- r.currentMap.lastBlockId = r.f.targetView.BlockId(r.currentMap.lastBlock)
- totalTime += time.Since(start)
- mapRenderTimer.Update(totalTime)
- mapLogValueMeter.Mark(logValuesProcessed)
- mapBlockMeter.Mark(blocksProcessed)
- return true, nil
-}
-
-// writeFinishedMaps writes rendered maps to the database and updates
-// filterMapsRange and indexedView accordingly.
-func (r *mapRenderer) writeFinishedMaps(pauseCb func() bool) error {
- var totalTime time.Duration
- start := time.Now()
- if len(r.finishedMaps) == 0 {
- return nil
- }
- r.f.indexLock.Lock()
- defer r.f.indexLock.Unlock()
-
- oldRange := r.f.indexedRange
- tempRange, err := r.getTempRange()
- if err != nil {
- return fmt.Errorf("failed to get temporary rendered range: %v", err)
- }
- newRange, err := r.getUpdatedRange()
- if err != nil {
- return fmt.Errorf("failed to get updated rendered range: %v", err)
- }
- renderedView := r.f.targetView // pauseCb callback might still change targetView while writing finished maps
-
- batch := r.f.db.NewBatch()
- var writeCnt int
- checkWriteCnt := func() {
- writeCnt++
- if writeCnt == rowsPerBatch {
- writeCnt = 0
- if err := batch.Write(); err != nil {
- log.Crit("Error writing log index update batch", "error", err)
- }
- // do not exit while in partially written state but do allow processing
- // events and pausing while block processing is in progress
- r.f.indexLock.Unlock()
- totalTime += time.Since(start)
- pauseCb()
- start = time.Now()
- r.f.indexLock.Lock()
- batch = r.f.db.NewBatch()
- }
- }
-
- if tempRange != r.f.indexedRange {
- r.f.setRange(batch, r.f.indexedView, tempRange, true)
- }
- // add or update filter rows
- for rowIndex := uint32(0); rowIndex < r.f.mapHeight; rowIndex++ {
- var (
- mapIndices []uint32
- rows []FilterRow
- )
- for mapIndex := range r.finished.Iter() {
- row := r.finishedMaps[mapIndex].filterMap[rowIndex]
- if fm, _ := r.f.filterMapCache.Get(mapIndex); fm != nil && row.Equal(fm[rowIndex]) {
- continue
- }
- mapIndices = append(mapIndices, mapIndex)
- rows = append(rows, row)
- }
- if newRange.maps.AfterLast() == r.finished.AfterLast() { // head updated; remove future entries
- for mapIndex := r.finished.AfterLast(); mapIndex < oldRange.maps.AfterLast(); mapIndex++ {
- if fm, _ := r.f.filterMapCache.Get(mapIndex); fm != nil && len(fm[rowIndex]) == 0 {
- continue
- }
- mapIndices = append(mapIndices, mapIndex)
- rows = append(rows, nil)
- }
- }
- if err := r.f.storeFilterMapRows(batch, mapIndices, rowIndex, rows); err != nil {
- return fmt.Errorf("failed to store filter maps %v row %d: %v", mapIndices, rowIndex, err)
- }
- checkWriteCnt()
- }
- // update filter map cache
- if newRange.maps.AfterLast() == r.finished.AfterLast() {
- // head updated; cache new head maps and remove future entries
- for mapIndex := range r.finished.Iter() {
- r.f.filterMapCache.Add(mapIndex, r.finishedMaps[mapIndex].filterMap)
- }
- for mapIndex := r.finished.AfterLast(); mapIndex < oldRange.maps.AfterLast(); mapIndex++ {
- r.f.filterMapCache.Remove(mapIndex)
- }
- } else {
- // head not updated; do not cache maps during tail rendering because we
- // need head maps to be available in the cache
- for mapIndex := range r.finished.Iter() {
- r.f.filterMapCache.Remove(mapIndex)
- }
- }
- var blockNumber uint64
- if r.finished.First() > 0 {
- // in order to always ensure continuous block pointers, initialize
- // blockNumber based on the last block of the previous map, then verify
- // against the first block associated with each rendered map
- lastBlock, _, err := r.f.getLastBlockOfMap(r.finished.First() - 1)
- if err != nil {
- return fmt.Errorf("failed to get last block of previous map %d: %v", r.finished.First()-1, err)
- }
- blockNumber = lastBlock + 1
- }
- // add or update block pointers
- for mapIndex := range r.finished.Iter() {
- renderedMap := r.finishedMaps[mapIndex]
- if blockNumber != renderedMap.firstBlock() {
- return fmt.Errorf("non-continuous block numbers in rendered map %d (next expected: %d first rendered: %d)", mapIndex, blockNumber, renderedMap.firstBlock())
- }
- r.f.storeLastBlockOfMap(batch, mapIndex, renderedMap.lastBlock, renderedMap.lastBlockId)
- checkWriteCnt()
- for _, lvPtr := range renderedMap.blockLvPtrs {
- r.f.storeBlockLvPointer(batch, blockNumber, lvPtr)
- checkWriteCnt()
- blockNumber++
- }
- }
- if newRange.maps.AfterLast() == r.finished.AfterLast() { // head updated; remove future entries
- for mapIndex := r.finished.AfterLast(); mapIndex < oldRange.maps.AfterLast(); mapIndex++ {
- r.f.deleteLastBlockOfMap(batch, mapIndex)
- checkWriteCnt()
- }
- for ; blockNumber < oldRange.blocks.AfterLast(); blockNumber++ {
- r.f.deleteBlockLvPointer(batch, blockNumber)
- checkWriteCnt()
- }
- }
- r.finishedMaps = make(map[uint32]*renderedMap)
- r.finished.SetFirst(r.finished.AfterLast())
- r.f.setRange(batch, renderedView, newRange, false)
- if err := batch.Write(); err != nil {
- log.Crit("Error writing log index update batch", "error", err)
- }
- totalTime += time.Since(start)
- mapWriteTimer.Update(totalTime)
- return nil
-}
-
-// getTempRange returns a temporary filterMapsRange that is committed to the
-// database while the newly rendered maps are partially written. Writing all
-// processed maps in a single database batch would be a serious hit on db
-// performance so instead safety is ensured by first reverting the valid map
-// range to the unchanged region until all new map data is committed.
-func (r *mapRenderer) getTempRange() (filterMapsRange, error) {
- tempRange := r.f.indexedRange
- if err := tempRange.addRenderedRange(r.finished.First(), r.finished.First(), r.renderBefore, r.f.mapsPerEpoch); err != nil {
- return filterMapsRange{}, fmt.Errorf("failed to update temporary rendered range: %v", err)
- }
- if tempRange.maps.First() != r.f.indexedRange.maps.First() {
- // first rendered map changed; update first indexed block
- if tempRange.maps.First() > 0 {
- firstBlock, _, err := r.f.getLastBlockOfMap(tempRange.maps.First() - 1)
- if err != nil {
- return filterMapsRange{}, fmt.Errorf("failed to retrieve last block of map %d before temporary range: %v", tempRange.maps.First()-1, err)
- }
- tempRange.blocks.SetFirst(firstBlock + 1) // firstBlock is probably partially rendered
- } else {
- tempRange.blocks.SetFirst(0)
- }
- }
- if tempRange.maps.AfterLast() != r.f.indexedRange.maps.AfterLast() {
- // last rendered map changed; update last indexed block
- if !tempRange.maps.IsEmpty() {
- lastBlock, _, err := r.f.getLastBlockOfMap(tempRange.maps.Last())
- if err != nil {
- return filterMapsRange{}, fmt.Errorf("failed to retrieve last block of map %d at the end of temporary range: %v", tempRange.maps.Last(), err)
- }
- tempRange.blocks.SetAfterLast(lastBlock) // lastBlock is probably partially rendered
- } else {
- tempRange.blocks.SetAfterLast(0)
- }
- tempRange.headIndexed = false
- tempRange.headDelimiter = 0
- }
- return tempRange, nil
-}
-
-// getUpdatedRange returns the updated filterMapsRange after writing the newly
-// rendered maps.
-func (r *mapRenderer) getUpdatedRange() (filterMapsRange, error) {
- // update filterMapsRange
- newRange := r.f.indexedRange
- if err := newRange.addRenderedRange(r.finished.First(), r.finished.AfterLast(), r.renderBefore, r.f.mapsPerEpoch); err != nil {
- return filterMapsRange{}, fmt.Errorf("failed to update rendered range: %v", err)
- }
- if newRange.maps.First() != r.f.indexedRange.maps.First() {
- // first rendered map changed; update first indexed block
- if newRange.maps.First() > 0 {
- firstBlock, _, err := r.f.getLastBlockOfMap(newRange.maps.First() - 1)
- if err != nil {
- return filterMapsRange{}, fmt.Errorf("failed to retrieve last block of map %d before rendered range: %v", newRange.maps.First()-1, err)
- }
- newRange.blocks.SetFirst(firstBlock + 1) // firstBlock is probably partially rendered
- } else {
- newRange.blocks.SetFirst(0)
- }
- }
- if newRange.maps.AfterLast() == r.finished.AfterLast() {
- // last rendered map changed; update last indexed block and head pointers
- lm := r.finishedMaps[r.finished.Last()]
- newRange.headIndexed = lm.finished
- if lm.finished {
- newRange.blocks.SetLast(r.f.targetView.HeadNumber())
- if lm.lastBlock != r.f.targetView.HeadNumber() {
- panic("map rendering finished but last block != head block")
- }
- newRange.headDelimiter = lm.headDelimiter
- } else {
- newRange.blocks.SetAfterLast(lm.lastBlock) // lastBlock is probably partially rendered
- newRange.headDelimiter = 0
- }
- } else {
- // last rendered map not replaced; ensure that target chain view matches
- // indexed chain view on the rendered section
- if lastBlock := r.finishedMaps[r.finished.Last()].lastBlock; !matchViews(r.f.indexedView, r.f.targetView, lastBlock) {
- return filterMapsRange{}, errChainUpdate
- }
- }
- return newRange, nil
-}
-
-// addRenderedRange adds the range [firstRendered, afterLastRendered) and
-// removes [afterLastRendered, afterLastRemoved) from the set of rendered maps.
-func (fmr *filterMapsRange) addRenderedRange(firstRendered, afterLastRendered, afterLastRemoved, mapsPerEpoch uint32) error {
- if !fmr.initialized {
- return errors.New("log index not initialized")
- }
-
- // Here we create a slice of endpoints for the rendered sections. There are two endpoints
- // for each section: the index of the first map, and the index after the last map in the
- // section. We then iterate the endpoints -- adding d values -- to determine whether the
- // sections are contiguous or whether they have a gap.
- type endpoint struct {
- m uint32
- d int
- }
- endpoints := []endpoint{{fmr.maps.First(), 1}, {fmr.maps.AfterLast(), -1}, {firstRendered, 1}, {afterLastRendered, -101}, {afterLastRemoved, 100}}
- if fmr.tailPartialEpoch > 0 {
- endpoints = append(endpoints, []endpoint{{fmr.maps.First() - mapsPerEpoch, 1}, {fmr.maps.First() - mapsPerEpoch + fmr.tailPartialEpoch, -1}}...)
- }
- sort.Slice(endpoints, func(i, j int) bool { return endpoints[i].m < endpoints[j].m })
- var (
- sum int
- merged []uint32
- last bool
- )
- for i, e := range endpoints {
- sum += e.d
- if i < len(endpoints)-1 && endpoints[i+1].m == e.m {
- continue
- }
- if (sum > 0) != last {
- merged = append(merged, e.m)
- last = !last
- }
- }
-
- switch len(merged) {
- case 0:
- // Initialized database, but no finished maps yet.
- fmr.tailPartialEpoch = 0
- fmr.maps = common.NewRange(firstRendered, 0)
-
- case 2:
- // One rendered section (no partial tail epoch).
- fmr.tailPartialEpoch = 0
- fmr.maps = common.NewRange(merged[0], merged[1]-merged[0])
-
- case 4:
- // Two rendered sections (with a gap).
- // First section (merged[0]-merged[1]) is for the partial tail epoch,
- // and it has to start exactly one epoch before the main section.
- if merged[2] != merged[0]+mapsPerEpoch {
- return fmt.Errorf("invalid tail partial epoch: %v", merged)
- }
- fmr.tailPartialEpoch = merged[1] - merged[0]
- fmr.maps = common.NewRange(merged[2], merged[3]-merged[2])
-
- default:
- return fmt.Errorf("invalid number of rendered sections: %v", merged)
- }
- return nil
-}
-
-// logIterator iterates on the linear log value index range.
-type logIterator struct {
- params *Params
- chainView *ChainView
- blockNumber uint64
- receipts types.Receipts
- blockStart, delimiter, skipToBoundary, finished bool
- txIndex, logIndex, topicIndex int
- lvIndex uint64
-}
-
-var errUnindexedRange = errors.New("unindexed range")
-
-// newLogIteratorFromBlockDelimiter creates a logIterator starting at the
-// given block's first log value entry (the block delimiter), according to the
-// current targetView.
-func (f *FilterMaps) newLogIteratorFromBlockDelimiter(blockNumber, lvIndex uint64) (*logIterator, error) {
- if blockNumber > f.targetView.HeadNumber() {
- return nil, fmt.Errorf("iterator entry point %d after target chain head block %d", blockNumber, f.targetView.HeadNumber())
- }
- if !f.indexedRange.blocks.Includes(blockNumber) {
- return nil, errUnindexedRange
- }
- finished := blockNumber == f.targetView.HeadNumber()
- l := &logIterator{
- chainView: f.targetView,
- params: &f.Params,
- blockNumber: blockNumber,
- finished: finished,
- delimiter: !finished,
- lvIndex: lvIndex,
- }
- l.enforceValidState()
- return l, nil
-}
-
-// newLogIteratorFromMapBoundary creates a logIterator starting at the given
-// map boundary, according to the current targetView.
-func (f *FilterMaps) newLogIteratorFromMapBoundary(mapIndex uint32, startBlock, startLvPtr uint64) (*logIterator, error) {
- if startBlock > f.targetView.HeadNumber() {
- return nil, fmt.Errorf("iterator entry point %d after target chain head block %d", startBlock, f.targetView.HeadNumber())
- }
- // get block receipts
- receipts := f.targetView.RawReceipts(startBlock)
- if receipts == nil {
- return nil, fmt.Errorf("receipts not found for start block %d", startBlock)
- }
- // initialize iterator at block start
- l := &logIterator{
- chainView: f.targetView,
- params: &f.Params,
- blockNumber: startBlock,
- receipts: receipts,
- blockStart: true,
- lvIndex: startLvPtr,
- }
- l.enforceValidState()
- targetIndex := uint64(mapIndex) << f.logValuesPerMap
- if l.lvIndex > targetIndex {
- return nil, fmt.Errorf("log value pointer %d of last block of map is after map boundary %d", l.lvIndex, targetIndex)
- }
- // iterate to map boundary
- for l.lvIndex < targetIndex {
- if l.finished {
- return nil, fmt.Errorf("iterator already finished at %d before map boundary target %d", l.lvIndex, targetIndex)
- }
- if err := l.next(); err != nil {
- return nil, fmt.Errorf("failed to advance log iterator at %d before map boundary target %d: %v", l.lvIndex, targetIndex, err)
- }
- }
- return l, nil
-}
-
-// updateChainView updates the iterator's chain view if it still matches the
-// previous view at the current position. Returns true if successful.
-func (l *logIterator) updateChainView(cv *ChainView) bool {
- if !matchViews(cv, l.chainView, l.blockNumber) {
- return false
- }
- l.chainView = cv
- return true
-}
-
-// getValueHash returns the log value hash at the current position.
-func (l *logIterator) getValueHash() common.Hash {
- if l.delimiter || l.finished || l.skipToBoundary {
- return common.Hash{}
- }
- log := l.receipts[l.txIndex].Logs[l.logIndex]
- if l.topicIndex == 0 {
- return addressValue(log.Address)
- }
- return topicValue(log.Topics[l.topicIndex-1])
-}
-
-// next moves the iterator to the next log value index.
-func (l *logIterator) next() error {
- if l.skipToBoundary {
- l.lvIndex++
- if l.lvIndex%l.params.valuesPerMap == 0 {
- l.skipToBoundary = false
- }
- return nil
- }
- if l.finished {
- return nil
- }
- if l.delimiter {
- l.delimiter = false
- l.blockNumber++
- l.receipts = l.chainView.RawReceipts(l.blockNumber)
- if l.receipts == nil {
- return fmt.Errorf("receipts not found for block %d", l.blockNumber)
- }
- l.txIndex, l.logIndex, l.topicIndex, l.blockStart = 0, 0, 0, true
- } else {
- l.topicIndex++
- l.blockStart = false
- }
- l.lvIndex++
- l.enforceValidState()
- return nil
-}
-
-// enforceValidState updates the internal transaction, log and topic index pointers
-// to the next existing log value of the given block if necessary.
-// Note that enforceValidState does not advance the log value index pointer.
-func (l *logIterator) enforceValidState() {
- if l.delimiter || l.finished || l.skipToBoundary {
- return
- }
- for ; l.txIndex < len(l.receipts); l.txIndex++ {
- receipt := l.receipts[l.txIndex]
- for ; l.logIndex < len(receipt.Logs); l.logIndex++ {
- log := receipt.Logs[l.logIndex]
- if l.topicIndex == 0 && uint64(len(log.Topics)+1) > l.params.valuesPerMap-l.lvIndex%l.params.valuesPerMap {
- // next log would be split by map boundary; skip to boundary
- l.skipToBoundary = true
- return
- }
- if l.topicIndex <= len(log.Topics) {
- return
- }
- l.topicIndex = 0
- }
- l.logIndex = 0
- }
- if l.blockNumber == l.chainView.HeadNumber() {
- l.finished = true
- } else {
- l.delimiter = true
- }
-}
diff --git a/core/filtermaps/map_storage.go b/core/filtermaps/map_storage.go
new file mode 100644
index 0000000000..3efa90a5e4
--- /dev/null
+++ b/core/filtermaps/map_storage.go
@@ -0,0 +1,711 @@
+// Copyright 2025 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package filtermaps
+
+import (
+ "errors"
+ "fmt"
+ "math"
+ "sync"
+ "sync/atomic"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/log"
+)
+
+var ErrOutOfRange = errors.New("pointer out of indexed range")
+
+// mapStorage implements a filter map storage layer over mapDatabase that ensures
+// efficient database usage while also providing a low latency interface to the
+// indexer. It uses a memory layer over mapDatabase allowing consistently quick
+// addMap and deleteMaps calls while doing the actual database updates in the
+// background asynchronously.
+type mapStorage struct {
+ params *Params
+ mapDb *mapDatabase
+ triggerCh, closeCh chan struct{}
+ closeWg sync.WaitGroup
+ mtForceWrite, mtBusy uint32
+
+ lock sync.RWMutex
+ initialized bool
+ knownEpochs uint32 // epochs initialized with last map block pointer and corresponding reverse block lv pointer
+ knownEpochBlocks uint64
+ valid, dirty rangeSet[uint32] // valid and dirty maps in database
+ writeInProgress, deleteInProgress rangeSet[uint32] // write cycle in progress
+ overlay rangeSet[uint32] // memory maps
+ overlayCount uint32
+ overlayMaps map[uint32]*finishedMap
+ validBlocks, overlayBlocks rangeSet[uint64]
+ epochTrigger rangeSet[uint32]
+ suspended uint32
+
+ testHookCh chan bool
+}
+
+// newMapStorage creates a new mapStorage layer over the given mapDatabase.
+func newMapStorage(params *Params, mapDb *mapDatabase, testHookCh chan bool) *mapStorage {
+ m := &mapStorage{
+ params: params,
+ mapDb: mapDb,
+ triggerCh: make(chan struct{}, 1),
+ closeCh: make(chan struct{}),
+ overlayMaps: make(map[uint32]*finishedMap),
+ testHookCh: testHookCh,
+
+ mtForceWrite: params.rowGroupSize[0] * 9 / 8,
+ mtBusy: params.rowGroupSize[0] * 17 / 8,
+ }
+ if valid, dirty, knownEpochs, ok := m.mapDb.loadMapRange(); ok {
+ m.valid, m.dirty, m.knownEpochs, m.initialized = valid, dirty, knownEpochs, true
+ if knownEpochs > 0 {
+ if lastBlock, _, err := m.mapDb.getLastBlockOfMap(m.params.lastEpochMap(knownEpochs - 1)); err == nil {
+ m.knownEpochBlocks = lastBlock + 1
+ } else {
+ m.resetWithError(fmt.Sprintf("could not initialize known epoch range: %v", err))
+ }
+ }
+ if err := m.updateValidBlocks(); err != nil {
+ m.resetWithError(fmt.Sprintf("could not initialize valid block range: %v", err))
+ }
+ }
+ m.closeWg.Add(1)
+ go m.eventLoop()
+ return m
+}
+
+// stop stops mapStorage.
+func (m *mapStorage) stop() {
+ close(m.closeCh)
+ m.closeWg.Wait()
+}
+
+// isReady returns false if there are too many memory overlay maps at the moment.
+// In this case the caller should suspend the indexing process until some maps
+// are written to the database.
+func (m *mapStorage) isReady() bool {
+ m.lock.Lock()
+ defer m.lock.Unlock()
+
+ if m.overlayCount < m.mtBusy {
+ m.trigger()
+ return true
+ }
+ return false
+}
+
+// tailEpoch returns the first epoch of the continuous rendered range. If it is
+// larger than zero then the tail renderer can start indexing the previous epoch
+// if necessary. After checkpoint initialization it points right after the last
+// known epoch boundary (in this case the tail epoch can be empty).
+func (m *mapStorage) tailEpoch() uint32 {
+ m.lock.Lock()
+ defer m.lock.Unlock()
+
+ mapRange := m.valid.union(m.overlay)
+ if len(mapRange) > 0 && m.params.mapEpoch(mapRange[len(mapRange)-1].AfterLast()) >= m.knownEpochs {
+ return min(m.knownEpochs, m.params.mapEpoch(mapRange[len(mapRange)-1].First()+m.params.mapsPerEpoch-1))
+ }
+ return m.knownEpochs
+}
+
+// tailNumberOfEpoch returns the number of the first block that starts in the
+// given epoch.
+func (m *mapStorage) tailNumberOfEpoch(epoch uint32) (uint64, error) {
+ if epoch == 0 {
+ return 0, nil
+ }
+ number, _, err := m.getLastBlockOfMap(m.params.lastEpochMap(epoch - 1))
+ if err != nil {
+ return 0, err
+ }
+ return number + 1, nil
+}
+
+// lastBoundaryBefore returns the most recent map index that is less than
+// or equal to the given mapIndex parameter and is either located after a stored
+// map or after a known epoch boundary.
+// The returned map index position may or may not contain a stored map but if it
+// is empty then it is always suitable to start a rendering process.
+func (m *mapStorage) lastBoundaryBefore(mapIndex uint32) uint32 {
+ m.lock.Lock()
+ defer m.lock.Unlock()
+
+ if mapIndex == 0 {
+ return 0
+ }
+ lastBoundary := m.params.firstEpochMap(min(m.params.mapEpoch(mapIndex), m.knownEpochs))
+ if m, ok := m.valid.closestLte(mapIndex - 1); ok {
+ lastBoundary = max(lastBoundary, m+1)
+ }
+ if m, ok := m.overlay.closestLte(mapIndex - 1); ok {
+ lastBoundary = max(lastBoundary, m+1)
+ }
+ return lastBoundary
+}
+
+// matchKnownEpochs returns true if the given list of checkpoints matches the
+// checkpoints already stored in the database (always true with an empty database).
+func (m *mapStorage) matchKnownEpochs(cpList checkpointList) bool {
+ m.lock.Lock()
+ defer m.lock.Unlock()
+
+ if m.knownEpochs == 0 || len(cpList) == 0 {
+ return true
+ }
+ epoch := min(m.knownEpochs, uint32(len(cpList))) - 1
+ number, hash, err := m.getLastBlockOfMap(m.params.lastEpochMap(epoch))
+ if err != nil {
+ m.resetWithError(fmt.Sprintf("could not read last known epoch boundary: %v", err))
+ return true
+ }
+ return number == cpList[epoch].BlockNumber && hash == cpList[epoch].BlockHash
+}
+
+// addKnownEpochs adds the known epoch boundaries based on the given checkpoints.
+func (m *mapStorage) addKnownEpochs(cpList checkpointList) error {
+ m.lock.Lock()
+ defer m.lock.Unlock()
+
+ if uint32(len(cpList)) <= m.knownEpochs {
+ return errors.New("checkpoint init list has no new epochs")
+ }
+ if m.knownEpochs > 0 {
+ lastNumber, lastHash, err := m.mapDb.getLastBlockOfMap(m.params.lastEpochMap(m.knownEpochs - 1))
+ if err != nil {
+ return err
+ }
+ lvPointer, err := m.mapDb.getBlockLvPointer(lastNumber)
+ if err != nil {
+ return err
+ }
+ if cp := cpList[m.knownEpochs-1]; cp.BlockNumber != lastNumber || cp.BlockHash != lastHash || cp.FirstIndex != lvPointer {
+ return errors.New("checkpoint init list does not match known epochs")
+ }
+ }
+
+ m.mapDb.storeCheckpointList(m.knownEpochs, cpList[m.knownEpochs:])
+ m.knownEpochs = uint32(len(cpList))
+ m.knownEpochBlocks = cpList[len(cpList)-1].BlockNumber + 1
+ m.mapDb.storeMapRange(m.valid, m.dirty, m.knownEpochs)
+ return nil
+}
+
+// addMap adds a new map to the storage. If forceCommit is true then a write is
+// always triggered. If it is false then a write is only triggered when a row
+// group boundary is reached or if the total number of memory maps reaches a limit.
+// addMap always returns right after adding the new map to the memory layer.
+func (m *mapStorage) addMap(mapIndex uint32, fm *finishedMap, forceCommit bool) {
+ m.lock.Lock()
+ defer m.lock.Unlock()
+
+ if fm == nil {
+ panic("trying to add nil map")
+ }
+ if m.valid.includes(mapIndex) || m.overlay.includes(mapIndex) {
+ panic("addMap to non-empty map index")
+ }
+ epoch := m.params.mapEpoch(mapIndex)
+ if (epoch > m.knownEpochs || mapIndex != m.params.firstEpochMap(epoch)) &&
+ !m.valid.includes(mapIndex-1) && !m.overlay.includes(mapIndex-1) {
+ panic("addMap to map index with no known boundary")
+ }
+ mapRs := singleRangeSet[uint32](common.NewRange[uint32](mapIndex, 1))
+ m.overlay = m.overlay.union(mapRs)
+ m.writeInProgress = m.writeInProgress.exclude(mapRs)
+ m.overlayMaps[mapIndex] = fm
+ m.updateOverlayBlocks()
+ if forceCommit || (mapIndex+1)%m.params.rowGroupSize[0] == 0 {
+ m.epochTrigger = m.epochTrigger.union(singleRangeSet[uint32](common.NewRange[uint32](epoch, 1)))
+ m.trigger()
+ }
+}
+
+// deleteMaps deletes the given map range. Note that similarly to addMap, it only
+// performs memory operations before returning. The deleted map range is marked
+// dirty, then later the database update goroutine will actually delete or
+// overwrite the dirty maps.
+func (m *mapStorage) deleteMaps(maps common.Range[uint32]) {
+ m.lock.Lock()
+ defer m.lock.Unlock()
+
+ dr := singleRangeSet[uint32](maps)
+ for i := range dr.intersection(m.overlay).iter() {
+ delete(m.overlayMaps, i)
+ }
+ m.writeInProgress = m.writeInProgress.exclude(dr)
+ knownEpochs := m.knownEpochs
+ if maps.Includes(m.params.firstEpochMap(knownEpochs)) {
+ knownEpochs = m.params.mapEpoch(maps.First())
+ }
+ if err := m.updateRange(m.valid.exclude(dr), m.dirty.union(dr.intersection(m.valid)), m.overlay.exclude(dr), knownEpochs); err != nil {
+ m.resetWithError(fmt.Sprintf("could not revert valid block range: %v", err))
+ }
+ m.trigger()
+}
+
+// suspendOrResume suspends or resumes the background database update operations.
+func (m *mapStorage) suspendOrResume(suspend bool) {
+ var suspendU32 uint32
+ if suspend {
+ suspendU32 = 1
+ }
+ old := atomic.SwapUint32(&m.suspended, suspendU32)
+ if suspendU32 < old {
+ m.trigger()
+ }
+}
+
+// eventLoop is the main event loop of the database update operations.
+func (m *mapStorage) eventLoop() {
+ defer m.closeWg.Done()
+
+ var stopped bool
+
+ selectEvent := func(blocking bool) {
+ if m.testHookCh != nil {
+ select {
+ case <-m.closeCh:
+ stopped = true
+ return
+ case m.testHookCh <- blocking:
+ }
+ }
+ if blocking {
+ select {
+ case <-m.closeCh:
+ stopped = true
+ return
+ case <-m.triggerCh:
+ }
+ } else {
+ select {
+ case <-m.closeCh:
+ stopped = true
+ return
+ case <-m.triggerCh:
+ default:
+ }
+ }
+ if m.testHookCh != nil {
+ select {
+ case <-m.closeCh:
+ stopped = true
+ return
+ case <-m.testHookCh:
+ }
+ }
+ }
+
+ stopCallback := func() bool {
+ selectEvent(atomic.LoadUint32(&m.suspended) == 1)
+ return stopped
+ }
+
+ for !stopped {
+ if !m.initialized {
+ if done, _ := m.mapDb.reset(stopCallback); !done {
+ return // node stopped before cleaning old database
+ }
+ m.lock.Lock()
+ m.initialized = true
+ m.lock.Unlock()
+ }
+ more, err := m.doWriteCycle(stopCallback)
+ if err != nil {
+ m.resetWithError(fmt.Sprintf("write cycle failed: %v", err))
+ continue
+ }
+ selectEvent(!more && !stopped) // wait for next event if no changes done
+ }
+}
+
+// trigger ensures that the database update cycle will restart if it was in a
+// waiting state.
+func (m *mapStorage) trigger() {
+ select {
+ case m.triggerCh <- struct{}{}:
+ default:
+ }
+}
+
+// getBlockLvPointer returns the starting log value index where the log values
+// generated by the given block are located.
+func (m *mapStorage) getBlockLvPointer(blockNumber uint64) (uint64, error) {
+ m.lock.RLock()
+ defer m.lock.RUnlock()
+
+ if m.overlayBlocks.includes(blockNumber) {
+ for _, fm := range m.overlayMaps {
+ if fm.blocks().Includes(blockNumber) {
+ return fm.blockPtrs[blockNumber-fm.firstBlock()], nil
+ }
+ }
+ return 0, errors.New("memory overlay block pointer not found")
+ }
+ if blockNumber < m.knownEpochBlocks || m.validBlocks.includes(blockNumber) {
+ return m.mapDb.getBlockLvPointer(blockNumber)
+ }
+ return 0, ErrOutOfRange
+}
+
+// getLastBlockOfMap returns the number and hash of the block that generated the
+// last log value entry of the given map.
+func (m *mapStorage) getLastBlockOfMap(mapIndex uint32) (uint64, common.Hash, error) {
+ m.lock.RLock()
+ defer m.lock.RUnlock()
+
+ if m.overlay.includes(mapIndex) {
+ fm := m.overlayMaps[mapIndex]
+ if fm == nil {
+ return 0, common.Hash{}, errors.New("memory overlay map not found")
+ }
+ return fm.lastBlock.number, fm.lastBlock.hash, nil
+ }
+ if mapIndex < m.params.firstEpochMap(m.knownEpochs) || m.valid.includes(mapIndex) || m.valid.includes(mapIndex+1) {
+ return m.mapDb.getLastBlockOfMap(mapIndex)
+ }
+ return 0, common.Hash{}, ErrOutOfRange
+}
+
+// getFilterMapRows returns a batch of filter maps rows from the same row index,
+// each truncated to the length limit of the specified mapping layer.
+// The function assumes that the map indices are in strictly ascending order.
+func (m *mapStorage) getFilterMapRows(mapIndices []uint32, rowIndex, layerIndex uint32) ([]FilterRow, error) {
+ m.lock.RLock()
+ defer m.lock.RUnlock()
+
+ rows := make([]FilterRow, len(mapIndices))
+ dbMaps := make([]uint32, 0, len(mapIndices))
+ for i, mapIndex := range mapIndices {
+ if m.overlay.includes(mapIndex) {
+ fm := m.overlayMaps[mapIndex]
+ if fm == nil {
+ return nil, errors.New("memory overlay map not found")
+ }
+ rows[i] = fm.getRow(rowIndex, m.params.getMaxRowLength(layerIndex))
+ } else {
+ dbMaps = append(dbMaps, mapIndex)
+ }
+ }
+ dbRows, err := m.mapDb.getFilterMapRows(dbMaps, rowIndex, layerIndex)
+ if err != nil {
+ return nil, err
+ }
+ var j int
+ for i, row := range rows {
+ if row == nil { // zero length row is represented as zero length slice
+ rows[i] = dbRows[j]
+ j++
+ }
+ }
+ if j != len(dbMaps) {
+ panic("rows length mismatch")
+ }
+ return rows, nil
+}
+
+// getFilterMap returns the filter map at the specified index.
+func (m *mapStorage) getFilterMap(mapIndex uint32) (*finishedMap, error) {
+ m.lock.RLock()
+ defer m.lock.RUnlock()
+
+ if m.overlay.includes(mapIndex) {
+ fm := m.overlayMaps[mapIndex]
+ if fm == nil {
+ return nil, errors.New("memory overlay map not found")
+ }
+ return fm, nil
+ }
+ if m.valid.includes(mapIndex) {
+ return m.mapDb.getFilterMap(mapIndex)
+ }
+ return nil, nil
+}
+
+// extendDeletedPointerRange takes a map range where pointers need to be deleted
+// and extends it so that on both ends there is a neighboring stored map, a
+// known epoch boundary or one end of the map index range. This is required
+// in order to determine the block range covered by the deleted map range.
+func (m *mapStorage) extendDeletedPointerRange(deleteRange common.Range[uint32]) common.Range[uint32] {
+ epoch := m.params.mapEpoch(deleteRange.First())
+ if m.params.mapEpoch(deleteRange.Last()) != epoch {
+ panic("deleted map range crosses epoch boundary")
+ }
+ first := m.params.firstEpochMap(min(m.knownEpochs, epoch))
+ if deleteRange.First() > 0 {
+ if c, ok := m.valid.closestLte(deleteRange.First() - 1); ok {
+ first = max(first, c+1)
+ }
+ }
+ afterLast := uint32(math.MaxUint32)
+ if epoch < m.knownEpochs {
+ afterLast = m.params.firstEpochMap(epoch + 1)
+ }
+ if fa, ok := m.valid.closestGte(deleteRange.AfterLast()); ok {
+ afterLast = min(afterLast, fa)
+ }
+ return common.NewRange[uint32](first, afterLast-first)
+}
+
+// selectEpochTriggeredWrite selects the next epoch to update if one of the epochs
+// in the canSelect list has been triggered by addMap.
+func (m *mapStorage) selectEpochTriggeredWrite(canSelect []uint32) (uint32, bool) {
+ for _, epoch := range canSelect {
+ if !m.epochTrigger.includes(epoch) {
+ continue
+ }
+ m.epochTrigger = m.epochTrigger.exclude(singleRangeSet[uint32](common.NewRange[uint32](epoch, 1)))
+ epochRange := common.NewRange[uint32](m.params.firstEpochMap(epoch), m.params.mapsPerEpoch)
+ if len(m.overlay.intersection(singleRangeSet[uint32](epochRange))) > 0 {
+ return epoch, true
+ }
+ }
+ return 0, false
+}
+
+// selectEpochForcedWrite selects the next epoch to update if the total number
+// of memory maps has reached a threshold.
+func (m *mapStorage) selectEpochForcedWrite(canSelect []uint32) (uint32, bool) {
+ if m.overlayCount < m.mtForceWrite {
+ return 0, false
+ }
+ var best, count uint32
+ for _, epoch := range canSelect {
+ epochRange := common.NewRange[uint32](m.params.firstEpochMap(epoch), m.params.mapsPerEpoch)
+ if c := m.overlay.intersection(singleRangeSet[uint32](epochRange)).count(); c > count {
+ best, count = epoch, c
+ }
+ }
+ if count == 0 {
+ return 0, false
+ }
+ return best, true
+}
+
+// mapToEpochRange returns the set of epochs that are either fully or partially
+// covered by the specified mapRange.
+func (m *mapStorage) mapToEpochRange(mapRange rangeSet[uint32]) rangeSet[uint32] {
+ vb := make(rangeBoundaries[uint32], 0, len(mapRange)*2)
+ for _, r := range mapRange {
+ first := m.params.mapEpoch(r.First())
+ last := m.params.mapEpoch(r.Last())
+ vb.add(common.NewRange[uint32](first, last+1-first), 1)
+ }
+ return vb.makeSet(1)
+}
+
+// selectEpochDeleteOnly selects an epoch to update where there are no memory
+// maps to write, only dirty maps to clean up.
+func (m *mapStorage) selectEpochDeleteOnly() (uint32, bool) {
+ epochs := m.mapToEpochRange(m.dirty).exclude(m.mapToEpochRange(m.overlay))
+ if len(epochs) == 0 {
+ return 0, false
+ }
+ return epochs[0].First(), true
+}
+
+// doWriteCycle selects an epoch where there are memory overlay maps and/or
+// dirty maps in the underlying database, cleans up dirty data and writes the
+// new maps to the database.
+// If an epoch has been successfully updated then the function returns (true, nil).
+// In this case a new write cycle can be attempted immediately until there is
+// nothing more left to do.
+// If there was nothing to do or if stopCallback aborted the operation then the
+// function returns (false, nil). In case of an error it returns (false, err).
+// Note that if the operation is aborted or failed then the partially updated
+// maps are left as marked dirty and the database remains consistent.
+func (m *mapStorage) doWriteCycle(stopCallback func() bool) (bool, error) {
+ m.lock.Lock()
+ defer m.lock.Unlock()
+
+ // always operate on a single epoch
+ var canSelect []uint32
+ if len(m.valid) == 0 {
+ canSelect = []uint32{m.knownEpochs}
+ } else {
+ lastValid := m.valid[len(m.valid)-1]
+ tailEpoch := m.params.mapEpoch(lastValid.First())
+ headEpoch := m.params.mapEpoch(lastValid.AfterLast())
+ if tailEpoch > 0 {
+ canSelect = []uint32{headEpoch, tailEpoch - 1}
+ } else {
+ canSelect = []uint32{headEpoch}
+ }
+ }
+
+ epoch, ok := m.selectEpochTriggeredWrite(canSelect)
+ if !ok {
+ epoch, ok = m.selectEpochForcedWrite(canSelect)
+ if !ok {
+ epoch, ok = m.selectEpochDeleteOnly()
+ if !ok {
+ return false, nil
+ }
+ }
+ }
+ epochRange := singleRangeSet[uint32](common.NewRange[uint32](m.params.firstEpochMap(epoch), m.params.mapsPerEpoch))
+ writeMaps := epochRange.intersection(m.overlay).singleRange()
+ validInEpoch := epochRange.intersection(m.valid).singleRange()
+ dirtyInEpoch := epochRange.intersection(m.dirty).singleRange()
+ keepEmptyFrom := max(m.params.firstEpochMap(epoch), writeMaps.AfterLast(), validInEpoch.AfterLast(), dirtyInEpoch.AfterLast())
+ keepEmptyInEpoch := common.NewRange[uint32](keepEmptyFrom, m.params.firstEpochMap(epoch+1)-keepEmptyFrom)
+ // delete old pointers
+ if !dirtyInEpoch.IsEmpty() {
+ m.mapDb.deletePointers(m.extendDeletedPointerRange(dirtyInEpoch), stopCallback)
+ }
+ if writeMaps.IsEmpty() && validInEpoch.IsEmpty() {
+ // delete map rows of entire epoch if nothing to write or keep
+ m.lock.Unlock()
+ done, err := m.mapDb.deleteEpochRows(epoch, stopCallback)
+ m.lock.Lock()
+ if done {
+ if err := m.updateRange(m.valid, m.dirty.exclude(epochRange), m.overlay, m.knownEpochs); err != nil {
+ return false, err
+ }
+ }
+ return done, err
+ }
+ maps := make([]*finishedMap, writeMaps.Count())
+ for i := range writeMaps.Iter() {
+ maps[i-writeMaps.First()] = m.overlayMaps[i]
+ }
+ // temporarily mark newly written maps as dirty (replaced/deleted maps are already dirty)
+ writeInProgress := singleRangeSet[uint32](writeMaps)
+ deleteInProgress := singleRangeSet[uint32](dirtyInEpoch).exclude(writeInProgress)
+ if err := m.updateRange(m.valid, m.dirty.union(writeInProgress), m.overlay, m.knownEpochs); err != nil {
+ return false, err
+ }
+ m.writeInProgress = writeInProgress
+ m.deleteInProgress = deleteInProgress
+ m.lock.Unlock()
+ // write/overwrite map rows and delete dirty map data, write new pointers
+ done, err := m.mapDb.writeMapRows(writeMaps, dirtyInEpoch, keepEmptyInEpoch, maps, stopCallback)
+ if done {
+ done, err = m.mapDb.writePointers(writeMaps, maps, stopCallback)
+ }
+ m.lock.Lock()
+ writeInProgress = m.writeInProgress
+ m.writeInProgress, m.deleteInProgress = nil, nil
+ if !done {
+ return false, err
+ }
+ for mapIndex := range writeInProgress.iter() {
+ delete(m.overlayMaps, mapIndex)
+ }
+ knownEpochs := m.knownEpochs
+ if len(writeInProgress) > 0 {
+ knownEpochs = max(knownEpochs, m.params.mapEpoch(writeInProgress[len(writeInProgress)-1].Last()))
+ }
+ if err := m.updateRange(m.valid.union(writeInProgress), m.dirty.exclude(writeInProgress.union(deleteInProgress)), m.overlay.exclude(writeInProgress), knownEpochs); err != nil {
+ return false, err
+ }
+ return true, nil
+}
+
+// updateRange updates the stored valid, dirty and memory overlay map ranges and
+// the number of known epoch boundaries. It also stores these ranges in the
+// database and determines the corresponding block ranges.
+func (m *mapStorage) updateRange(valid, dirty, overlay rangeSet[uint32], knownEpochs uint32) error {
+ if !valid.equal(m.valid) {
+ m.valid = valid
+ if err := m.updateValidBlocks(); err != nil {
+ return err
+ }
+ }
+ if knownEpochs != m.knownEpochs {
+ m.knownEpochs = knownEpochs
+ if err := m.updateKnownEpochBlocks(); err != nil {
+ return err
+ }
+ }
+ m.dirty = dirty
+ if !overlay.equal(m.overlay) {
+ m.overlay = overlay
+ m.updateOverlayBlocks()
+ }
+ m.mapDb.storeMapRange(valid, dirty, m.knownEpochs)
+ return nil
+}
+
+// resetWithError resets an invalid log index database after an unrecoverable error.
+func (m *mapStorage) resetWithError(errStr string) {
+ log.Error("Resetting invalid log index database", "error", errStr)
+ m.uninitialize()
+}
+
+// uninitialize resets the map storage and returns to its non-initialized state.
+func (m *mapStorage) uninitialize() {
+ m.valid, m.dirty, m.overlay, m.knownEpochs, m.initialized = nil, nil, nil, 0, false
+ m.mapDb.deleteMapRange()
+ m.trigger()
+}
+
+// updateKnownEpochBlocks determines the block number belonging to the last known
+// epoch boundary.
+func (m *mapStorage) updateKnownEpochBlocks() error {
+ if m.knownEpochs > 0 {
+ if lastBlock, _, err := m.mapDb.getLastBlockOfMap(m.params.lastEpochMap(m.knownEpochs - 1)); err == nil {
+ m.knownEpochBlocks = lastBlock + 1
+ } else {
+ return err
+ }
+ } else {
+ m.knownEpochBlocks = 0
+ }
+ return nil
+}
+
+// updateValidBlocks determines the the set of blocks where the starting log value
+// pointer points into the valid map set.
+func (m *mapStorage) updateValidBlocks() error {
+ if len(m.valid) > 2 || (len(m.valid) == 2 && m.valid[0].Count() >= m.params.mapsPerEpoch) {
+ panic("invalid mapStorage.valid")
+ return errors.New("invalid mapStorage.valid range set")
+ }
+ vb := make(rangeBoundaries[uint64], 0, len(m.valid)*2)
+ for _, vr := range m.valid {
+ var first uint64
+ if vr.First() > 0 {
+ lb, _, err := m.mapDb.getLastBlockOfMap(vr.First() - 1)
+ if err != nil {
+ return err
+ }
+ first = lb + 1
+ }
+ last, _, err := m.mapDb.getLastBlockOfMap(vr.Last())
+ if err != nil {
+ return err
+ }
+ vb.add(common.NewRange[uint64](first, last+1-first), 1)
+ }
+ m.validBlocks = vb.makeSet(1)
+ return nil
+}
+
+// updateOverlayBlocks determines the the set of blocks where the starting log
+// value pointer points into the memory overlay map set.
+func (m *mapStorage) updateOverlayBlocks() {
+ ob := make(rangeBoundaries[uint64], 0, len(m.overlay)*2)
+ for _, or := range m.overlay {
+ first := m.overlayMaps[or.First()].firstBlock()
+ last := m.overlayMaps[or.Last()].lastBlock.number
+ ob.add(common.NewRange[uint64](first, last+1-first), 1)
+ }
+ m.overlayBlocks = ob.makeSet(1)
+ m.overlayCount = m.overlay.count()
+}
diff --git a/core/filtermaps/map_storage_test.go b/core/filtermaps/map_storage_test.go
new file mode 100644
index 0000000000..0d2939ef5f
--- /dev/null
+++ b/core/filtermaps/map_storage_test.go
@@ -0,0 +1,150 @@
+// Copyright 2025 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package filtermaps
+
+import (
+ "math"
+ "testing"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/ethdb/memorydb"
+)
+
+func TestMapStorage(t *testing.T) {
+ params := testParams
+ params.sanitize()
+ db := memorydb.New()
+ mapDb := newMapDatabase(¶ms, db, false)
+ ms := newMapStorage(¶ms, mapDb, make(chan bool))
+ <-ms.testHookCh
+ defer ms.stop()
+
+ reader := mapReader{
+ getFilterMapRows: ms.getFilterMapRows,
+ getFilterMap: ms.getFilterMap,
+ getBlockLvPointer: ms.getBlockLvPointer,
+ getLastBlockOfMap: ms.getLastBlockOfMap,
+ }
+ waitCycle := func() bool {
+ ms.testHookCh <- true
+ return !<-ms.testHookCh
+ }
+ expKnownEpochs := func(testCase int, exp uint32) {
+ if ms.knownEpochs != exp {
+ t.Fatalf("Invalid known epochs number in test case #%d (expected %d, got %d)", testCase, exp, ms.knownEpochs)
+ }
+ }
+ expTailEpoch := func(testCase int, exp uint32) {
+ if ms.tailEpoch() != exp {
+ t.Fatalf("Invalid tail epoch in test case #%d (expected %d, got %d)", testCase, exp, ms.tailEpoch())
+ }
+ }
+ expReadRows := func(testCase int, exp bool) {
+ if mapDb.testReadRows != exp {
+ t.Fatalf("Invalid read rows flag in test case #%d (expected %v, got %v)", testCase, exp, mapDb.testReadRows)
+ }
+ mapDb.testReadRows = false
+ }
+ // initialize database with checkpoints
+ maps := generateTestMaps(¶ms, nil, 0x200)
+ cpList := generateTestCheckpoints(¶ms, maps)
+ ms.addKnownEpochs(cpList)
+ expTailEpoch(0, 8)
+ expKnownEpochs(0, 8)
+ // add new maps to the head
+ maps = generateTestMaps(¶ms, maps, 0x50)
+ for m := uint32(0x200); m < 0x250; m++ {
+ ms.addMap(m, maps[m], true)
+ }
+ for waitCycle() {
+ testMapReader(t, "mapStorage test #1", ¶ms, reader, cpList, maps[0x200:])
+ }
+ expReadRows(1, false)
+ expTailEpoch(1, 8)
+ expKnownEpochs(1, 9)
+ // backfill previous epoch with a single write
+ for m := uint32(0x1c0); m < 0x200; m++ {
+ ms.addMap(m, maps[m], false)
+ }
+ for waitCycle() {
+ testMapReader(t, "mapDatabase test #2", ¶ms, reader, cpList[:7], maps[0x1c0:])
+ }
+ expReadRows(2, false)
+ expTailEpoch(2, 7)
+ expKnownEpochs(2, 9)
+ // backfill previous epoch in two steps
+ for m := uint32(0x180); m < 0x192; m++ {
+ ms.addMap(m, maps[m], true)
+ }
+ for waitCycle() {
+ }
+ expReadRows(3, false)
+ expTailEpoch(3, 7)
+ for m := uint32(0x192); m < 0x1c0; m++ {
+ ms.addMap(m, maps[m], false)
+ }
+ for waitCycle() {
+ testMapReader(t, "mapDatabase test #3", ¶ms, reader, cpList[:6], maps[0x180:])
+ }
+ expReadRows(3, true)
+ expTailEpoch(3, 6)
+ expKnownEpochs(3, 9)
+ // add new maps while reorging some existing ones
+ maps = generateTestMaps(¶ms, maps[:0x234], 0x30)
+ ms.deleteMaps(common.NewRange[uint32](0x234, math.MaxUint32-0x234))
+ expKnownEpochs(4, 8)
+ for m := uint32(0x234); m < 0x264; m++ {
+ ms.addMap(m, maps[m], true)
+ }
+ for waitCycle() {
+ testMapReader(t, "mapDatabase test #4", ¶ms, reader, cpList[:6], maps[0x180:])
+ }
+ expReadRows(4, true)
+ expTailEpoch(4, 6)
+ expKnownEpochs(4, 9)
+ // unindex tail epoch
+ ms.deleteMaps(common.NewRange[uint32](0x180, 0x40))
+ expTailEpoch(5, 7)
+ for waitCycle() {
+ testMapReader(t, "mapDatabase test #5", ¶ms, reader, cpList[:7], maps[0x1c0:])
+ }
+ expReadRows(5, false)
+ expTailEpoch(5, 7)
+ expKnownEpochs(5, 9)
+ // remove head maps
+ maps = maps[:0x253]
+ ms.deleteMaps(common.NewRange[uint32](0x253, math.MaxUint32-0x253))
+ for waitCycle() {
+ testMapReader(t, "mapDatabase test #6", ¶ms, reader, cpList[:7], maps[0x1c0:])
+ }
+ expReadRows(6, true)
+ expTailEpoch(6, 7)
+ expKnownEpochs(6, 9)
+ // add maps until epoch boundary and check known epochs increase
+ maps = generateTestMaps(¶ms, maps, 0x30)
+ for m := uint32(0x253); m < 0x283; m++ {
+ ms.addMap(m, maps[m], true)
+ }
+ expTailEpoch(7, 7)
+ expKnownEpochs(7, 9)
+ for waitCycle() {
+ testMapReader(t, "mapDatabase test #7", ¶ms, reader, cpList[:7], maps[0x1c0:])
+ }
+ expReadRows(7, true)
+ expTailEpoch(7, 7)
+ expKnownEpochs(7, 10)
+}
diff --git a/core/filtermaps/matcher.go b/core/filtermaps/matcher.go
index 238723fe1d..706b01f123 100644
--- a/core/filtermaps/matcher.go
+++ b/core/filtermaps/matcher.go
@@ -28,6 +28,18 @@ import (
"github.com/ethereum/go-ethereum/common/mclock"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/log"
+ "github.com/ethereum/go-ethereum/metrics"
+)
+
+var (
+ matchRequestTimer = metrics.NewRegisteredTimer("filtermaps/match/requesttime", nil) // processing time a matching request in a single epoch
+ matchEpochTimer = metrics.NewRegisteredTimer("filtermaps/match/epochtime", nil) // total processing time a matching request
+ matchBaseRowAccessMeter = metrics.NewRegisteredMeter("filtermaps/match/baserowaccess", nil) // number of accessed rows on layer 0
+ matchBaseRowSizeMeter = metrics.NewRegisteredMeter("filtermaps/match/baserowsize", nil) // size of accessed rows on layer 0
+ matchExtRowAccessMeter = metrics.NewRegisteredMeter("filtermaps/match/extrowaccess", nil) // number of accessed rows on higher layers
+ matchExtRowSizeMeter = metrics.NewRegisteredMeter("filtermaps/match/extrowsize", nil) // size of accessed rows on higher layers
+ matchLogLookup = metrics.NewRegisteredMeter("filtermaps/match/loglookup", nil) // number of log lookups based on potential matches
+ matchAllMeter = metrics.NewRegisteredMeter("filtermaps/match/matchall", nil) // number of requests returned with ErrMatchAll
)
const doRuntimeStats = false
@@ -37,32 +49,32 @@ const doRuntimeStats = false
// would actually be slower than reverting to legacy filter.
var ErrMatchAll = errors.New("match all patterns not supported")
-// MatcherBackend defines the functions required for searching in the log index
-// data structure. It is currently implemented by FilterMapsMatcherBackend but
-// once EIP-7745 is implemented and active, these functions can also be trustlessly
-// served by a remote prover.
-type MatcherBackend interface {
- GetParams() *Params
- GetBlockLvPointer(ctx context.Context, blockNumber uint64) (uint64, error)
- GetFilterMapRows(ctx context.Context, mapIndices []uint32, rowIndex uint32, baseLayerOnly bool) ([]FilterRow, error)
- GetLogByLvIndex(ctx context.Context, lvIndex uint64) (*types.Log, error)
- SyncLogIndex(ctx context.Context) (SyncRange, error)
- Close()
+type LogPosition struct {
+ BlockNumber, startPtr, logPtr, valuesPerMap uint64
}
-// SyncRange is returned by MatcherBackend.SyncLogIndex. It contains the latest
-// chain head, the indexed range that is currently consistent with the chain
-// and the valid range that has not been changed and has been consistent with
-// all states of the chain since the previous SyncLogIndex or the creation of
-// the matcher backend.
-type SyncRange struct {
- IndexedView *ChainView
- // block range where the index has not changed since the last matcher sync
- // and therefore the set of matches found in this region is guaranteed to
- // be valid and complete.
- ValidBlocks common.Range[uint64]
- // block range indexed according to the given chain head.
- IndexedBlocks common.Range[uint64]
+func (l *LogPosition) GetLog(receipts types.Receipts) (*types.Log, error) {
+ ptr := l.startPtr
+ for _, receipt := range receipts {
+ for _, log := range receipt.Logs {
+ step := uint64(len(log.Topics) + 1)
+ subPtr := ptr % l.valuesPerMap
+ if subPtr+step > l.valuesPerMap {
+ ptr += l.valuesPerMap - subPtr
+ }
+ if ptr == l.logPtr {
+ return log, nil
+ }
+ ptr += step
+ if ptr > l.logPtr {
+ return nil, nil // log position is in block range but there is no log there (false positive)
+ }
+ }
+ }
+ if ptr == l.logPtr {
+ return nil, nil // points to the block delimiter (false positive)
+ }
+ return nil, fmt.Errorf("invalid log position: block #%d start pointer %d log pointer %d", l.BlockNumber, l.startPtr, l.logPtr)
}
// GetPotentialMatches returns a list of logs that are potential matches for the
@@ -70,14 +82,14 @@ type SyncRange struct {
// missing or changed during the search process then the resulting logs belonging
// to that block range might be missing or incorrect.
// Also note that the returned list may contain false positives.
-func GetPotentialMatches(ctx context.Context, backend MatcherBackend, firstBlock, lastBlock uint64, addresses []common.Address, topics [][]common.Hash) ([]*types.Log, error) {
+func GetPotentialMatches(ctx context.Context, backend *IndexView, firstBlock, lastBlock uint64, addresses []common.Address, topics [][]common.Hash) ([]LogPosition, error) {
params := backend.GetParams()
// find the log value index range to search
- firstIndex, err := backend.GetBlockLvPointer(ctx, firstBlock)
+ firstIndex, err := backend.GetBlockLvPointer(firstBlock)
if err != nil {
return nil, fmt.Errorf("failed to retrieve log value pointer for first block %d: %v", firstBlock, err)
}
- lastIndex, err := backend.GetBlockLvPointer(ctx, lastBlock+1)
+ lastIndex, err := backend.GetBlockLvPointer(lastBlock + 1)
if err != nil {
return nil, fmt.Errorf("failed to retrieve log value pointer after last block %d: %v", lastBlock, err)
}
@@ -112,10 +124,11 @@ func GetPotentialMatches(ctx context.Context, backend MatcherBackend, firstBlock
matcher := newMatchSequence(params, matchers)
m := &matcherEnv{
- ctx: ctx,
backend: backend,
params: params,
matcher: matcher,
+ firstBlock: firstBlock,
+ lastBlock: lastBlock,
firstIndex: firstIndex,
lastIndex: lastIndex,
firstMap: uint32(firstIndex >> params.logValuesPerMap),
@@ -123,7 +136,7 @@ func GetPotentialMatches(ctx context.Context, backend MatcherBackend, firstBlock
}
start := time.Now()
- res, err := m.process()
+ res, err := m.process(ctx)
matchRequestTimer.Update(time.Since(start))
if doRuntimeStats {
@@ -142,18 +155,18 @@ func GetPotentialMatches(ctx context.Context, backend MatcherBackend, firstBlock
type matcherEnv struct {
getLogStats runtimeStats // 64 bit aligned
- ctx context.Context
- backend MatcherBackend
+ backend *IndexView
params *Params
matcher matcher
+ firstBlock, lastBlock uint64
firstIndex, lastIndex uint64
firstMap, lastMap uint32
}
-func (m *matcherEnv) process() ([]*types.Log, error) {
+func (m *matcherEnv) process(ctx context.Context) ([]LogPosition, error) {
type task struct {
epochIndex uint32
- logs []*types.Log
+ logs []LogPosition
err error
done chan struct{}
}
@@ -182,7 +195,7 @@ func (m *matcherEnv) process() ([]*types.Log, error) {
}
firstEpoch, lastEpoch := m.firstMap>>m.params.logMapsPerEpoch, m.lastMap>>m.params.logMapsPerEpoch
- var logs []*types.Log
+ var logs []LogPosition
// startEpoch is the next task to send whenever a worker can accept it.
// waitEpoch is the next task we are waiting for to finish in order to append
// results in the correct order.
@@ -214,15 +227,17 @@ func (m *matcherEnv) process() ([]*types.Log, error) {
tasks[waitEpoch] = &task{epochIndex: waitEpoch, done: make(chan struct{})}
}
}
+ /*case <-ctx.Done(): //TODO
+ return nil, ctx.Err()*/
}
}
return logs, nil
}
// processEpoch returns the potentially matching logs from the given epoch.
-func (m *matcherEnv) processEpoch(epochIndex uint32) ([]*types.Log, error) {
+func (m *matcherEnv) processEpoch(epochIndex uint32) ([]LogPosition, error) {
start := time.Now()
- var logs []*types.Log
+ var logs []LogPosition
// create a list of map indices to process
fm, lm := epochIndex<> m.params.logValuesPerMap)
+ lastBlockNumber, _, err := m.backend.GetLastBlockOfMap(mapIndex)
+ if err != nil {
+ return LogPosition{}, fmt.Errorf("failed to retrieve last block of map %d containing searched log value index %d: %v", mapIndex, lvIndex, err)
+ }
+ var firstBlockNumber uint64
+ if mapIndex > 0 {
+ firstBlockNumber, _, err = m.backend.GetLastBlockOfMap(mapIndex - 1)
+ if err != nil {
+ return LogPosition{}, fmt.Errorf("failed to retrieve last block of map %d before searched log value index %d: %v", mapIndex, lvIndex, err)
+ }
+ }
+ firstBlockNumber = max(firstBlockNumber, m.firstBlock)
+ lastBlockNumber = min(lastBlockNumber, m.lastBlock)
+ // find block with binary search based on block to log value index pointers
+ for firstBlockNumber < lastBlockNumber {
+ midBlockNumber := (firstBlockNumber + lastBlockNumber + 1) / 2
+ midLvPointer, err := m.backend.GetBlockLvPointer(midBlockNumber)
+ if err != nil {
+ return LogPosition{}, fmt.Errorf("failed to retrieve log value pointer of block %d while binary searching log value index %d: %v", midBlockNumber, lvIndex, err)
+ }
+ if lvIndex < midLvPointer {
+ lastBlockNumber = midBlockNumber - 1
+ } else {
+ firstBlockNumber = midBlockNumber
+ }
+ }
+ blockLvPointer, err := m.backend.GetBlockLvPointer(firstBlockNumber)
+ if err != nil {
+ return LogPosition{}, fmt.Errorf("failed to retrieve log value pointer of block %d containing searched log value index %d: %v", firstBlockNumber, lvIndex, err)
+ }
+ return LogPosition{BlockNumber: firstBlockNumber, startPtr: blockLvPointer, logPtr: lvIndex, valuesPerMap: m.params.valuesPerMap}, nil
+}
+
// getLogsFromMatches returns the list of potentially matching logs located at
// the given list of matching log indices. Matches outside the firstIndex to
// lastIndex range are not returned.
-func (m *matcherEnv) getLogsFromMatches(matches potentialMatches) ([]*types.Log, error) {
- var logs []*types.Log
+func (m *matcherEnv) getLogsFromMatches(matches potentialMatches) ([]LogPosition, error) {
+ var logs []LogPosition
for _, match := range matches {
if match < m.firstIndex || match > m.lastIndex {
continue
}
- log, err := m.backend.GetLogByLvIndex(m.ctx, match)
+ log, err := m.getLogPosition(match)
if err != nil {
return logs, fmt.Errorf("failed to retrieve log at index %d: %v", match, err)
}
- if log != nil {
- logs = append(logs, log)
- }
+ logs = append(logs, log)
matchLogLookup.Mark(1)
}
return logs, nil
@@ -288,7 +337,7 @@ func (m *matcherEnv) getAllMatches(mapIndices []uint32) ([]potentialMatches, err
instance := m.matcher.newInstance(mapIndices)
resultsMap := make(map[uint32]potentialMatches)
for layerIndex := uint32(0); len(resultsMap) < len(mapIndices); layerIndex++ {
- results, err := instance.getMatchesForLayer(m.ctx, layerIndex)
+ results, err := instance.getMatchesForLayer(layerIndex)
if err != nil {
return nil, err
}
@@ -319,7 +368,7 @@ type matcher interface {
// a result has been returned has no effect. Exactly one matcherResult is
// returned per requested map index unless dropped.
type matcherInstance interface {
- getMatchesForLayer(ctx context.Context, layerIndex uint32) ([]matcherResult, error)
+ getMatchesForLayer(layerIndex uint32) ([]matcherResult, error)
dropIndices(mapIndices []uint32)
}
@@ -332,7 +381,7 @@ type matcherResult struct {
// singleMatcher implements matcher by returning matches for a single log value hash.
type singleMatcher struct {
- backend MatcherBackend
+ backend *IndexView
value common.Hash
stats runtimeStats
}
@@ -360,7 +409,7 @@ func (m *singleMatcher) newInstance(mapIndices []uint32) matcherInstance {
}
// getMatchesForLayer implements matcherInstance.
-func (m *singleMatcherInstance) getMatchesForLayer(ctx context.Context, layerIndex uint32) (results []matcherResult, err error) {
+func (m *singleMatcherInstance) getMatchesForLayer(layerIndex uint32) (results []matcherResult, err error) {
var st int
m.stats.setState(&st, stOther)
params := m.backend.GetParams()
@@ -378,7 +427,7 @@ func (m *singleMatcherInstance) getMatchesForLayer(ctx context.Context, layerInd
} else {
m.stats.setState(&st, stFetchMore)
}
- groupRows, err := m.backend.GetFilterMapRows(ctx, m.mapIndices[ptr:ptr+groupLength], rowIndex, layerIndex == 0)
+ groupRows, err := m.backend.GetFilterMapRows(m.mapIndices[ptr:ptr+groupLength], rowIndex, layerIndex)
if err != nil {
m.stats.setState(&st, stNone)
return nil, fmt.Errorf("failed to retrieve filter map %d row %d: %v", m.mapIndices[ptr], rowIndex, err)
@@ -400,7 +449,7 @@ func (m *singleMatcherInstance) getMatchesForLayer(ctx context.Context, layerInd
}
m.stats.addAmount(st, int64(len(filterRow)))
filterRows = append(filterRows, filterRow)
- if uint32(len(filterRow)) < params.maxRowLength(layerIndex) {
+ if uint32(len(filterRow)) < params.getMaxRowLength(layerIndex) {
m.stats.setState(&st, stProcess)
matches := params.potentialMatches(filterRows, mapIndex, m.value)
m.stats.addAmount(st, int64(len(matches)))
@@ -416,6 +465,12 @@ func (m *singleMatcherInstance) getMatchesForLayer(ctx context.Context, layerInd
}
ptr += groupLength
}
+ if len(m.mapIndices) > 0 {
+ var r int
+ for _, res := range results {
+ r += len(res.matches)
+ }
+ }
m.cleanMapIndices()
m.stats.setState(&st, stNone)
return results, nil
@@ -491,7 +546,7 @@ func (m matchAny) newInstance(mapIndices []uint32) matcherInstance {
}
// getMatchesForLayer implements matcherInstance.
-func (m *matchAnyInstance) getMatchesForLayer(ctx context.Context, layerIndex uint32) (mergedResults []matcherResult, err error) {
+func (m *matchAnyInstance) getMatchesForLayer(layerIndex uint32) (mergedResults []matcherResult, err error) {
if len(m.matchAny) == 0 {
// return "wild card" results (potentialMatches(nil) is interpreted as a
// potential match at every log value index of the map).
@@ -504,7 +559,7 @@ func (m *matchAnyInstance) getMatchesForLayer(ctx context.Context, layerIndex ui
return mergedResults, nil
}
for i, childInstance := range m.childInstances {
- results, err := childInstance.getMatchesForLayer(ctx, layerIndex)
+ results, err := childInstance.getMatchesForLayer(layerIndex)
if err != nil {
return nil, fmt.Errorf("failed to evaluate child matcher on layer %d: %v", layerIndex, err)
}
@@ -698,19 +753,19 @@ type matchSequenceInstance struct {
}
// getMatchesForLayer implements matcherInstance.
-func (m *matchSequenceInstance) getMatchesForLayer(ctx context.Context, layerIndex uint32) (matchedResults []matcherResult, err error) {
+func (m *matchSequenceInstance) getMatchesForLayer(layerIndex uint32) (matchedResults []matcherResult, err error) {
// decide whether to evaluate base or next matcher first
baseFirst := m.baseFirst()
if baseFirst {
- if err := m.evalBase(ctx, layerIndex); err != nil {
+ if err := m.evalBase(layerIndex); err != nil {
return nil, err
}
}
- if err := m.evalNext(ctx, layerIndex); err != nil {
+ if err := m.evalNext(layerIndex); err != nil {
return nil, err
}
if !baseFirst {
- if err := m.evalBase(ctx, layerIndex); err != nil {
+ if err := m.evalBase(layerIndex); err != nil {
return nil, err
}
}
@@ -753,8 +808,8 @@ func (m *matchSequenceInstance) dropIndices(dropIndices []uint32) {
// evalBase evaluates the base child matcher and drops map indices from the
// next matcher if possible.
-func (m *matchSequenceInstance) evalBase(ctx context.Context, layerIndex uint32) error {
- results, err := m.baseInstance.getMatchesForLayer(ctx, layerIndex)
+func (m *matchSequenceInstance) evalBase(layerIndex uint32) error {
+ results, err := m.baseInstance.getMatchesForLayer(layerIndex)
if err != nil {
return fmt.Errorf("failed to evaluate base matcher on layer %d: %v", layerIndex, err)
}
@@ -781,8 +836,8 @@ func (m *matchSequenceInstance) evalBase(ctx context.Context, layerIndex uint32)
// evalNext evaluates the next child matcher and drops map indices from the
// base matcher if possible.
-func (m *matchSequenceInstance) evalNext(ctx context.Context, layerIndex uint32) error {
- results, err := m.nextInstance.getMatchesForLayer(ctx, layerIndex)
+func (m *matchSequenceInstance) evalNext(layerIndex uint32) error {
+ results, err := m.nextInstance.getMatchesForLayer(layerIndex)
if err != nil {
return fmt.Errorf("failed to evaluate next matcher on layer %d: %v", layerIndex, err)
}
diff --git a/core/filtermaps/matcher_backend.go b/core/filtermaps/matcher_backend.go
deleted file mode 100644
index abd0fd9283..0000000000
--- a/core/filtermaps/matcher_backend.go
+++ /dev/null
@@ -1,201 +0,0 @@
-// Copyright 2024 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see .
-
-package filtermaps
-
-import (
- "context"
-
- "github.com/ethereum/go-ethereum/common"
- "github.com/ethereum/go-ethereum/core/types"
-)
-
-// FilterMapsMatcherBackend implements MatcherBackend.
-type FilterMapsMatcherBackend struct {
- f *FilterMaps
-
- // these fields should be accessed under f.matchersLock mutex.
- validBlocks common.Range[uint64]
- syncCh chan SyncRange
-}
-
-// NewMatcherBackend returns a FilterMapsMatcherBackend after registering it in
-// the active matcher set.
-// Note that Close should always be called when the matcher is no longer used.
-func (f *FilterMaps) NewMatcherBackend() *FilterMapsMatcherBackend {
- f.indexLock.RLock()
- f.matchersLock.Lock()
- defer func() {
- f.matchersLock.Unlock()
- f.indexLock.RUnlock()
- }()
-
- fm := &FilterMapsMatcherBackend{f: f}
- if f.indexedRange.initialized {
- fm.validBlocks = f.indexedRange.blocks
- }
- f.matchers[fm] = struct{}{}
- return fm
-}
-
-// GetParams returns the filtermaps parameters.
-// GetParams implements MatcherBackend.
-func (fm *FilterMapsMatcherBackend) GetParams() *Params {
- return &fm.f.Params
-}
-
-// Close removes the matcher from the set of active matchers and ensures that
-// any SyncLogIndex calls are cancelled.
-// Close implements MatcherBackend.
-func (fm *FilterMapsMatcherBackend) Close() {
- fm.f.matchersLock.Lock()
- defer fm.f.matchersLock.Unlock()
-
- delete(fm.f.matchers, fm)
-}
-
-// GetFilterMapRows returns the given row of the given map. If the row is empty
-// then a non-nil zero length row is returned. If baseLayerOnly is true then
-// only the first baseRowLength entries of the row are guaranteed to be
-// returned.
-// Note that the returned slices should not be modified, they should be copied
-// on write.
-// GetFilterMapRows implements MatcherBackend.
-func (fm *FilterMapsMatcherBackend) GetFilterMapRows(ctx context.Context, mapIndices []uint32, rowIndex uint32, baseLayerOnly bool) ([]FilterRow, error) {
- return fm.f.getFilterMapRows(mapIndices, rowIndex, baseLayerOnly)
-}
-
-// GetBlockLvPointer returns the starting log value index where the log values
-// generated by the given block are located. If blockNumber is beyond the last
-// indexed block then the pointer will point right after this block, ensuring
-// that the matcher does not fail and can return a set of results where the
-// valid range is correct.
-// GetBlockLvPointer implements MatcherBackend.
-func (fm *FilterMapsMatcherBackend) GetBlockLvPointer(ctx context.Context, blockNumber uint64) (uint64, error) {
- fm.f.indexLock.RLock()
- defer fm.f.indexLock.RUnlock()
-
- if blockNumber >= fm.f.indexedRange.blocks.AfterLast() {
- if fm.f.indexedRange.headIndexed {
- // return index after head block
- return fm.f.indexedRange.headDelimiter + 1, nil
- }
- if fm.f.indexedRange.blocks.Count() > 0 {
- // return index at the beginning of the last, partially indexed
- // block (after the last fully indexed one)
- blockNumber = fm.f.indexedRange.blocks.Last()
- }
- }
- return fm.f.getBlockLvPointer(blockNumber)
-}
-
-// GetLogByLvIndex returns the log at the given log value index.
-// Note that this function assumes that the log index structure is consistent
-// with the canonical chain at the point where the given log value index points.
-// If this is not the case then an invalid result may be returned or certain
-// logs might not be returned at all.
-// No error is returned though because of an inconsistency between the chain and
-// the log index. It is the caller's responsibility to verify this consistency
-// using SyncLogIndex and re-process certain blocks if necessary.
-// GetLogByLvIndex implements MatcherBackend.
-func (fm *FilterMapsMatcherBackend) GetLogByLvIndex(ctx context.Context, lvIndex uint64) (*types.Log, error) {
- fm.f.indexLock.RLock()
- defer fm.f.indexLock.RUnlock()
-
- return fm.f.getLogByLvIndex(lvIndex)
-}
-
-// synced signals to the matcher that has triggered a synchronisation that it
-// has been finished and the log index is consistent with the chain head passed
-// as a parameter.
-//
-// Note that if the log index head was far behind the chain head then it might not
-// be synced up to the given head in a single step. Still, the latest chain head
-// should be passed as a parameter and the existing log index should be consistent
-// with that chain.
-//
-// Note: acquiring the indexLock read lock is unnecessary here, as this function
-// is always called within the indexLoop.
-func (fm *FilterMapsMatcherBackend) synced() {
- fm.f.matchersLock.Lock()
- defer fm.f.matchersLock.Unlock()
-
- indexedBlocks := fm.f.indexedRange.blocks
- if !fm.f.indexedRange.headIndexed && !indexedBlocks.IsEmpty() {
- indexedBlocks.SetAfterLast(indexedBlocks.Last()) // remove partially indexed last block
- }
- fm.syncCh <- SyncRange{
- IndexedView: fm.f.indexedView,
- ValidBlocks: fm.validBlocks,
- IndexedBlocks: indexedBlocks,
- }
- fm.validBlocks = indexedBlocks
- fm.syncCh = nil
-}
-
-// SyncLogIndex ensures that the log index is consistent with the current state
-// of the chain and is synced up to the current head. It blocks until this state
-// is achieved or the context is cancelled.
-// If successful, it returns a SyncRange that contains the latest chain head,
-// the indexed range that is currently consistent with the chain and the valid
-// range that has not been changed and has been consistent with all states of the
-// chain since the previous SyncLogIndex or the creation of the matcher backend.
-func (fm *FilterMapsMatcherBackend) SyncLogIndex(ctx context.Context) (SyncRange, error) {
- syncCh := make(chan SyncRange, 1)
- fm.f.matchersLock.Lock()
- fm.syncCh = syncCh
- fm.f.matchersLock.Unlock()
-
- select {
- case fm.f.matcherSyncCh <- fm:
- case <-ctx.Done():
- return SyncRange{}, ctx.Err()
- case <-fm.f.disabledCh:
- // Note: acquiring the indexLock read lock is unnecessary here,
- // as the indexer has already been terminated.
- return SyncRange{IndexedView: fm.f.indexedView}, nil
- }
- select {
- case vr := <-syncCh:
- return vr, nil
- case <-ctx.Done():
- return SyncRange{}, ctx.Err()
- case <-fm.f.disabledCh:
- // Note: acquiring the indexLock read lock is unnecessary here,
- // as the indexer has already been terminated.
- return SyncRange{IndexedView: fm.f.indexedView}, nil
- }
-}
-
-// updateMatchersValidRange iterates through active matchers and limits their
-// valid range with the current indexed range. This function should be called
-// whenever a part of the log index has been removed, before adding new blocks
-// to it.
-//
-// Note: acquiring the indexLock read lock is unnecessary here, as this function
-// is always called within the indexLoop.
-func (f *FilterMaps) updateMatchersValidRange() {
- f.matchersLock.Lock()
- defer f.matchersLock.Unlock()
-
- for fm := range f.matchers {
- if !f.indexedRange.initialized {
- fm.validBlocks = common.Range[uint64]{}
- continue
- }
- fm.validBlocks = fm.validBlocks.Intersection(f.indexedRange.blocks)
- }
-}
diff --git a/core/filtermaps/matcher_test.go b/core/filtermaps/matcher_test.go
index 5a49ce66ec..d8c1747e5e 100644
--- a/core/filtermaps/matcher_test.go
+++ b/core/filtermaps/matcher_test.go
@@ -16,6 +16,7 @@
package filtermaps
+/*
import (
"context"
crand "crypto/rand"
@@ -85,3 +86,4 @@ func TestMatcher(t *testing.T) {
}
}
}
+*/
diff --git a/core/filtermaps/math.go b/core/filtermaps/math.go
index 68fd6debd6..8453510e5c 100644
--- a/core/filtermaps/math.go
+++ b/core/filtermaps/math.go
@@ -21,66 +21,59 @@ import (
"encoding/binary"
"fmt"
"hash/fnv"
+ "iter"
"math"
"slices"
+ "sort"
"github.com/ethereum/go-ethereum/common"
)
+// FilterRow encodes a single row of a filter map as a list of column indices.
+// Note that the values are always stored in the same order as they were added
+// and if the same column index is added twice, it is also stored twice.
+// Order of column indices and potential duplications do not matter when searching
+// for a value but leaving the original order makes reverting to a previous state
+// simpler.
+type FilterRow []uint32
+
// Params defines the basic parameters of the log index structure.
type Params struct {
- logMapHeight uint // The number of bits required to represent the map height
- logMapWidth uint // The number of bits required to represent the map width
- logMapsPerEpoch uint // The number of bits required to represent the number of maps per epoch
- logValuesPerMap uint // The number of bits required to represent the number of log values per map
-
- // baseRowLengthRatio represents the ratio of base row length
- // to the average row length.
- baseRowLengthRatio uint
-
- // logLayerDiff defines the logarithmic growth factor (base 2) of
- // the maximum row length per layer. It indicates how much the maximum
- // row length increases as the layer depth increases.
- //
- // Specifically:
- // - the row length in base layer (layer == 0) is baseRowLength
- // - the row length in layer x is baseRowLength << (logLayerDiff * x)
- logLayerDiff uint
+ logMapHeight uint // The number of bits required to represent the map height
+ logMapWidth uint // The number of bits required to represent the map width
+ logMapsPerEpoch uint // The number of bits required to represent the number of maps per epoch
+ logValuesPerMap uint // The number of bits required to represent the number of log values per map
+ logMappingFrequency []uint
+ maxRowLength []uint32
// These fields can be derived with the information above
- mapHeight uint32 // The number of rows in the filter map
- mapsPerEpoch uint32 // The number of maps in an epoch
- valuesPerMap uint64 // The number of log values marked on each filter map
- baseRowLength uint32 // maximum number of log values per row on layer 0
+ mapHeight uint32 // The number of rows in the filter map
+ mapsPerEpoch uint32 // The number of maps in an epoch
+ valuesPerMap uint64 // The number of log values marked on each filter map
- // baseRowGroupSize defines the number of base row entries grouped together
- // as a single database entry in the local database to optimize storage
- // and retrieval efficiency.
- //
- // This value can be configured based on the specific implementation.
- baseRowGroupSize uint32
+ rowGroupSize []uint32
}
// DefaultParams is the set of parameters used on mainnet.
var DefaultParams = Params{
- logMapHeight: 16,
- logMapWidth: 24,
- logMapsPerEpoch: 10,
- logValuesPerMap: 16,
- baseRowGroupSize: 32,
- baseRowLengthRatio: 8,
- logLayerDiff: 4,
+ logMapHeight: 16,
+ logMapWidth: 24,
+ logMapsPerEpoch: 10,
+ logValuesPerMap: 16,
+ logMappingFrequency: []uint{10, 6, 2, 0},
+ maxRowLength: []uint32{8, 168, 2728, 10920},
+ rowGroupSize: []uint32{256, 16, 1, 1},
}
// RangeTestParams puts one log value per epoch, ensuring block exact tail unindexing for testing
var RangeTestParams = Params{
- logMapHeight: 4,
- logMapWidth: 24,
- logMapsPerEpoch: 0,
- logValuesPerMap: 0,
- baseRowGroupSize: 32,
- baseRowLengthRatio: 16, // baseRowLength >= 1
- logLayerDiff: 4,
+ logMapHeight: 4,
+ logMapWidth: 24,
+ logMapsPerEpoch: 0,
+ logValuesPerMap: 0,
+ logMappingFrequency: []uint{10, 6, 2, 0},
+ maxRowLength: []uint32{8, 168, 2728, 10920},
+ rowGroupSize: []uint32{16, 4, 1, 1},
}
// deriveFields calculates the derived fields of the parameter set.
@@ -88,7 +81,25 @@ func (p *Params) deriveFields() {
p.mapHeight = uint32(1) << p.logMapHeight
p.mapsPerEpoch = uint32(1) << p.logMapsPerEpoch
p.valuesPerMap = uint64(1) << p.logValuesPerMap
- p.baseRowLength = uint32(p.valuesPerMap * uint64(p.baseRowLengthRatio) / uint64(p.mapHeight))
+}
+
+// mapRowIndex calculates the unified storage index where the given row of the
+// given map is stored. Note that this indexing scheme is the same as the one
+// proposed in EIP-7745 for tree-hashing the filter map structure and for the
+// same data proximity reasons it is also suitable for database representation.
+// See also:
+// https://eips.ethereum.org/EIPS/eip-7745#hash-tree-structure
+func (p *Params) mapRowIndex(mapIndex, rowIndex uint32) uint64 {
+ epochIndex, mapSubIndex := mapIndex>>p.logMapsPerEpoch, mapIndex&(p.mapsPerEpoch-1)
+ return (uint64(epochIndex)< 32 { // column index stored as uint32
+ return fmt.Errorf("invalid configuration: logMapWidth (%d) should not exceed 32", p.logMapWidth)
+ }
+ if p.logMapHeight > 16 { // row index stored as uint16 in finishedMap
+ return fmt.Errorf("invalid configuration: logMapHeight (%d) should not exceed 32", p.logMapHeight)
+ }
+ for _, maxRowLength := range p.maxRowLength {
+ if maxRowLength >= 0x10000 { // index wrap-around issue in finishedMap
+ return fmt.Errorf("invalid configuration: maxRowLength entry (%d) should not exceed 2**16", maxRowLength)
+ }
+ }
+ for _, groupSize := range p.rowGroupSize {
+ if groupSize == 0 || (groupSize&(groupSize-1)) != 0 {
+ return fmt.Errorf("invalid configuration: rowGroupSize entry (%d) must be a power of 2", groupSize)
+ }
+ }
+ if len(p.maxRowLength) != len(p.rowGroupSize) {
+ return fmt.Errorf("invalid configuration: length of maxRowLength (%d) and rowGroupSize entry (%d) should be equal", len(p.maxRowLength), len(p.rowGroupSize))
}
return nil
}
// mapGroupIndex returns the start index of the base row group that contains the
// given map index. Assumes baseRowGroupSize is a power of 2.
-func (p *Params) mapGroupIndex(index uint32) uint32 {
- return index & ^(p.baseRowGroupSize - 1)
+func (p *Params) mapGroupIndex(index, dbLayer uint32) uint32 {
+ return index & ^(p.rowGroupSize[dbLayer] - 1)
}
// mapGroupOffset returns the offset of the given map index within its base row group.
-func (p *Params) mapGroupOffset(index uint32) uint32 {
- return index & (p.baseRowGroupSize - 1)
+func (p *Params) mapGroupOffset(index, dbLayer uint32) uint32 {
+ return index & (p.rowGroupSize[dbLayer] - 1)
}
// mapEpoch returns the epoch number that the given map index belongs to.
@@ -181,38 +208,12 @@ func (p *Params) columnIndex(lvIndex uint64, logValue *common.Hash) uint32 {
return uint32(lvIndex%p.valuesPerMap)<>(64-hashBits)) ^ uint32(hash)>>(32-hashBits))
}
-// maxRowLength returns the maximum length filter rows are populated up to when
-// using the given mapping layer.
-//
-// A log value can be marked on the map according to a given mapping layer if
-// the row mapping on that layer points to a row that has not yet reached the
-// maxRowLength belonging to that layer.
-//
-// This means that a row that is considered full on a given layer may still be
-// extended further on a higher order layer.
-//
-// Each value is marked on the lowest order layer possible, assuming that marks
-// are added in ascending log value index order. When searching for a log value
-// one should consider all layers and process corresponding rows up until the
-// first one where the row mapped to the given layer is not full.
-func (p *Params) maxRowLength(layerIndex uint32) uint32 {
- logLayerDiff := uint(layerIndex) * p.logLayerDiff
- if logLayerDiff > p.logMapsPerEpoch {
- logLayerDiff = p.logMapsPerEpoch
- }
- return p.baseRowLength << logLayerDiff
-}
-
// maskedMapIndex returns the index used for row mapping calculation on the
// given layer. On layer zero the mapping changes once per epoch, then the
// frequency of re-mapping increases with every new layer until it reaches
// the frequency where it is different for every mapIndex.
func (p *Params) maskedMapIndex(mapIndex, layerIndex uint32) uint32 {
- logLayerDiff := uint(layerIndex) * p.logLayerDiff
- if logLayerDiff > p.logMapsPerEpoch {
- logLayerDiff = p.logMapsPerEpoch
- }
- return mapIndex & (uint32(math.MaxUint32) << (p.logMapsPerEpoch - logLayerDiff))
+ return mapIndex & (uint32(math.MaxUint32) << p.getLogMappingFrequency(layerIndex))
}
// potentialMatches returns the list of log value indices potentially matching
@@ -229,7 +230,7 @@ func (p *Params) potentialMatches(rows []FilterRow, mapIndex uint32, logValue co
results := make(potentialMatches, 0, 8)
mapFirst := uint64(mapIndex) << p.logValuesPerMap
for i, row := range rows {
- rowLen, maxLen := len(row), int(p.maxRowLength(uint32(i)))
+ rowLen, maxLen := len(row), int(p.getMaxRowLength(uint32(i)))
if rowLen > maxLen {
rowLen = maxLen // any additional entries are generated by another log value on a higher mapping layer
}
@@ -264,3 +265,177 @@ func (p *Params) potentialMatches(rows []FilterRow, mapIndex uint32, logValue co
// indices in the filter map range are potential matches. If there are no
// potential matches in the given map's range then an empty slice should be used.
type potentialMatches []uint64
+
+func (p potentialMatches) Len() int { return len(p) }
+func (p potentialMatches) Less(i, j int) bool { return p[i] < p[j] }
+func (p potentialMatches) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
+
+type rangeSet[T uint32 | uint64] []common.Range[T]
+
+func (a rangeSet[T]) includes(v T) bool {
+ for _, r := range a {
+ if r.Includes(v) {
+ return true
+ }
+ }
+ return false
+}
+
+func (a rangeSet[T]) closestLte(v T) (last T, found bool) {
+ for _, r := range a {
+ if r.First() > v {
+ return
+ }
+ if r.AfterLast() > v {
+ return v, true
+ }
+ last, found = r.Last(), true
+ }
+ return
+}
+
+func (a rangeSet[T]) closestGte(v T) (last T, found bool) {
+ for _, r := range a {
+ if r.First() > v {
+ return r.First(), true
+ }
+ if r.AfterLast() > v {
+ return v, true
+ }
+ }
+ return
+}
+
+type rangeBoundary[T uint32 | uint64] struct {
+ v T
+ d int
+}
+
+type rangeBoundaries[T uint32 | uint64] []rangeBoundary[T]
+
+func (rb *rangeBoundaries[T]) add(r common.Range[T], d int) {
+ *rb = append((*rb), rangeBoundary[T]{v: r.First(), d: d}, rangeBoundary[T]{v: r.AfterLast(), d: -d})
+}
+
+func (rb rangeBoundaries[T]) makeSet(threshold int) rangeSet[T] {
+ res := make(rangeSet[T], 0, len(rb)/2)
+ sort.Slice(rb, func(i, j int) bool {
+ return rb[i].v < rb[j].v
+ })
+ var (
+ sum int
+ lastCmp bool
+ start T
+ )
+ for i, r := range rb {
+ sum += r.d
+ cmp := sum >= threshold
+ if cmp != lastCmp && (i == len(rb)-1 || rb[i+1].v != r.v) {
+ if cmp {
+ start = r.v
+ } else {
+ res = append(res, common.NewRange[T](start, r.v-start))
+ }
+ lastCmp = cmp
+ }
+ }
+ return res
+}
+
+func (a rangeSet[T]) intersection(b rangeSet[T]) rangeSet[T] {
+ if len(a) == 0 || len(b) == 0 {
+ return nil
+ }
+ rb := make(rangeBoundaries[T], 0, (len(a)+len(b))*2)
+ for _, r := range a {
+ rb.add(r, 1)
+ }
+ for _, r := range b {
+ rb.add(r, 1)
+ }
+ return rb.makeSet(2)
+}
+
+func (a rangeSet[T]) exclude(b rangeSet[T]) rangeSet[T] {
+ if len(a) == 0 {
+ return nil
+ }
+ if len(b) == 0 {
+ return a
+ }
+ rb := make(rangeBoundaries[T], 0, (len(a)+len(b))*2)
+ for _, r := range a {
+ rb.add(r, 1)
+ }
+ for _, r := range b {
+ rb.add(r, -1)
+ }
+ return rb.makeSet(1)
+}
+
+func (a rangeSet[T]) union(b rangeSet[T]) rangeSet[T] {
+ if len(a) == 0 {
+ return b
+ }
+ if len(b) == 0 {
+ return a
+ }
+ rb := make(rangeBoundaries[T], 0, (len(a)+len(b))*2)
+ for _, r := range a {
+ rb.add(r, 1)
+ }
+ for _, r := range b {
+ rb.add(r, 1)
+ }
+ return rb.makeSet(1)
+}
+
+// iter iterates all integers in the range set.
+func (r rangeSet[T]) iter() iter.Seq[T] {
+ return func(yield func(T) bool) {
+ for _, rr := range r {
+ for i := range rr.Iter() {
+ if !yield(i) {
+ break
+ }
+ }
+ }
+ }
+}
+
+func (a rangeSet[T]) count() T {
+ var count T
+ for _, r := range a {
+ count += r.Count()
+ }
+ return count
+}
+
+func (a rangeSet[T]) singleRange() common.Range[T] {
+ if len(a) > 1 {
+ panic("singleRange called for non-continuous rangeSet")
+ }
+ if len(a) == 1 {
+ return a[0]
+ }
+ return common.NewRange[T](0, 0)
+}
+
+func singleRangeSet[T uint32 | uint64](r common.Range[T]) rangeSet[T] {
+ if r.IsEmpty() {
+ return nil
+ }
+ return rangeSet[T]{r}
+}
+
+func (a rangeSet[T]) equal(b rangeSet[T]) bool {
+ if len(a) != len(b) {
+ return false
+ }
+ for i, r := range a {
+ if b[i] != r {
+ return false
+ }
+ }
+ return true
+}
diff --git a/core/filtermaps/math_test.go b/core/filtermaps/math_test.go
index a4c1609059..92d5abe97a 100644
--- a/core/filtermaps/math_test.go
+++ b/core/filtermaps/math_test.go
@@ -87,7 +87,7 @@ func TestPotentialMatches(t *testing.T) {
// split up into a list of rows if longer than allowed
var rows []FilterRow
for layerIndex := uint32(0); row != nil; layerIndex++ {
- maxLen := int(params.maxRowLength(layerIndex))
+ maxLen := int(params.getMaxRowLength(layerIndex))
if len(row) > maxLen {
rows = append(rows, row[:maxLen])
row = row[maxLen:]
diff --git a/core/filtermaps/memory_map.go b/core/filtermaps/memory_map.go
new file mode 100644
index 0000000000..516cdfd790
--- /dev/null
+++ b/core/filtermaps/memory_map.go
@@ -0,0 +1,171 @@
+// Copyright 2025 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package filtermaps
+
+import (
+ "slices"
+
+ "github.com/ethereum/go-ethereum/common"
+)
+
+// memoryMap is an in-memory representation of a filter map that represents rows
+// as linked lists and stores entries in a single slice.
+// memoryMap allows adding new elements and is used for rendering maps. Completed
+// maps can be transformed to an immutable finishedMap.
+type memoryMap struct {
+ entries []mmEntry // size = valuesPerMap
+ rows []mmRow // size = mapHeight
+ nextEntry uint32
+ blockPtrs []uint64
+ lastBlock lastBlockOfMap
+}
+
+// mmEntry is a linked list entry. The field next points to the next entry in the
+// entries slice. Zero singals the end of the list (entries[0] is always the first
+// entry of a list).
+type mmEntry struct {
+ value, next uint32
+}
+
+// mmRow stores the index of the first and last entry of a linked list in the
+// entries slice. If length is zero then first and last are not initialized.
+// Note that length could be determined by traversing the linked list but it is
+// explicitly stored because map rendering needs quick access to row length.
+type mmRow struct {
+ first, last, length uint32
+}
+
+// newMemoryMap creates an empty memoryMap.
+func (p *Params) newMemoryMap() *memoryMap {
+ return &memoryMap{
+ entries: make([]mmEntry, p.valuesPerMap),
+ rows: make([]mmRow, p.mapHeight),
+ }
+}
+
+// clone creates a copy of the map.
+func (m *memoryMap) clone() *memoryMap {
+ return &memoryMap{
+ //TODO no need to clone entries if one of the maps will never be changed further.
+ entries: slices.Clone(m.entries),
+ rows: slices.Clone(m.rows),
+ nextEntry: m.nextEntry,
+ blockPtrs: slices.Clone(m.blockPtrs),
+ lastBlock: m.lastBlock,
+ }
+}
+
+func (m *memoryMap) firstBlock() uint64 {
+ return m.lastBlock.number + 1 - uint64(len(m.blockPtrs))
+}
+
+func (m *memoryMap) blocks() common.Range[uint64] {
+ l := uint64(len(m.blockPtrs))
+ return common.NewRange[uint64](m.lastBlock.number+1-l, l)
+}
+
+// addToRow adds a new entry to the specified row.
+func (m *memoryMap) addToRow(rowIndex, value uint32) {
+ row := &m.rows[rowIndex]
+ if row.length == 0 {
+ row.first = m.nextEntry
+ } else {
+ m.entries[row.last].next = m.nextEntry
+ }
+ row.last = m.nextEntry
+ row.length++
+ m.entries[m.nextEntry].value = value
+ m.nextEntry++
+}
+
+// rowLength returns the length of a row.
+func (m *memoryMap) rowLength(rowIndex uint32) uint32 {
+ return m.rows[rowIndex].length
+}
+
+// getRow returns a row of the map, truncated if maxLen is smaller than the actual
+// row length.
+func (m *memoryMap) getRow(rowIndex, maxLen uint32) FilterRow {
+ row := m.rows[rowIndex]
+ length := min(row.length, maxLen)
+ res := make(FilterRow, length)
+ next := row.first
+ for i := range length {
+ entry := m.entries[next]
+ res[i], next = entry.value, entry.next
+ }
+ return res
+}
+
+// finishedMap is an immutable memory representation of a single filter map.
+// It is more compact and allows more efficient row lookup than memoryMap.
+// Note that it assumes params.mapHeight <= 2**16 which is checked in deriveFields.
+type finishedMap struct {
+ rowPtrs []uint16 // points to rowData index after end of row; 2**16 can wrap around to 0
+ rowData []uint32
+ blockPtrs []uint64
+ lastBlock lastBlockOfMap
+}
+
+// finished creates a new finishedMap from a memoryMap.
+func (m *memoryMap) finished() *finishedMap {
+ fm := &finishedMap{
+ rowPtrs: make([]uint16, len(m.rows)),
+ rowData: make([]uint32, m.nextEntry),
+ blockPtrs: slices.Clone(m.blockPtrs),
+ lastBlock: m.lastBlock,
+ }
+ var ptr uint16
+ for i, row := range m.rows {
+ next := row.first
+ for range row.length {
+ entry := m.entries[next]
+ fm.rowData[ptr] = entry.value
+ next = entry.next
+ ptr++
+ }
+ fm.rowPtrs[i] = ptr
+ }
+ return fm
+}
+
+func (fm *finishedMap) firstBlock() uint64 {
+ return fm.lastBlock.number + 1 - uint64(len(fm.blockPtrs))
+}
+
+func (fm *finishedMap) blocks() common.Range[uint64] {
+ l := uint64(len(fm.blockPtrs))
+ return common.NewRange[uint64](fm.lastBlock.number+1-l, l)
+}
+
+// getRow returns a row of the map, truncated if maxLen is smaller than the actual
+// row length. If the row has zero length then it returns a zero length slice.
+func (fm *finishedMap) getRow(rowIndex, maxLen uint32) FilterRow {
+ var start uint16
+ if rowIndex > 0 {
+ start = fm.rowPtrs[rowIndex-1]
+ }
+ // Note: uint16 subtraction ensures correct result if
+ // rowPtrs[rowIndex] has wrapped around to zero. For example if there are
+ // exactly 2**16 entries and last row has a length of 5 then
+ // rowPtr[0xFFFE] == 0xFFFB and rowPtr[0xFFFF] == 0x0000 and the uint16
+ // subtraction is 0x0000-0xFFFB == 0x0005.
+ // Also note that it is guaranteed by the filter map design that no single
+ // row can have a length of 0x10000.
+ length := fm.rowPtrs[rowIndex] - start
+ return FilterRow(fm.rowData[start : uint32(start)+min(maxLen, uint32(length))])
+}
diff --git a/core/filtermaps/test_helpers.go b/core/filtermaps/test_helpers.go
new file mode 100644
index 0000000000..095dac8445
--- /dev/null
+++ b/core/filtermaps/test_helpers.go
@@ -0,0 +1,226 @@
+// Copyright 2025 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package filtermaps
+
+import (
+ "math"
+ "math/rand"
+ "reflect"
+ "slices"
+ "testing"
+
+ "github.com/ethereum/go-ethereum/common"
+)
+
+const (
+ testFilterMapRowsCount = 1000
+ testFilterMapsCount = 100
+ testPointersCount = 1000
+ testMaxBlocksPerMap = 4
+)
+
+type mapReader struct {
+ getFilterMapRows func(mapIndices []uint32, rowIndex, layerIndex uint32) ([]FilterRow, error)
+ getFilterMap func(mapIndex uint32) (*finishedMap, error)
+ getBlockLvPointer func(blockNumber uint64) (uint64, error)
+ getLastBlockOfMap func(mapIndex uint32) (uint64, common.Hash, error)
+}
+
+func equalRows(a, b []FilterRow) bool {
+ if len(a) != len(b) {
+ return false
+ }
+ for i := range a {
+ if !slices.Equal(a[i], b[i]) {
+ return false
+ }
+ }
+ return true
+}
+
+func testMapReader(t *testing.T, testCase string, params *Params, reader mapReader, knownEpochs checkpointList, maps []*finishedMap) {
+ mapRange := common.NewRange[uint32](uint32(len(knownEpochs))*params.mapsPerEpoch, uint32(len(maps)))
+ for range testFilterMapRowsCount {
+ rowIndex := uint32(rand.Intn(int(params.mapHeight)))
+ lastDbLayer := uint32(rand.Intn(len(params.maxRowLength)))
+ maxLen := params.maxRowLength[lastDbLayer]
+ mid := mapRange.First() + uint32(rand.Intn(int(mapRange.Count())))
+ count := uint32(rand.Intn(int(params.mapsPerEpoch)))
+ first := max(mid, count/2) - count/2
+ testRange := common.NewRange[uint32](first, count)
+ mapIndices := make([]uint32, 0, testRange.Count())
+ expResults := make([]FilterRow, 0, testRange.Count())
+ for mapIndex := range testRange.Iter() {
+ if rand.Intn(2) == 1 {
+ mapIndices = append(mapIndices, mapIndex)
+ if mapRange.Includes(mapIndex) {
+ expResults = append(expResults, maps[mapIndex-mapRange.First()].getRow(rowIndex, maxLen))
+ } else {
+ expResults = append(expResults, nil)
+ }
+ }
+ }
+ rows, err := reader.getFilterMapRows(mapIndices, rowIndex, lastDbLayer)
+ if err != nil {
+ t.Fatalf("Test case %s: error reading %d map indices in range %d <= i < %d: %v", testCase, len(mapIndices), testRange.First(), testRange.AfterLast(), err)
+ } else if !equalRows(rows, expResults) {
+ t.Fatalf("Test case %s: incorrect results when reading %d map indices in range %d <= i < %d", testCase, len(mapIndices), testRange.First(), testRange.AfterLast())
+ }
+ }
+ for range testFilterMapsCount {
+ mapIndex := mapRange.First() + uint32(rand.Intn(int(mapRange.Count())))
+ expResults := maps[mapIndex-mapRange.First()]
+ fm, err := reader.getFilterMap(mapIndex)
+ if err != nil {
+ t.Fatalf("Test case %s: error reading map %d: %v", testCase, mapIndex, err)
+ } else if !reflect.DeepEqual(fm, expResults) {
+ t.Fatalf("Test case %s: incorrect results when reading map %d", testCase, mapIndex)
+ }
+ }
+
+ testPointers := func(mapIndex uint32, expLastBlock lastBlockOfMap, blockNumber, expBlockPtr uint64) {
+ lastBlock, lbHash, err := reader.getLastBlockOfMap(mapIndex)
+ if err != nil {
+ t.Fatalf("Test case %s: error reading last block of map %d: %v", testCase, mapIndex, err)
+ } else if (lastBlockOfMap{number: lastBlock, hash: lbHash}) != expLastBlock {
+ t.Fatalf("Test case %s: incorrect results when reading last block of map %d (expected %v, got %v)", testCase, mapIndex, expLastBlock, lastBlockOfMap{number: lastBlock, hash: lbHash})
+ }
+ if blockNumber != math.MaxUint64 {
+ blockPtr, err := reader.getBlockLvPointer(blockNumber)
+ if err != nil {
+ t.Fatalf("Test case %s: error reading lv pointer of block %d: %v", testCase, blockNumber, err)
+ } else if blockPtr != expBlockPtr {
+ t.Fatalf("Test case %s: incorrect results when reading lv pointer of block %d (expected %v, got %v)", testCase, blockNumber, expBlockPtr, blockPtr)
+ }
+ }
+ }
+ testNoPointer := func(mapIndex uint32) {
+ if _, _, err := reader.getLastBlockOfMap(mapIndex); err == nil {
+ t.Fatalf("Test case %s: unexpected last block of map pointer at map %d", testCase, mapIndex)
+ }
+ }
+ testKnownEpoch := func(epoch uint32) {
+ testNoPointer(params.firstEpochMap(epoch))
+ testPointers(params.lastEpochMap(epoch), lastBlockOfMap{number: knownEpochs[epoch].BlockNumber, hash: knownEpochs[epoch].BlockHash}, knownEpochs[epoch].BlockNumber, knownEpochs[epoch].FirstIndex)
+ }
+ if len(knownEpochs) > 0 {
+ for range testPointersCount {
+ epoch := uint32(rand.Intn(len(knownEpochs)))
+ testKnownEpoch(epoch)
+ }
+ testKnownEpoch(uint32(len(knownEpochs) - 1))
+ }
+ testMapPointers := func(mapIndex uint32, blockPos int) { // 0: first 1: random 2: last
+ fm := maps[mapIndex-mapRange.First()]
+ if len(fm.blockPtrs) == 0 {
+ testPointers(mapIndex, fm.lastBlock, math.MaxUint64, 0)
+ } else {
+ var subIndex int
+ switch blockPos {
+ case 1:
+ subIndex = rand.Intn(len(fm.blockPtrs))
+ case 2:
+ subIndex = len(fm.blockPtrs) - 1
+ }
+ testPointers(mapIndex, fm.lastBlock, fm.firstBlock()+uint64(subIndex), fm.blockPtrs[subIndex])
+ }
+ }
+ if !mapRange.IsEmpty() {
+ testMapPointers(mapRange.First(), 0)
+ for range testPointersCount {
+ testMapPointers(mapRange.First()+uint32(rand.Intn(int(mapRange.Count()))), 1)
+ }
+ testMapPointers(mapRange.Last(), 2)
+ }
+ testNoPointer(mapRange.AfterLast())
+ testNoPointer(params.lastEpochMap(params.mapEpoch(mapRange.AfterLast())))
+ testNoPointer(params.firstEpochMap(params.mapEpoch(mapRange.AfterLast()) + 1))
+}
+
+func generateTestMaps(params *Params, maps []*finishedMap, amount uint32) []*finishedMap {
+ var lastBlock lastBlockOfMap
+ genesis := len(maps) == 0
+ if !genesis {
+ lastBlock = maps[len(maps)-1].lastBlock
+ }
+ for range amount {
+ blockCount := rand.Intn(testMaxBlocksPerMap + 1)
+ if genesis && blockCount == 0 {
+ blockCount = 1
+ }
+ fm := &finishedMap{
+ rowPtrs: make([]uint16, params.mapHeight),
+ rowData: make([]uint32, params.valuesPerMap),
+ blockPtrs: make([]uint64, blockCount),
+ }
+ addValues := params.valuesPerMap
+ for addValues > 0 {
+ addToRow := uint64(rand.Intn(int(min(addValues+1, params.valuesPerMap/2))))
+ addValues -= addToRow
+ var rowIndex uint32
+ for {
+ rowIndex = uint32(rand.Intn(int(params.mapHeight)))
+ if uint64(fm.rowPtrs[rowIndex])+addToRow <= params.valuesPerMap/2 {
+ break
+ }
+ }
+ fm.rowPtrs[rowIndex] += uint16(addToRow)
+ }
+ var ptr uint16
+ for i, r := range fm.rowPtrs {
+ ptr += r
+ fm.rowPtrs[i] = ptr
+ }
+ for i := range fm.rowData {
+ fm.rowData[i] = uint32(rand.Intn(1 << params.logMapWidth))
+ }
+ if blockCount > 0 {
+ startPtr := uint64(len(maps)) * params.valuesPerMap
+ for i := range fm.blockPtrs {
+ if genesis {
+ genesis = false
+ } else {
+ lastBlock.number++
+ fm.blockPtrs[i] = startPtr + uint64(i)*params.valuesPerMap/uint64(blockCount) + uint64(rand.Intn(int(params.valuesPerMap)/blockCount/2))
+ }
+ }
+ rand.Read(lastBlock.hash[:])
+ }
+ fm.lastBlock = lastBlock
+ maps = append(maps, fm)
+ }
+ return maps
+}
+
+func generateEpochCheckpoint(mapIndex uint32, maps []*finishedMap) epochCheckpoint {
+ for len(maps[mapIndex].blockPtrs) == 0 {
+ mapIndex--
+ }
+ return epochCheckpoint{
+ BlockNumber: maps[mapIndex].lastBlock.number,
+ BlockHash: maps[mapIndex].lastBlock.hash,
+ FirstIndex: maps[mapIndex].blockPtrs[len(maps[mapIndex].blockPtrs)-1],
+ }
+}
+
+func generateTestCheckpoints(params *Params, maps []*finishedMap) checkpointList {
+ cpList := make(checkpointList, uint32(len(maps))/params.mapsPerEpoch)
+ for i := range cpList {
+ cpList[i] = generateEpochCheckpoint(params.lastEpochMap(uint32(i)), maps)
+ }
+ return cpList
+}
diff --git a/core/rawdb/accessors_indexes.go b/core/rawdb/accessors_indexes.go
index a725f144d4..32f44e1c8d 100644
--- a/core/rawdb/accessors_indexes.go
+++ b/core/rawdb/accessors_indexes.go
@@ -320,12 +320,12 @@ func ReadCanonicalRawReceipt(db ethdb.Reader, blockHash common.Hash, blockNumber
// same data proximity reasons it is also suitable for database representation.
// See also:
// https://eips.ethereum.org/EIPS/eip-7745#hash-tree-structure
-func ReadFilterMapExtRow(db ethdb.KeyValueReader, mapRowIndex uint64, bitLength uint) ([]uint32, error) {
+func ReadFilterMapSingleRow(db ethdb.KeyValueReader, mapRowIndex uint64, dbLayer uint32, bitLength uint) ([]uint32, error) {
byteLength := int(bitLength) / 8
if int(bitLength) != byteLength*8 {
panic("invalid bit length")
}
- key := filterMapRowKey(mapRowIndex, false)
+ key := filterMapRowKey(mapRowIndex, dbLayer)
has, err := db.Has(key)
if err != nil {
return nil, err
@@ -349,18 +349,19 @@ func ReadFilterMapExtRow(db ethdb.KeyValueReader, mapRowIndex uint64, bitLength
return row, nil
}
-func ReadFilterMapBaseRows(db ethdb.KeyValueReader, mapRowIndex uint64, rowCount uint32, bitLength uint) ([][]uint32, error) {
+func ReadFilterMapRowGroup(db ethdb.KeyValueReader, mapRowIndex uint64, dbLayer, rowCount uint32, bitLength uint) ([][]uint32, error) {
byteLength := int(bitLength) / 8
if int(bitLength) != byteLength*8 {
panic("invalid bit length")
}
- key := filterMapRowKey(mapRowIndex, true)
+ key := filterMapRowKey(mapRowIndex, dbLayer)
has, err := db.Has(key)
if err != nil {
return nil, err
}
rows := make([][]uint32, rowCount)
if !has {
+ //fmt.Println(" no entry")
return rows, nil
}
encRows, err := db.Get(key)
@@ -368,6 +369,7 @@ func ReadFilterMapBaseRows(db ethdb.KeyValueReader, mapRowIndex uint64, rowCount
return nil, err
}
encLen := len(encRows)
+ //fmt.Println(" encLen", encLen)
var (
entryCount, entriesInRow, rowIndex, headerLen, headerBits int
headerByte byte
@@ -411,7 +413,7 @@ func ReadFilterMapBaseRows(db ethdb.KeyValueReader, mapRowIndex uint64, rowCount
// WriteFilterMapExtRow stores an extended filter map row at the given mapRowIndex
// or deletes any existing entry if the row is empty.
-func WriteFilterMapExtRow(db ethdb.KeyValueWriter, mapRowIndex uint64, row []uint32, bitLength uint) {
+func WriteFilterMapSingleRow(db ethdb.KeyValueWriter, mapRowIndex uint64, dbLayer uint32, row []uint32, bitLength uint) {
byteLength := int(bitLength) / 8
if int(bitLength) != byteLength*8 {
panic("invalid bit length")
@@ -424,16 +426,24 @@ func WriteFilterMapExtRow(db ethdb.KeyValueWriter, mapRowIndex uint64, row []uin
binary.LittleEndian.PutUint32(b[:], c)
copy(encRow[i*byteLength:(i+1)*byteLength], b[:byteLength])
}
- err = db.Put(filterMapRowKey(mapRowIndex, false), encRow)
+ err = db.Put(filterMapRowKey(mapRowIndex, dbLayer), encRow)
} else {
- err = db.Delete(filterMapRowKey(mapRowIndex, false))
+ err = db.Delete(filterMapRowKey(mapRowIndex, dbLayer))
}
if err != nil {
log.Crit("Failed to store extended filter map row", "err", err)
}
}
-func WriteFilterMapBaseRows(db ethdb.KeyValueWriter, mapRowIndex uint64, rows [][]uint32, bitLength uint) {
+func WriteFilterMapRowGroup(db ethdb.KeyValueWriter, mapRowIndex uint64, dbLayer uint32, rows [][]uint32, bitLength uint) {
+ /*var totalsize int //TODO
+ for _, row := range rows {
+ totalsize += len(row)
+ }
+ if totalsize > 0 {
+ fmt.Println("WriteFilterMapRowGroup", mapRowIndex, dbLayer, totalsize)
+ }*/
+
byteLength := int(bitLength) / 8
if int(bitLength) != byteLength*8 {
panic("invalid bit length")
@@ -476,9 +486,10 @@ func WriteFilterMapBaseRows(db ethdb.KeyValueWriter, mapRowIndex uint64, rows []
addHeaderBit(false)
zeroBits--
}
- err = db.Put(filterMapRowKey(mapRowIndex, true), encRows)
+ //fmt.Println(" write encLen", len(encRows))
+ err = db.Put(filterMapRowKey(mapRowIndex, dbLayer), encRows)
} else {
- err = db.Delete(filterMapRowKey(mapRowIndex, true))
+ err = db.Delete(filterMapRowKey(mapRowIndex, dbLayer))
}
if err != nil {
log.Crit("Failed to store base filter map rows", "err", err)
@@ -486,7 +497,7 @@ func WriteFilterMapBaseRows(db ethdb.KeyValueWriter, mapRowIndex uint64, rows []
}
func DeleteFilterMapRows(db ethdb.KeyValueStore, mapRows common.Range[uint64], hashScheme bool, stopCallback func(bool) bool) error {
- return SafeDeleteRange(db, filterMapRowKey(mapRows.First(), false), filterMapRowKey(mapRows.AfterLast(), false), hashScheme, stopCallback)
+ return SafeDeleteRange(db, filterMapRowKey(mapRows.First(), 0), filterMapRowKey(mapRows.AfterLast(), 0), hashScheme, stopCallback)
}
// ReadFilterMapLastBlock retrieves the number of the block that generated the
@@ -565,12 +576,9 @@ func DeleteBlockLvPointers(db ethdb.KeyValueStore, blocks common.Range[uint64],
// FilterMapsRange is a storage representation of the block range covered by the
// filter maps structure and the corresponting log value index range.
type FilterMapsRange struct {
- Version uint32
- HeadIndexed bool
- HeadDelimiter uint64
- BlocksFirst, BlocksAfterLast uint64
- MapsFirst, MapsAfterLast uint32
- TailPartialEpoch uint32
+ Version uint32
+ ValidMaps, DirtyMaps []uint32
+ KnownEpochs uint32
}
// ReadFilterMapsRange retrieves the filter maps range data. Note that if the
diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go
index d9140c5fd6..b01e800f28 100644
--- a/core/rawdb/schema.go
+++ b/core/rawdb/schema.go
@@ -152,7 +152,7 @@ var (
SyncCommitteeKey = []byte("committee-") // bigEndian64(syncPeriod) -> serialized committee
// new log index
- filterMapsPrefix = "fm-"
+ filterMapsPrefix = "fm*" //TODO fm-
filterMapsRangeKey = []byte(filterMapsPrefix + "R")
filterMapRowPrefix = []byte(filterMapsPrefix + "r") // filterMapRowPrefix + mapRowIndex (uint64 big endian) -> filter row
filterMapLastBlockPrefix = []byte(filterMapsPrefix + "b") // filterMapLastBlockPrefix + mapIndex (uint32 big endian) -> block number (uint64 big endian)
@@ -353,15 +353,12 @@ func IsStorageTrieNode(key []byte) bool {
}
// filterMapRowKey = filterMapRowPrefix + mapRowIndex (uint64 big endian)
-func filterMapRowKey(mapRowIndex uint64, base bool) []byte {
- extLen := 8
- if base {
- extLen = 9
- }
+func filterMapRowKey(mapRowIndex uint64, dbLayer uint32) []byte {
l := len(filterMapRowPrefix)
- key := make([]byte, l+extLen)
+ key := make([]byte, l+9)
copy(key[:l], filterMapRowPrefix)
binary.BigEndian.PutUint64(key[l:l+8], mapRowIndex)
+ key[l+8] = byte(dbLayer)
return key
}
diff --git a/eth/api_backend.go b/eth/api_backend.go
index 766a99fc1e..50af177dbe 100644
--- a/eth/api_backend.go
+++ b/eth/api_backend.go
@@ -459,7 +459,7 @@ func (b *EthAPIBackend) RPCTxFeeCap() float64 {
return b.eth.config.RPCTxFeeCap
}
-func (b *EthAPIBackend) CurrentView() *filtermaps.ChainView {
+func (b *EthAPIBackend) CurrentChainView() *filtermaps.ChainView {
head := b.eth.blockchain.CurrentBlock()
if head == nil {
return nil
@@ -467,8 +467,8 @@ func (b *EthAPIBackend) CurrentView() *filtermaps.ChainView {
return filtermaps.NewChainView(b.eth.blockchain, head.Number.Uint64(), head.Hash())
}
-func (b *EthAPIBackend) NewMatcherBackend() filtermaps.MatcherBackend {
- return b.eth.filterMaps.NewMatcherBackend()
+func (b *EthAPIBackend) GetIndexView(headBlockHash common.Hash) *filtermaps.IndexView {
+ return b.eth.logIndexer.GetIndexView(headBlockHash)
}
func (b *EthAPIBackend) Engine() consensus.Engine {
diff --git a/eth/backend.go b/eth/backend.go
index 8509561822..ee5bb7330f 100644
--- a/eth/backend.go
+++ b/eth/backend.go
@@ -108,8 +108,7 @@ type Ethereum struct {
engine consensus.Engine
accountManager *accounts.Manager
- filterMaps *filtermaps.FilterMaps
- closeFilterMaps chan chan struct{}
+ logIndexer *filtermaps.Indexer
APIBackend *EthAPIBackend
@@ -285,18 +284,8 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
ExportFileName: config.LogExportCheckpoints,
HashScheme: scheme == rawdb.HashScheme,
}
- chainView := eth.newChainView(eth.blockchain.CurrentBlock())
- historyCutoff, _ := eth.blockchain.HistoryPruningCutoff()
- var finalBlock uint64
- if fb := eth.blockchain.CurrentFinalBlock(); fb != nil {
- finalBlock = fb.Number.Uint64()
- }
- filterMaps, err := filtermaps.NewFilterMaps(chainDb, chainView, historyCutoff, finalBlock, filtermaps.DefaultParams, fmConfig)
- if err != nil {
- return nil, err
- }
- eth.filterMaps = filterMaps
- eth.closeFilterMaps = make(chan chan struct{})
+ eth.logIndexer = filtermaps.NewIndexer(chainDb, filtermaps.DefaultParams, fmConfig)
+ eth.blockchain.RegisterIndexer(eth.logIndexer, "log index")
// TxPool
if config.TxPool.Journal != "" {
@@ -452,71 +441,9 @@ func (s *Ethereum) Start() error {
// Start the connection manager
s.dropper.Start(s.p2pServer, func() bool { return !s.Synced() })
-
- // start log indexer
- s.filterMaps.Start()
- go s.updateFilterMapsHeads()
return nil
}
-func (s *Ethereum) newChainView(head *types.Header) *filtermaps.ChainView {
- if head == nil {
- return nil
- }
- return filtermaps.NewChainView(s.blockchain, head.Number.Uint64(), head.Hash())
-}
-
-func (s *Ethereum) updateFilterMapsHeads() {
- headEventCh := make(chan core.ChainEvent, 10)
- blockProcCh := make(chan bool, 10)
- sub := s.blockchain.SubscribeChainEvent(headEventCh)
- sub2 := s.blockchain.SubscribeBlockProcessingEvent(blockProcCh)
- defer func() {
- sub.Unsubscribe()
- sub2.Unsubscribe()
- for {
- select {
- case <-headEventCh:
- case <-blockProcCh:
- default:
- return
- }
- }
- }()
-
- var head *types.Header
- setHead := func(newHead *types.Header) {
- if newHead == nil {
- return
- }
- if head == nil || newHead.Hash() != head.Hash() {
- head = newHead
- chainView := s.newChainView(head)
- historyCutoff, _ := s.blockchain.HistoryPruningCutoff()
- var finalBlock uint64
- if fb := s.blockchain.CurrentFinalBlock(); fb != nil {
- finalBlock = fb.Number.Uint64()
- }
- s.filterMaps.SetTarget(chainView, historyCutoff, finalBlock)
- }
- }
- setHead(s.blockchain.CurrentBlock())
-
- for {
- select {
- case ev := <-headEventCh:
- setHead(ev.Header)
- case blockProc := <-blockProcCh:
- s.filterMaps.SetBlockProcessing(blockProc)
- case <-time.After(time.Second * 10):
- setHead(s.blockchain.CurrentBlock())
- case ch := <-s.closeFilterMaps:
- close(ch)
- return
- }
- }
-}
-
func (s *Ethereum) setupDiscovery() error {
eth.StartENRUpdater(s.blockchain, s.p2pServer.LocalNode())
@@ -575,10 +502,6 @@ func (s *Ethereum) Stop() error {
s.handler.Stop()
// Then stop everything else.
- ch := make(chan struct{})
- s.closeFilterMaps <- ch
- <-ch
- s.filterMaps.Stop()
s.txPool.Close()
s.blockchain.Stop()
s.engine.Close()
diff --git a/eth/filters/filter.go b/eth/filters/filter.go
index 422e5cd67b..a8e3396896 100644
--- a/eth/filters/filter.go
+++ b/eth/filters/filter.go
@@ -19,9 +19,11 @@ package filters
import (
"context"
"errors"
+ "fmt"
"math"
"math/big"
"slices"
+ "sync"
"time"
"github.com/ethereum/go-ethereum/common"
@@ -43,7 +45,12 @@ type Filter struct {
block *common.Hash // Block hash if filtering a single block
begin, end int64 // Range interval if filtering multiple blocks
- rangeLogsTestHook chan rangeLogsTestEvent
+ testFilterRanges []testFilterRange
+}
+
+type testFilterRange struct {
+ begin, end uint64
+ indexed bool
}
// NewRangeFilter creates a new filter which uses a bloom filter on blocks to
@@ -146,272 +153,101 @@ func (f *Filter) Logs(ctx context.Context) ([]*types.Log, error) {
return f.rangeLogs(ctx, begin, end)
}
-const (
- rangeLogsTestDone = iota // zero range
- rangeLogsTestSync // before sync; zero range
- rangeLogsTestSynced // after sync; valid blocks range
- rangeLogsTestIndexed // individual search range
- rangeLogsTestUnindexed // individual search range
- rangeLogsTestResults // results range after search iteration
- rangeLogsTestReorg // results range trimmed by reorg
-)
-
-type rangeLogsTestEvent struct {
- event int
- blocks common.Range[uint64]
-}
-
-// searchSession represents a single search session.
-type searchSession struct {
- ctx context.Context
- filter *Filter
- mb filtermaps.MatcherBackend
- syncRange filtermaps.SyncRange // latest synchronized state with the matcher
- chainView *filtermaps.ChainView // can be more recent than the indexed view in syncRange
- // block ranges always refer to the current chainView
- firstBlock, lastBlock uint64 // specified search range; MaxUint64 means latest block
- searchRange common.Range[uint64] // actual search range; end trimmed to latest head
- matchRange common.Range[uint64] // range in which we have results (subset of searchRange)
- matches []*types.Log // valid set of matches in matchRange
- forceUnindexed bool // revert to unindexed search
-}
-
-// newSearchSession returns a new searchSession.
-func newSearchSession(ctx context.Context, filter *Filter, mb filtermaps.MatcherBackend, firstBlock, lastBlock uint64) (*searchSession, error) {
- s := &searchSession{
- ctx: ctx,
- filter: filter,
- mb: mb,
- firstBlock: firstBlock,
- lastBlock: lastBlock,
- }
- // enforce a consistent state before starting the search in order to be able
- // to determine valid range later
- var err error
- s.syncRange, err = s.mb.SyncLogIndex(s.ctx)
- if err != nil {
- return nil, err
- }
- if err := s.updateChainView(); err != nil {
- return nil, err
- }
- return s, nil
-}
-
-// updateChainView updates to the latest view of the underlying chain and sets
-// searchRange by replacing MaxUint64 (meaning latest block) with actual head
-// number in the specified search range.
-// If the session already had an existing chain view and set of matches then
-// it also trims part of the match set that a chain reorg might have invalidated.
-func (s *searchSession) updateChainView() error {
- // update chain view based on current chain head (might be more recent than
- // the indexed view of syncRange as the indexer updates it asynchronously
- // with some delay
- newChainView := s.filter.sys.backend.CurrentView()
- if newChainView == nil {
- return errors.New("head block not available")
- }
- head := newChainView.HeadNumber()
-
- // update actual search range based on current head number
- firstBlock, lastBlock := s.firstBlock, s.lastBlock
- if firstBlock == math.MaxUint64 {
- firstBlock = head
- }
- if lastBlock == math.MaxUint64 {
- lastBlock = head
- }
- if firstBlock > lastBlock || lastBlock > head {
- return errInvalidBlockRange
- }
- s.searchRange = common.NewRange(firstBlock, lastBlock+1-firstBlock)
-
- // Trim existing match set in case a reorg may have invalidated some results
- if !s.matchRange.IsEmpty() {
- trimRange := newChainView.SharedRange(s.chainView).Intersection(s.searchRange)
- s.matchRange, s.matches = s.trimMatches(trimRange, s.matchRange, s.matches)
- }
- s.chainView = newChainView
- return nil
-}
-
-// trimMatches removes any entries from the specified set of matches that is
-// outside the given range.
-func (s *searchSession) trimMatches(trimRange, matchRange common.Range[uint64], matches []*types.Log) (common.Range[uint64], []*types.Log) {
- newRange := matchRange.Intersection(trimRange)
- if newRange == matchRange {
- return matchRange, matches
- }
- if newRange.IsEmpty() {
- return newRange, nil
- }
- for len(matches) > 0 && matches[0].BlockNumber < newRange.First() {
- matches = matches[1:]
- }
- for len(matches) > 0 && matches[len(matches)-1].BlockNumber > newRange.Last() {
- matches = matches[:len(matches)-1]
- }
- return newRange, matches
-}
-
-// searchInRange performs a single range search, either indexed or unindexed.
-func (s *searchSession) searchInRange(r common.Range[uint64], indexed bool) (common.Range[uint64], []*types.Log, error) {
- if indexed {
- if s.filter.rangeLogsTestHook != nil {
- s.filter.rangeLogsTestHook <- rangeLogsTestEvent{rangeLogsTestIndexed, r}
- }
- results, err := s.filter.indexedLogs(s.ctx, s.mb, r.First(), r.Last())
- if err != nil && !errors.Is(err, filtermaps.ErrMatchAll) {
- return common.Range[uint64]{}, nil, err
- }
- if err == nil {
- // sync with filtermaps matcher
- if s.filter.rangeLogsTestHook != nil {
- s.filter.rangeLogsTestHook <- rangeLogsTestEvent{rangeLogsTestSync, common.Range[uint64]{}}
- }
- var syncErr error
- if s.syncRange, syncErr = s.mb.SyncLogIndex(s.ctx); syncErr != nil {
- return common.Range[uint64]{}, nil, syncErr
- }
- if s.filter.rangeLogsTestHook != nil {
- s.filter.rangeLogsTestHook <- rangeLogsTestEvent{rangeLogsTestSynced, s.syncRange.ValidBlocks}
- }
- // discard everything that might be invalid
- trimRange := s.syncRange.ValidBlocks.Intersection(s.chainView.SharedRange(s.syncRange.IndexedView))
- matchRange, matches := s.trimMatches(trimRange, r, results)
- return matchRange, matches, nil
- }
- // "match all" filters are not supported by filtermaps; fall back to
- // unindexed search which is the most efficient in this case
- s.forceUnindexed = true
- // fall through to unindexed case
- }
- if s.filter.rangeLogsTestHook != nil {
- s.filter.rangeLogsTestHook <- rangeLogsTestEvent{rangeLogsTestUnindexed, r}
- }
- matches, err := s.filter.unindexedLogs(s.ctx, s.chainView, r.First(), r.Last())
- if err != nil {
- return common.Range[uint64]{}, nil, err
- }
- return r, matches, nil
-}
-
-// doSearchIteration performs a search on a range missing from an incomplete set
-// of results, adds the new section and removes invalidated entries.
-func (s *searchSession) doSearchIteration() error {
- switch {
- case s.matchRange.IsEmpty():
- // no results yet; try search in entire range
- indexedSearchRange := s.searchRange.Intersection(s.syncRange.IndexedBlocks)
- if s.forceUnindexed = indexedSearchRange.IsEmpty(); !s.forceUnindexed {
- // indexed search on the intersection of indexed and searched range
- matchRange, matches, err := s.searchInRange(indexedSearchRange, true)
- if err != nil {
- return err
- }
- s.matchRange = matchRange
- s.matches = matches
- return nil
- } else {
- // no intersection of indexed and searched range; unindexed search on
- // the whole searched range
- matchRange, matches, err := s.searchInRange(s.searchRange, false)
- if err != nil {
- return err
- }
- s.matchRange = matchRange
- s.matches = matches
- return nil
- }
-
- case !s.matchRange.IsEmpty() && s.matchRange.First() > s.searchRange.First():
- // Results are available, but the tail section is missing. Perform an unindexed
- // search for the missing tail, while still allowing indexed search for the head.
- //
- // The unindexed search is necessary because the tail portion of the indexes
- // has been pruned.
- tailRange := common.NewRange(s.searchRange.First(), s.matchRange.First()-s.searchRange.First())
- _, tailMatches, err := s.searchInRange(tailRange, false)
- if err != nil {
- return err
- }
- s.matches = append(tailMatches, s.matches...)
- s.matchRange = tailRange.Union(s.matchRange)
- return nil
-
- case !s.matchRange.IsEmpty() && s.matchRange.First() == s.searchRange.First() && s.searchRange.AfterLast() > s.matchRange.AfterLast():
- // Results are available, but the head section is missing. Try to perform
- // the indexed search for the missing head, or fallback to unindexed search
- // if the tail portion of indexed range has been pruned.
- headRange := common.NewRange(s.matchRange.AfterLast(), s.searchRange.AfterLast()-s.matchRange.AfterLast())
- if !s.forceUnindexed {
- indexedHeadRange := headRange.Intersection(s.syncRange.IndexedBlocks)
- if !indexedHeadRange.IsEmpty() && indexedHeadRange.First() == headRange.First() {
- headRange = indexedHeadRange
- } else {
- // The tail portion of the indexes has been pruned, falling back
- // to unindexed search.
- s.forceUnindexed = true
- }
- }
- headMatchRange, headMatches, err := s.searchInRange(headRange, !s.forceUnindexed)
- if err != nil {
- return err
- }
- if headMatchRange.First() != s.matchRange.AfterLast() {
- // improbable corner case, first part of new head range invalidated by tail unindexing
- s.matches, s.matchRange = headMatches, headMatchRange
- return nil
- }
- s.matches = append(s.matches, headMatches...)
- s.matchRange = s.matchRange.Union(headMatchRange)
- return nil
-
- default:
- panic("invalid search session state")
- }
-}
-
func (f *Filter) rangeLogs(ctx context.Context, firstBlock, lastBlock uint64) ([]*types.Log, error) {
- if f.rangeLogsTestHook != nil {
- defer func() {
- f.rangeLogsTestHook <- rangeLogsTestEvent{rangeLogsTestDone, common.Range[uint64]{}}
- close(f.rangeLogsTestHook)
- }()
- }
-
if firstBlock > lastBlock {
return nil, nil
}
- mb := f.sys.backend.NewMatcherBackend()
- defer mb.Close()
+ chainView := f.sys.backend.CurrentChainView()
- session, err := newSearchSession(ctx, f, mb, firstBlock, lastBlock)
+ if firstBlock > chainView.HeadNumber() {
+ firstBlock = chainView.HeadNumber()
+ }
+ if lastBlock > chainView.HeadNumber() {
+ lastBlock = chainView.HeadNumber()
+ }
+
+ indexView := f.sys.backend.GetIndexView(chainView.BlockHash(chainView.HeadNumber()))
+ if indexView == nil {
+ return f.unindexedLogs(ctx, chainView, firstBlock, lastBlock)
+ }
+ searchRange := common.NewRange[uint64](firstBlock, lastBlock+1-firstBlock)
+ indexedRange := indexView.BlockRange().Intersection(searchRange)
+ if indexedRange.IsEmpty() {
+ indexView.Release()
+ return f.unindexedLogs(ctx, chainView, firstBlock, lastBlock)
+ }
+ var (
+ res1, res2, res3 []*types.Log
+ err error
+ )
+ res2, err = f.indexedLogs(ctx, chainView, indexView, indexedRange.First(), indexedRange.Last())
+ indexView.Release()
+ if err == filtermaps.ErrMatchAll {
+ return f.unindexedLogs(ctx, chainView, firstBlock, lastBlock)
+ }
if err != nil {
return nil, err
}
- for session.searchRange != session.matchRange {
- if err := session.doSearchIteration(); err != nil {
+ if searchRange.First() < indexedRange.First() {
+ res1, err = f.unindexedLogs(ctx, chainView, searchRange.First(), indexedRange.First()-1)
+ if err != nil {
return nil, err
}
- if f.rangeLogsTestHook != nil {
- f.rangeLogsTestHook <- rangeLogsTestEvent{rangeLogsTestResults, session.matchRange}
- }
- mr := session.matchRange
- if err := session.updateChainView(); err != nil {
- return nil, err
- }
- if f.rangeLogsTestHook != nil && session.matchRange != mr {
- f.rangeLogsTestHook <- rangeLogsTestEvent{rangeLogsTestReorg, session.matchRange}
- }
}
- return session.matches, nil
+ if indexedRange.Last() < searchRange.Last() {
+ res3, err = f.unindexedLogs(ctx, chainView, indexedRange.AfterLast(), searchRange.Last())
+ if err != nil {
+ return nil, err
+ }
+ }
+ return append(append(res1, res2...), res3...), nil
}
-func (f *Filter) indexedLogs(ctx context.Context, mb filtermaps.MatcherBackend, begin, end uint64) ([]*types.Log, error) {
+func (f *Filter) indexedLogs(ctx context.Context, chainView *filtermaps.ChainView, indexView *filtermaps.IndexView, begin, end uint64) ([]*types.Log, error) {
+ if f.testFilterRanges != nil {
+ f.testFilterRanges = append(f.testFilterRanges, testFilterRange{begin: begin, end: end, indexed: true})
+ }
start := time.Now()
- potentialMatches, err := filtermaps.GetPotentialMatches(ctx, mb, begin, end, f.addresses, f.topics)
- matches := filterLogs(potentialMatches, nil, nil, f.addresses, f.topics)
+ potentialMatches, err := filtermaps.GetPotentialMatches(ctx, indexView, begin, end, f.addresses, f.topics)
+ potentialLogs := make([]*types.Log, len(potentialMatches))
+ indexCh := make(chan int)
+ errCh := make(chan error, 1)
+ var wg sync.WaitGroup
+
+ worker := func() {
+ defer wg.Done()
+ for i := range indexCh {
+ logPosition := potentialMatches[i]
+ receipts := chainView.Receipts(logPosition.BlockNumber)
+ if receipts == nil {
+ select {
+ case errCh <- fmt.Errorf("receipts for block #%d not found", logPosition.BlockNumber):
+ default:
+ }
+ return
+ }
+ log, err := logPosition.GetLog(receipts)
+ if err != nil {
+ select {
+ case errCh <- err:
+ default:
+ }
+ return
+ }
+ potentialLogs[i] = log
+ }
+ }
+
+ for range 4 { //TODO
+ wg.Add(1)
+ go worker()
+ }
+ for i := range potentialMatches {
+ indexCh <- i
+ }
+ close(indexCh)
+ wg.Wait()
+ matches := filterLogs(potentialLogs, nil, nil, f.addresses, f.topics)
log.Trace("Performed indexed log search", "begin", begin, "end", end, "true matches", len(matches), "false positives", len(potentialMatches)-len(matches), "elapsed", common.PrettyDuration(time.Since(start)))
return matches, err
}
@@ -419,6 +255,9 @@ func (f *Filter) indexedLogs(ctx context.Context, mb filtermaps.MatcherBackend,
// unindexedLogs returns the logs matching the filter criteria based on raw block
// iteration and bloom matching.
func (f *Filter) unindexedLogs(ctx context.Context, chainView *filtermaps.ChainView, begin, end uint64) ([]*types.Log, error) {
+ if f.testFilterRanges != nil {
+ f.testFilterRanges = append(f.testFilterRanges, testFilterRange{begin: begin, end: end, indexed: false})
+ }
start := time.Now()
log.Debug("Performing unindexed log search", "begin", begin, "end", end)
var matches []*types.Log
@@ -517,7 +356,7 @@ func filterLogs(logs []*types.Log, fromBlock, toBlock *big.Int, addresses []comm
}
var ret []*types.Log
for _, log := range logs {
- if check(log) {
+ if log != nil && check(log) {
ret = append(ret, log)
}
}
diff --git a/eth/filters/filter_system.go b/eth/filters/filter_system.go
index f10e6a277b..3cada008bf 100644
--- a/eth/filters/filter_system.go
+++ b/eth/filters/filter_system.go
@@ -72,8 +72,8 @@ type Backend interface {
SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription
SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription
- CurrentView() *filtermaps.ChainView
- NewMatcherBackend() filtermaps.MatcherBackend
+ CurrentChainView() *filtermaps.ChainView
+ GetIndexView(headBlockHash common.Hash) *filtermaps.IndexView
}
// FilterSystem holds resources shared by all filters.
diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go
index e5a1a2b25f..8eb9f76c8e 100644
--- a/eth/filters/filter_system_test.go
+++ b/eth/filters/filter_system_test.go
@@ -42,7 +42,7 @@ import (
type testBackend struct {
db ethdb.Database
- fm *filtermaps.FilterMaps
+ logIndexer *filtermaps.Indexer
txFeed event.Feed
logsFeed event.Feed
rmLogsFeed event.Feed
@@ -160,31 +160,51 @@ func (b *testBackend) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subsc
return b.chainFeed.Subscribe(ch)
}
-func (b *testBackend) CurrentView() *filtermaps.ChainView {
+func (b *testBackend) CurrentChainView() *filtermaps.ChainView {
head := b.CurrentBlock()
return filtermaps.NewChainView(b, head.Number.Uint64(), head.Hash())
}
-func (b *testBackend) NewMatcherBackend() filtermaps.MatcherBackend {
- return b.fm.NewMatcherBackend()
+func (b *testBackend) GetIndexView(headBlockHash common.Hash) *filtermaps.IndexView {
+ return b.logIndexer.GetIndexView(headBlockHash)
}
func (b *testBackend) startFilterMaps(history uint64, disabled bool, params filtermaps.Params) {
head := b.CurrentBlock()
- chainView := filtermaps.NewChainView(b, head.Number.Uint64(), head.Hash())
config := filtermaps.Config{
History: history,
Disabled: disabled,
ExportFileName: "",
}
- b.fm, _ = filtermaps.NewFilterMaps(b.db, chainView, 0, 0, params, config)
- b.fm.Start()
- b.fm.WaitIdle()
+ b.logIndexer = filtermaps.NewIndexer(b.db, params, config)
+ if !disabled {
+ b.indexHead(head.Number.Uint64())
+ }
+}
+
+func (b *testBackend) indexHead(head uint64) {
+ header, _ := b.HeaderByNumber(context.Background(), rpc.BlockNumber(head))
+ receipts, _ := b.GetReceipts(context.Background(), header.Hash())
+ b.logIndexer.AddBlockData([]*types.Header{header}, []types.Receipts{receipts})
+ b.backfillBlocks()
+}
+
+func (b *testBackend) backfillBlocks() {
+ ready, needBlocks := b.logIndexer.Status()
+ for !ready || !needBlocks.IsEmpty() {
+ for !ready {
+ time.Sleep(time.Millisecond)
+ ready, needBlocks = b.logIndexer.Status()
+ }
+ header, _ := b.HeaderByNumber(context.Background(), rpc.BlockNumber(needBlocks.First()))
+ receipts, _ := b.GetReceipts(context.Background(), header.Hash())
+ ready, needBlocks = b.logIndexer.AddBlockData([]*types.Header{header}, []types.Receipts{receipts})
+ }
}
func (b *testBackend) stopFilterMaps() {
- b.fm.Stop()
- b.fm = nil
+ b.logIndexer.Stop()
+ b.logIndexer = nil
}
func (b *testBackend) setPending(block *types.Block, receipts types.Receipts) {
diff --git a/eth/filters/filter_test.go b/eth/filters/filter_test.go
index edec3e027f..64a5032657 100644
--- a/eth/filters/filter_test.go
+++ b/eth/filters/filter_test.go
@@ -20,6 +20,7 @@ import (
"context"
"encoding/json"
"math/big"
+ "slices"
"strings"
"testing"
"time"
@@ -431,173 +432,45 @@ func TestRangeLogs(t *testing.T) {
t.Fatal(err)
}
chain, _ := core.GenerateChain(gspec.Config, gspec.ToBlock(), ethash.NewFaker(), db, 1000, func(i int, gen *core.BlockGen) {})
-
options := core.DefaultConfig().WithStateScheme(rawdb.HashScheme)
options.TxLookupLimit = 0 // index all txs
bc, err := core.NewBlockChain(db, gspec, ethash.NewFaker(), options)
if err != nil {
t.Fatal(err)
}
- _, err = bc.InsertChain(chain[:600])
- if err != nil {
- t.Fatal(err)
- }
- backend.startFilterMaps(200, false, filtermaps.RangeTestParams)
- defer backend.stopFilterMaps()
-
- var (
- testCase, event int
- filter *Filter
- addresses = []common.Address{{}}
- )
-
- expEvent := func(expEvent int, expFirst, expAfterLast uint64) {
- exp := rangeLogsTestEvent{expEvent, common.NewRange[uint64](expFirst, expAfterLast-expFirst)}
- event++
- ev := <-filter.rangeLogsTestHook
- if ev != exp {
- t.Fatalf("Test case #%d: wrong test event #%d received (got %v, expected %v)", testCase, event, ev, exp)
+ rangeTest := func(testCase int, begin, end int64, expRanges []testFilterRange) {
+ filter := sys.NewRangeFilter(begin, end, []common.Address{{}}, nil)
+ filter.testFilterRanges = []testFilterRange{}
+ filter.Logs(context.Background())
+ if !slices.Equal(filter.testFilterRanges, expRanges) {
+ t.Fatalf("Test case #%d: wrong list of filtered ranges received (got %v, expected %v)", testCase, filter.testFilterRanges, expRanges)
}
}
- newFilter := func(begin, end int64) {
- testCase++
- event = 0
- filter = sys.NewRangeFilter(begin, end, addresses, nil)
- filter.rangeLogsTestHook = make(chan rangeLogsTestEvent)
- go func(filter *Filter) {
- filter.Logs(context.Background())
- // ensure that filter will not be blocked if we exit early
- for range filter.rangeLogsTestHook {
- }
- }(filter)
- }
+ bc.InsertChain(chain[:600])
+ backend.startFilterMaps(200, false, filtermaps.RangeTestParams)
+ defer backend.stopFilterMaps()
- updateHead := func() {
- head := bc.CurrentBlock()
- backend.fm.SetTarget(filtermaps.NewChainView(backend, head.Number.Uint64(), head.Hash()), 0, 0)
- backend.fm.WaitIdle()
- }
+ rangeTest(1, 0, 300, []testFilterRange{{0, 300, false}})
+ rangeTest(2, 300, 500, []testFilterRange{{400, 500, true}, {300, 399, false}})
+ rangeTest(3, 400, 10000, []testFilterRange{{400, 600, true}})
+ bc.SetCanonical(chain[99])
+ backend.logIndexer.Revert(100)
+ rangeTest(4, 0, 10000, []testFilterRange{{0, 100, false}})
+ backend.backfillBlocks()
+ bc.InsertChain(chain[100:120])
+ backend.indexHead(120)
+ rangeTest(5, 0, 10000, []testFilterRange{{0, 120, true}})
+ bc.InsertChain(chain[120:800])
+ backend.indexHead(800)
+ rangeTest(6, 400, 800, []testFilterRange{{600, 800, true}, {400, 599, false}})
+ backend.stopFilterMaps()
- // test case #1
- newFilter(300, 500)
- expEvent(rangeLogsTestIndexed, 401, 501)
- expEvent(rangeLogsTestSync, 0, 0)
- expEvent(rangeLogsTestSynced, 401, 601)
- expEvent(rangeLogsTestResults, 401, 501)
- expEvent(rangeLogsTestUnindexed, 300, 401)
- if _, err := bc.InsertChain(chain[600:700]); err != nil {
- t.Fatal(err)
- }
- updateHead()
- expEvent(rangeLogsTestResults, 300, 501)
- expEvent(rangeLogsTestDone, 0, 0)
+ backend.startFilterMaps(0, true, filtermaps.RangeTestParams)
+ rangeTest(7, 0, 10000, []testFilterRange{{0, 800, false}})
+ backend.stopFilterMaps()
- // test case #2
- newFilter(400, int64(rpc.LatestBlockNumber))
- expEvent(rangeLogsTestIndexed, 501, 701)
- if _, err := bc.InsertChain(chain[700:800]); err != nil {
- t.Fatal(err)
- }
- updateHead()
- expEvent(rangeLogsTestSync, 0, 0)
- expEvent(rangeLogsTestSynced, 601, 699)
- expEvent(rangeLogsTestResults, 601, 699)
- expEvent(rangeLogsTestUnindexed, 400, 601)
- expEvent(rangeLogsTestResults, 400, 699)
- expEvent(rangeLogsTestIndexed, 699, 801)
- if _, err := bc.SetCanonical(chain[749]); err != nil { // set head to block 750
- t.Fatal(err)
- }
- updateHead()
- expEvent(rangeLogsTestSync, 0, 0)
- expEvent(rangeLogsTestSynced, 601, 749)
- expEvent(rangeLogsTestResults, 400, 749)
- expEvent(rangeLogsTestIndexed, 749, 751)
- expEvent(rangeLogsTestSync, 0, 0)
- expEvent(rangeLogsTestSynced, 551, 751)
- expEvent(rangeLogsTestResults, 400, 751)
- expEvent(rangeLogsTestDone, 0, 0)
-
- // test case #3
- newFilter(int64(rpc.LatestBlockNumber), int64(rpc.LatestBlockNumber))
- expEvent(rangeLogsTestIndexed, 750, 751)
- if _, err := bc.SetCanonical(chain[739]); err != nil {
- t.Fatal(err)
- }
- updateHead()
- expEvent(rangeLogsTestSync, 0, 0)
- expEvent(rangeLogsTestSynced, 551, 739)
- expEvent(rangeLogsTestResults, 0, 0)
- expEvent(rangeLogsTestIndexed, 740, 741)
- if _, err := bc.InsertChain(chain[740:750]); err != nil {
- t.Fatal(err)
- }
- updateHead()
- expEvent(rangeLogsTestSync, 0, 0)
- expEvent(rangeLogsTestSynced, 551, 739)
- expEvent(rangeLogsTestResults, 0, 0)
- expEvent(rangeLogsTestIndexed, 750, 751)
- expEvent(rangeLogsTestSync, 0, 0)
- expEvent(rangeLogsTestSynced, 551, 751)
- expEvent(rangeLogsTestResults, 750, 751)
- expEvent(rangeLogsTestDone, 0, 0)
-
- // test case #4
- if _, err := bc.SetCanonical(chain[499]); err != nil {
- t.Fatal(err)
- }
- updateHead()
- newFilter(400, int64(rpc.LatestBlockNumber))
- expEvent(rangeLogsTestIndexed, 400, 501)
- if _, err := bc.InsertChain(chain[500:650]); err != nil {
- t.Fatal(err)
- }
- updateHead()
- expEvent(rangeLogsTestSync, 0, 0)
- expEvent(rangeLogsTestSynced, 451, 499)
- expEvent(rangeLogsTestResults, 451, 499)
- expEvent(rangeLogsTestUnindexed, 400, 451)
- expEvent(rangeLogsTestResults, 400, 499)
- // indexed head extension seems possible
- expEvent(rangeLogsTestIndexed, 499, 651)
- // further head extension causes tail unindexing in searched range
- if _, err := bc.InsertChain(chain[650:750]); err != nil {
- t.Fatal(err)
- }
- updateHead()
- expEvent(rangeLogsTestSync, 0, 0)
- expEvent(rangeLogsTestSynced, 551, 649)
- // tail trimmed to 551; cannot merge with existing results
- expEvent(rangeLogsTestResults, 551, 649)
- expEvent(rangeLogsTestUnindexed, 400, 551)
- expEvent(rangeLogsTestResults, 400, 649)
- expEvent(rangeLogsTestIndexed, 649, 751)
- expEvent(rangeLogsTestSync, 0, 0)
- expEvent(rangeLogsTestSynced, 551, 751)
- expEvent(rangeLogsTestResults, 400, 751)
- expEvent(rangeLogsTestDone, 0, 0)
-
- // test case #5
- newFilter(400, int64(rpc.LatestBlockNumber))
- expEvent(rangeLogsTestIndexed, 551, 751)
- expEvent(rangeLogsTestSync, 0, 0)
- expEvent(rangeLogsTestSynced, 551, 751)
- expEvent(rangeLogsTestResults, 551, 751)
- expEvent(rangeLogsTestUnindexed, 400, 551)
- if _, err := bc.InsertChain(chain[750:1000]); err != nil {
- t.Fatal(err)
- }
- updateHead()
- expEvent(rangeLogsTestResults, 400, 751)
- // indexed tail already beyond results head; revert to unindexed head search
- expEvent(rangeLogsTestUnindexed, 751, 1001)
- if _, err := bc.SetCanonical(chain[899]); err != nil {
- t.Fatal(err)
- }
- updateHead()
- expEvent(rangeLogsTestResults, 400, 1001)
- expEvent(rangeLogsTestReorg, 400, 901)
- expEvent(rangeLogsTestDone, 0, 0)
+ backend.startFilterMaps(0, false, filtermaps.RangeTestParams)
+ rangeTest(8, 0, 10000, []testFilterRange{{0, 800, true}})
}
diff --git a/internal/ethapi/api_test.go b/internal/ethapi/api_test.go
index aaa002b5ec..2c12a927ec 100644
--- a/internal/ethapi/api_test.go
+++ b/internal/ethapi/api_test.go
@@ -690,10 +690,10 @@ func (b testBackend) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent)
func (b testBackend) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription {
panic("implement me")
}
-func (b testBackend) CurrentView() *filtermaps.ChainView {
+func (b testBackend) CurrentChainView() *filtermaps.ChainView {
panic("implement me")
}
-func (b testBackend) NewMatcherBackend() filtermaps.MatcherBackend {
+func (b testBackend) GetIndexView(headBlockHash common.Hash) *filtermaps.IndexView {
panic("implement me")
}
diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go
index af3d592b82..db2399cc02 100644
--- a/internal/ethapi/backend.go
+++ b/internal/ethapi/backend.go
@@ -99,8 +99,8 @@ type Backend interface {
SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription
SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription
- CurrentView() *filtermaps.ChainView
- NewMatcherBackend() filtermaps.MatcherBackend
+ CurrentChainView() *filtermaps.ChainView
+ GetIndexView(headBlockHash common.Hash) *filtermaps.IndexView
}
func GetAPIs(apiBackend Backend) []rpc.API {
diff --git a/internal/ethapi/transaction_args_test.go b/internal/ethapi/transaction_args_test.go
index 30791f32b5..dc59ed531c 100644
--- a/internal/ethapi/transaction_args_test.go
+++ b/internal/ethapi/transaction_args_test.go
@@ -407,7 +407,7 @@ func (b *backendMock) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent)
func (b *backendMock) Engine() consensus.Engine { return nil }
-func (b *backendMock) CurrentView() *filtermaps.ChainView { return nil }
-func (b *backendMock) NewMatcherBackend() filtermaps.MatcherBackend { return nil }
+func (b *backendMock) CurrentChainView() *filtermaps.ChainView { return nil }
+func (b *backendMock) GetIndexView(headBlockHash common.Hash) *filtermaps.IndexView { return nil }
func (b *backendMock) HistoryPruningCutoff() uint64 { return 0 }