forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-07-22 20:56:42 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

core/filtermaps: implement log indexer as core.Indexer

Browse source
This commit is contained in:
zsfelfoldi 2025-07-28 23:41:15 +02:00
parent 8ce710a354
commit 4a8eaeb1d1
33 changed files with 3774 additions and 4046 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
common/range.go
View file

@ -27,7 +27,11 @@ type Range[T uint32 | uint64] struct {
// NewRange creates a new range based of first element and number of elements.
func NewRange[T uint32 | uint64](first, count T) Range[T] {
return Range[T]{first, first + count}
afterLast := first + count
if afterLast < first {
panic("range overflow")
}
return Range[T]{first, afterLast}
}
// First returns the first element of the range.
@ -97,6 +101,12 @@ func (r Range[T]) Intersection(q Range[T]) Range[T] {
// Union returns the union of two ranges. Panics for gapped ranges.
func (r Range[T]) Union(q Range[T]) Range[T] {
if r.IsEmpty() {
return q
}
if q.IsEmpty() {
return r
}
if max(r.first, q.first) > min(r.afterLast, q.afterLast) {
panic("cannot create union; gap between ranges")
}

31
core/filtermaps/chain_view.go
View file

@ -32,12 +32,11 @@ type blockchain interface {
GetRawReceipts(hash common.Hash, number uint64) types.Receipts
}
// ChainView represents an immutable view of a chain with a block id and a set
// ChainView represents an immutable view of a chain with a block hash and a set
// of receipts associated to each block number and a block hash associated with
// all block numbers except the head block. This is because in the future
// ChainView might represent a view where the head block is currently being
// created. Block id is a unique identifier that can also be calculated for the
// head block.
// created.
// Note that the view's head does not have to be the current canonical head
// of the underlying blockchain, it should only possess the block headers
// and receipts up until the expected chain view head.
@ -79,21 +78,6 @@ func (cv *ChainView) BlockHash(number uint64) common.Hash {
return cv.blockHash(number)
}
// BlockId returns the unique block id belonging to the given block number.
// Note that it is currently equal to the block hash. In the future it might
// be a different id for future blocks if the log index root becomes part of
// consensus and therefore rendering the index with the new head will happen
// before the hash of that new head is available.
func (cv *ChainView) BlockId(number uint64) common.Hash {
cv.lock.Lock()
defer cv.lock.Unlock()
if number > cv.headNumber {
panic("invalid block number")
}
return cv.blockHash(number)
}
// Header returns the block header at the given block number.
func (cv *ChainView) Header(number uint64) *types.Header {
return cv.chain.GetHeader(cv.BlockHash(number), number)
@ -128,7 +112,7 @@ func (cv *ChainView) SharedRange(cv2 *ChainView) common.Range[uint64] {
return common.Range[uint64]{}
}
sharedLen := min(cv.headNumber, cv2.headNumber) + 1
for sharedLen > 0 && cv.BlockId(sharedLen-1) != cv2.BlockId(sharedLen-1) {
for sharedLen > 0 && cv.BlockHash(sharedLen-1) != cv2.BlockHash(sharedLen-1) {
sharedLen--
}
return common.NewRange(0, sharedLen)
@ -139,7 +123,7 @@ func equalViews(cv1, cv2 *ChainView) bool {
if cv1 == nil || cv2 == nil {
return false
}
return cv1.headNumber == cv2.headNumber && cv1.BlockId(cv1.headNumber) == cv2.BlockId(cv2.headNumber)
return cv1.headNumber == cv2.headNumber && cv1.BlockHash(cv1.headNumber) == cv2.BlockHash(cv2.headNumber)
}
// matchViews returns true if the two chain views are equivalent up until the
@ -152,12 +136,7 @@ func matchViews(cv1, cv2 *ChainView, number uint64) bool {
if cv1.headNumber < number || cv2.headNumber < number {
return false
}
var h1, h2 common.Hash
if number == cv1.headNumber || number == cv2.headNumber {
h1, h2 = cv1.BlockId(number), cv2.BlockId(number)
} else {
h1, h2 = cv1.BlockHash(number), cv2.BlockHash(number)
}
h1, h2 := cv1.BlockHash(number), cv2.BlockHash(number)
return h1 == h2 && h1 != common.Hash{}
}

2
core/filtermaps/checkpoints.go
View file

@ -33,7 +33,7 @@ type checkpointList []epochCheckpoint
// be initialized at the epoch boundary.
type epochCheckpoint struct {
BlockNumber uint64 // block that generated the last log value of the given epoch
BlockId common.Hash
BlockHash common.Hash
FirstIndex uint64 // first log value index of the given block
}

52
core/filtermaps/checkpoints_holesky.json
View file

@ -1,28 +1,28 @@
[
{"blockNumber": 814410, "blockId": "0x6c38f0d4ff2c23ae187f581cebb0963c0ec2dbf051b289de1582c369c98a3244", "firstIndex": 67107349},
{"blockNumber": 914268, "blockId": "0xeff161573a11eb6e2ab930674a5245b2c5ffc5e9e63093503344ec6fa60578a3", "firstIndex": 134216064},
{"blockNumber": 1048866, "blockId": "0xebd3b95415dad9ab7f2b1d25e48c791e299063c09427bbde54217029c3e215ad", "firstIndex": 201325914},
{"blockNumber": 1144433, "blockId": "0x0c895438ef12a2c835b4372c209e40a499ba2b18ed0e658846813784de391392", "firstIndex": 268434935},
{"blockNumber": 1230406, "blockId": "0xd3512e7241efc9853e46b1ad565403d302f62457b6dd84917c31ff375fabf487", "firstIndex": 335544096},
{"blockNumber": 1309104, "blockId": "0xcf108c6e002e7a5657bf7587adde03edf5f20d03e95b66a194eb8080daaf4f97", "firstIndex": 402653143},
{"blockNumber": 1380499, "blockId": "0x984b0f8b6bf06f1240bbca8c1df9bef3880bedafd5b5c2a55cbc2f64c9efb974", "firstIndex": 469759822},
{"blockNumber": 1476950, "blockId": "0x8323b528bb8d80a96b172de92452be0f45078a9ca311cb4be6675f089e25a426", "firstIndex": 536870335},
{"blockNumber": 1533506, "blockId": "0x737dc416070aa3b522a0c2ab813a70c1820f062c718f27597394f309f0004106", "firstIndex": 603979618},
{"blockNumber": 1613764, "blockId": "0x9d6a5a505afdda86b1ebcc2b10cb687a1f89c2e2b74315945a3ddc6a0b3c9cd6", "firstIndex": 671088253},
{"blockNumber": 1719073, "blockId": "0x17bf6a51d9c55a908c3e9e652f80b2aa464b049c697abf3bd5a537e461aff0b9", "firstIndex": 738197366},
{"blockNumber": 1973133, "blockId": "0x14b288d70e688de3334d09283670301aacfed21c193da8a56fd63767f8177705", "firstIndex": 805305624},
{"blockNumber": 2274761, "blockId": "0xf02790b273b04219e001d2fcc93e8e47708cb228364e96ad754bc8050d08a9aa", "firstIndex": 872414871},
{"blockNumber": 2530470, "blockId": "0x157ea98b1a7e66dd3f045da8e25219800671f9a000211d3d94006efd33d89a80", "firstIndex": 939523234},
{"blockNumber": 2781706, "blockId": "0xa723663a584e280700d2302eba03d1b3b6549333db70c6152576d33e2911f594", "firstIndex": 1006632936},
{"blockNumber": 3101669, "blockId": "0xa6e66a18fed64d33178c7088f273c404be597662c34f6e6884cf2e24f3ea4ace", "firstIndex": 1073741353},
{"blockNumber": 3221725, "blockId": "0xe771f897dece48b1583cc1d1d10de8015da57407eb1fdf239fdbe46eaab85143", "firstIndex": 1140850137},
{"blockNumber": 3357164, "blockId": "0x6252d0aa54c79623b0680069c88d7b5c47983f0d5c4845b6c811b8d9b5e8ff3c", "firstIndex": 1207959453},
{"blockNumber": 3447019, "blockId": "0xeb7d585e1e063f3cc05ed399fbf6c2df63c271f62f030acb804e9fb1e74b6dc1", "firstIndex": 1275067542},
{"blockNumber": 3546397, "blockId": "0xdabdef7defa4281180a57c5af121877b82274f15ccf074ea0096146f4c246df2", "firstIndex": 1342176778},
{"blockNumber": 3867885, "blockId": "0x8be069dd7a3e2ffb869ee164d11b28555233d2510b134ab9d5484fdae55d2225", "firstIndex": 1409285539},
{"blockNumber": 3935446, "blockId": "0xc91a61bc215bbcccc3020c62e5c8153162df0d8bcc59813d74671b2d24903ed7", "firstIndex": 1476394742},
{"blockNumber": 3989508, "blockId": "0xc85dec36a767e44237842ef51915944c2a49780c8c394a3aa6cfb013c99cf58b", "firstIndex": 1543503452},
{"blockNumber": 4057078, "blockId": "0xccdb79f6705629cb6ab1667a1244938f60911236549143fcff23a3989213e67e", "firstIndex": 1610612030},
{"blockNumber": 4126499, "blockId": "0x92f2ef21fc911e87e81e38373d5f2915587b9648a0ab3cf4fcfe3e5aaffe7b85", "firstIndex": 1677720416},
{"blockNumber": 4239335, "blockId": "0x64fbd22965eb583a584552b7edb9b7ce26fb6aad247c1063d0d5a4d11cbcc58c", "firstIndex": 1744830176}
{"blockNumber": 814410, "blockHash": "0x6c38f0d4ff2c23ae187f581cebb0963c0ec2dbf051b289de1582c369c98a3244", "firstIndex": 67107349},
{"blockNumber": 914268, "blockHash": "0xeff161573a11eb6e2ab930674a5245b2c5ffc5e9e63093503344ec6fa60578a3", "firstIndex": 134216064},
{"blockNumber": 1048866, "blockHash": "0xebd3b95415dad9ab7f2b1d25e48c791e299063c09427bbde54217029c3e215ad", "firstIndex": 201325914},
{"blockNumber": 1144433, "blockHash": "0x0c895438ef12a2c835b4372c209e40a499ba2b18ed0e658846813784de391392", "firstIndex": 268434935},
{"blockNumber": 1230406, "blockHash": "0xd3512e7241efc9853e46b1ad565403d302f62457b6dd84917c31ff375fabf487", "firstIndex": 335544096},
{"blockNumber": 1309104, "blockHash": "0xcf108c6e002e7a5657bf7587adde03edf5f20d03e95b66a194eb8080daaf4f97", "firstIndex": 402653143},
{"blockNumber": 1380499, "blockHash": "0x984b0f8b6bf06f1240bbca8c1df9bef3880bedafd5b5c2a55cbc2f64c9efb974", "firstIndex": 469759822},
{"blockNumber": 1476950, "blockHash": "0x8323b528bb8d80a96b172de92452be0f45078a9ca311cb4be6675f089e25a426", "firstIndex": 536870335},
{"blockNumber": 1533506, "blockHash": "0x737dc416070aa3b522a0c2ab813a70c1820f062c718f27597394f309f0004106", "firstIndex": 603979618},
{"blockNumber": 1613764, "blockHash": "0x9d6a5a505afdda86b1ebcc2b10cb687a1f89c2e2b74315945a3ddc6a0b3c9cd6", "firstIndex": 671088253},
{"blockNumber": 1719073, "blockHash": "0x17bf6a51d9c55a908c3e9e652f80b2aa464b049c697abf3bd5a537e461aff0b9", "firstIndex": 738197366},
{"blockNumber": 1973133, "blockHash": "0x14b288d70e688de3334d09283670301aacfed21c193da8a56fd63767f8177705", "firstIndex": 805305624},
{"blockNumber": 2274761, "blockHash": "0xf02790b273b04219e001d2fcc93e8e47708cb228364e96ad754bc8050d08a9aa", "firstIndex": 872414871},
{"blockNumber": 2530470, "blockHash": "0x157ea98b1a7e66dd3f045da8e25219800671f9a000211d3d94006efd33d89a80", "firstIndex": 939523234},
{"blockNumber": 2781706, "blockHash": "0xa723663a584e280700d2302eba03d1b3b6549333db70c6152576d33e2911f594", "firstIndex": 1006632936},
{"blockNumber": 3101669, "blockHash": "0xa6e66a18fed64d33178c7088f273c404be597662c34f6e6884cf2e24f3ea4ace", "firstIndex": 1073741353},
{"blockNumber": 3221725, "blockHash": "0xe771f897dece48b1583cc1d1d10de8015da57407eb1fdf239fdbe46eaab85143", "firstIndex": 1140850137},
{"blockNumber": 3357164, "blockHash": "0x6252d0aa54c79623b0680069c88d7b5c47983f0d5c4845b6c811b8d9b5e8ff3c", "firstIndex": 1207959453},
{"blockNumber": 3447019, "blockHash": "0xeb7d585e1e063f3cc05ed399fbf6c2df63c271f62f030acb804e9fb1e74b6dc1", "firstIndex": 1275067542},
{"blockNumber": 3546397, "blockHash": "0xdabdef7defa4281180a57c5af121877b82274f15ccf074ea0096146f4c246df2", "firstIndex": 1342176778},
{"blockNumber": 3867885, "blockHash": "0x8be069dd7a3e2ffb869ee164d11b28555233d2510b134ab9d5484fdae55d2225", "firstIndex": 1409285539},
{"blockNumber": 3935446, "blockHash": "0xc91a61bc215bbcccc3020c62e5c8153162df0d8bcc59813d74671b2d24903ed7", "firstIndex": 1476394742},
{"blockNumber": 3989508, "blockHash": "0xc85dec36a767e44237842ef51915944c2a49780c8c394a3aa6cfb013c99cf58b", "firstIndex": 1543503452},
{"blockNumber": 4057078, "blockHash": "0xccdb79f6705629cb6ab1667a1244938f60911236549143fcff23a3989213e67e", "firstIndex": 1610612030},
{"blockNumber": 4126499, "blockHash": "0x92f2ef21fc911e87e81e38373d5f2915587b9648a0ab3cf4fcfe3e5aaffe7b85", "firstIndex": 1677720416},
{"blockNumber": 4239335, "blockHash": "0x64fbd22965eb583a584552b7edb9b7ce26fb6aad247c1063d0d5a4d11cbcc58c", "firstIndex": 1744830176}
]

580
core/filtermaps/checkpoints_mainnet.json
View file

@ -1,292 +1,292 @@
[
{"blockNumber": 4166212, "blockId": "0xd94b724fc1c7dceb3251b51b81f7a0f3220ae5b9add1e62917004f14f2a3532c", "firstIndex": 67108840},
{"blockNumber": 4513996, "blockId": "0xc0fd25fef5888609d05fa8c5620d8e18c31cdd675dd4ee926ff966668f0e5d76", "firstIndex": 134217480},
{"blockNumber": 4817399, "blockId": "0x8573c3166fb7f716e97cf6109d382f86441238e0b85828944a072ae8583a6cfa", "firstIndex": 201326547},
{"blockNumber": 5087706, "blockId": "0xcd2dfae45901e299b25faa04c10df60983d5c0a3d0e478e789c7aeec980cf691", "firstIndex": 268435422},
{"blockNumber": 5306085, "blockId": "0xe8026984c85873f2b24018a7e0bdf8c2df136b2fc49666b4addf0d2e067890da", "firstIndex": 335544018},
{"blockNumber": 5509898, "blockId": "0x41bc63dc8184f9a7d4b9a5a554e00eee32fee60ccfa2339264be11270ac28de1", "firstIndex": 402652789},
{"blockNumber": 5670367, "blockId": "0xe61c2ba463f2f458817ed1bf0ff9fb9d07e9d7354f46b48175b2d784b817bf3b", "firstIndex": 469761896},
{"blockNumber": 5826113, "blockId": "0xfbbb1fc5e03a5562bf6b7f1799d7a572d9ef66c7fd0e0b9f4fed7262767b5c86", "firstIndex": 536870852},
{"blockNumber": 5953008, "blockId": "0x6bbdccd094928e846de3c615a33f2952e3259afaa1929f4ee241a56907d0593b", "firstIndex": 603979150},
{"blockNumber": 6102812, "blockId": "0x2403f2502088a1fa26c1294f5245032fe3b7f3a8bb14c12d0ba8d577156bbc1b", "firstIndex": 671087962},
{"blockNumber": 6276672, "blockId": "0xeaa05a0e0574fbb164244628a7d14d9d47f1692941098aa737059d44104401df", "firstIndex": 738197399},
{"blockNumber": 6448662, "blockId": "0xe6f097fa18a2cef425514e863659e42e1aa8d74df49cb0bed5877e82a08d1f84", "firstIndex": 805306056},
{"blockNumber": 6655925, "blockId": "0xee5c9d87b06dc0c2e850fead07bcd13f85d45ad6b5a6e7ebee53cc8618772786", "firstIndex": 872415229},
{"blockNumber": 6873878, "blockId": "0x9a0ea103146da21fa1d9cab7ff609ec2c5e7f1856288a1f744239588e33904c2", "firstIndex": 939524080},
{"blockNumber": 7080893, "blockId": "0x35ad45055d7a1a3ef94efeb05e4122757bcc126ff1bc2b4ac72dc78ab3c0fd75", "firstIndex": 1006632327},
{"blockNumber": 7266955, "blockId": "0x7b80f70520f2c16eb5890f4963af8f60446f50af82b589ceaf009d09e704b302", "firstIndex": 1073741608},
{"blockNumber": 7466649, "blockId": "0xea8980b5c692c729013f9cc4c9f66e94d5ef2c5caad23f370be0953b6d85f32c", "firstIndex": 1140850335},
{"blockNumber": 7661736, "blockId": "0x615cdb03dbf29a22d59bbc769c03f3647c2c9abb7d1767f3f6529708209e7073", "firstIndex": 1207959232},
{"blockNumber": 7834494, "blockId": "0x05b7dd13e2eb8a833e4b278f9151ca01b294ae5fc5617769422ec85ffb446215", "firstIndex": 1275068098},
{"blockNumber": 7989998, "blockId": "0x4b5026add7f2fc2c02993b82c61bceba1b75cfcb8a78c5112cc2832bc0413be6", "firstIndex": 1342177025},
{"blockNumber": 8142950, "blockId": "0x9f56b7753a0e9964973f3e096e1385215e3aef232b448359e5bec05046b016a3", "firstIndex": 1409285545},
{"blockNumber": 8297598, "blockId": "0x2ddfeb161fe34ff73c4a995bb94256c3b522b39348169edb97bb3d876ccdf77d", "firstIndex": 1476394898},
{"blockNumber": 8465061, "blockId": "0xfdbe5435da5ae9fb34a9154e5ebe515afac9a128e383f798f1e69952be404fcd", "firstIndex": 1543503805},
{"blockNumber": 8655740, "blockId": "0x305137325a39a44aba997e214c05f32d965658f7e3fe20322f2e57e72f239977", "firstIndex": 1610612406},
{"blockNumber": 8807102, "blockId": "0xc32f9fd9a43d2b502dbcc4a683ffe324fca9a871753478bcb1f4fb9573da96a5", "firstIndex": 1677721343},
{"blockNumber": 8911110, "blockId": "0x806eabee7dc429b57afcb6c0b72b61728b4f13c62ee9942f1813790463f9ab5b", "firstIndex": 1744830172},
{"blockNumber": 8960262, "blockId": "0xa69f24032e9d5761565e726d484926a0932c6ca7fdb4c04a55bfc5e8f5879c4d", "firstIndex": 1811938729},
{"blockNumber": 9085906, "blockId": "0x922b4d6aeca64517f5048fa1ffa98d3e813bc415716a1763d07cf2bdea236896", "firstIndex": 1879048062},
{"blockNumber": 9230838, "blockId": "0xd892f513c48c95a859ae59c4e00ec4be0d684c549456c90e9c36669c11092f50", "firstIndex": 1946156580},
{"blockNumber": 9390449, "blockId": "0xcc0ced78fa66b570f338d129794d574560a23958e28d0d5288003d4542cec734", "firstIndex": 2013264376},
{"blockNumber": 9515554, "blockId": "0xc06d04737813c3e2c8e910b2d4543c5c684c7bfe235bb501445f68fd0663f3d9", "firstIndex": 2080374749},
{"blockNumber": 9659367, "blockId": "0x765fa33d9418b1ed648e88b497b4bf4aa66990e6413dfc155525cda61e3cc813", "firstIndex": 2147483344},
{"blockNumber": 9793940, "blockId": "0x9527dcbd85a100c51aee4ba90828fee23312f5f041859f3416a54fd87a437ce2", "firstIndex": 2214591669},
{"blockNumber": 9923756, "blockId": "0xc4da5fdd2de077459fe84f721d7fc01ef69683e03e634b0c5443186486792246", "firstIndex": 2281700949},
{"blockNumber": 10042325, "blockId": "0x6f554f195d20e4ba1262f73cf2c1cd44b14a71befe5ef0a5318fc309f8cde9fa", "firstIndex": 2348810131},
{"blockNumber": 10168768, "blockId": "0xdcdf88ec2c197d552343c80d01375a35d6462aff39c5bd976160f7176d2add64", "firstIndex": 2415918723},
{"blockNumber": 10289467, "blockId": "0x5ece23babcf6b8838695b891b044c3dcf8120cdc15acd83018e96ffc590f6cd0", "firstIndex": 2483027720},
{"blockNumber": 10386601, "blockId": "0xd398ee129aea1247019fa39dc2f5083819687d8f5ad17ec39ba1803d178fbcda", "firstIndex": 2550136043},
{"blockNumber": 10479586, "blockId": "0x61c1cccd60a546eacc29f2ae2a6facab402b4f35e2986c6629b302e6b1ebca3e", "firstIndex": 2617245577},
{"blockNumber": 10562579, "blockId": "0x14fd6a079050e3e110140372605c050945d6db03746fa5b84a8718cb2a0e9b86", "firstIndex": 2684353802},
{"blockNumber": 10641438, "blockId": "0x1998dcae09111b2c1bf6d0592e2f0b4a0143fe7db7115ff94b2b73b74542cee4", "firstIndex": 2751463334},
{"blockNumber": 10717094, "blockId": "0x9ce45a150bb6bd13687d0692c86716911ffc6aff5831a03f4015bdffb85f5093", "firstIndex": 2818571144},
{"blockNumber": 10784494, "blockId": "0xf65c936cf578e673696477f7db7189b47222fdf7ed13096564b9d47b15c86eb9", "firstIndex": 2885680709},
{"blockNumber": 10848591, "blockId": "0x84ea8e9d5789302ebe88888115488f91cfb75fd89e282dfbd29ba9d6225ab0ca", "firstIndex": 2952789491},
{"blockNumber": 10909102, "blockId": "0x22275047f643f4a3b82c349274a53d56a00da25beac89655877fb8cbfa3182c8", "firstIndex": 3019898024},
{"blockNumber": 10972828, "blockId": "0x38e8acfec19bb53ab9fe70dd3577ccd8bd7c41b4d468b8f099a19bf06ce56518", "firstIndex": 3087006561},
{"blockNumber": 11036532, "blockId": "0x478dac03f520e987a6e845cac1b3a756d69e4f5257f20758b8fc38456e7b6884", "firstIndex": 3154116463},
{"blockNumber": 11102457, "blockId": "0x1a36616b03351ebe481330fc86578f57d9da5c0dda59f4406d49b53a2fa9b6f6", "firstIndex": 3221224961},
{"blockNumber": 11168092, "blockId": "0xb23af483ca6dbc2fb8d2fada1f8189de2e155758ed9ecbbb20c234a30c7e093c", "firstIndex": 3288333510},
{"blockNumber": 11233640, "blockId": "0xdc4c80d8370823773425b31e70b3af59582b556514857d1e9a9b4c53b880a424", "firstIndex": 3355442901},
{"blockNumber": 11300454, "blockId": "0xdbd4d8e9c1756093037f431330db4604e5a3e67bb74f425f667633d5298fe34c", "firstIndex": 3422551962},
{"blockNumber": 11367550, "blockId": "0x9e00816bd5b39353657a99421d278091599a39f246bbf8c0898f83e6d70b7c7c", "firstIndex": 3489660743},
{"blockNumber": 11431820, "blockId": "0xd32d5e5871ce71acc9dcc7cf9d8cb9d964646182529c327a0ddf617814e27d74", "firstIndex": 3556769150},
{"blockNumber": 11497031, "blockId": "0xed823054d4b293294e3301d3131e2ae9f0ce63c9f07d2d80fc144e42d535d61d", "firstIndex": 3623878513},
{"blockNumber": 11560027, "blockId": "0x7098177efe77cfb5912dc63069fe366033111041ca5c539907a89e439c861d8f", "firstIndex": 3690987092},
{"blockNumber": 11625049, "blockId": "0x61fccb47104a076b905d390498a1b38c317348bb1567e69aa1b670bed7a6a13f", "firstIndex": 3758096043},
{"blockNumber": 11690298, "blockId": "0xaf6f9e9c92db9e155f1ffb35eb68765c09de1206c353c9a56a5d5fa53e2d88d2", "firstIndex": 3825204430},
{"blockNumber": 11755003, "blockId": "0xf9efb54f08a2b59086a607541005928f1bda54fa2c46fcc48352fa6aa461c03a", "firstIndex": 3892313541},
{"blockNumber": 11819591, "blockId": "0xa551deb3bd2ec0ca2683cb76adb8fb295d171a3787aa5ff071bd1fa94d3af82e", "firstIndex": 3959422322},
{"blockNumber": 11883278, "blockId": "0x60c7a359d4aeb09b63b64487fa7794aee87c41b11446f69ba83e24d5bce6947f", "firstIndex": 4026531239},
{"blockNumber": 11948436, "blockId": "0x1c46fb22653e5f9eb9503fe532151519845d05b121291766715973e027eb9c0c", "firstIndex": 4093639741},
{"blockNumber": 12013084, "blockId": "0x0b8d1da9e70c4494ba61190390c891230846e69d9b0be605df8b04c4199ffd1b", "firstIndex": 4160749561},
{"blockNumber": 12078612, "blockId": "0xb049af260be25c6153b67c227689b0556d8fb7d6033c3f61b8218f5c4ac5917a", "firstIndex": 4227857310},
{"blockNumber": 12143549, "blockId": "0x685dce41efbe2bab0ac9a4b9026d9e5faa3e01ae4637c8181cd2f1eac326fe6a", "firstIndex": 4294966232},
{"blockNumber": 12207918, "blockId": "0x53644ea8f781c6270f1136ed4bbcb956c98f12a4793c2f56f49b3105282b13ea", "firstIndex": 4362075410},
{"blockNumber": 12272370, "blockId": "0x503523e1a01cb7d68be0eb22f36280c0ee7a1fc802061a704e326fbb8efcfe27", "firstIndex": 4429184831},
{"blockNumber": 12329342, "blockId": "0xe6a1bf78dd827def078e23c768cf6e96666ee06b5fdc113380a4769d85f4bd33", "firstIndex": 4496293151},
{"blockNumber": 12382311, "blockId": "0x352fe793c16b1d8aea195af86ee9aec3dbaa83e7a32872a0d222a77e96fc0e57", "firstIndex": 4563402486},
{"blockNumber": 12436970, "blockId": "0x90276349edf0941bb3327dc1d749d0371fe550eaddafb5141fcacdd281de0976", "firstIndex": 4630510187},
{"blockNumber": 12489954, "blockId": "0x71894b1e1608aeba777662a6055e601ce6f17950dbe3844c3601e0f88cbd5848", "firstIndex": 4697619586},
{"blockNumber": 12541655, "blockId": "0xc34c7299041df72a81c4b86e81639d2dfcebfae603e287fc143881a636ce1188", "firstIndex": 4764728948},
{"blockNumber": 12597319, "blockId": "0x4c4dc71d8d2f1223ad0469ddfe09fa8934f64e3712ed83708d6e1b9c64eceebb", "firstIndex": 4831837390},
{"blockNumber": 12651863, "blockId": "0x406a642c8364fb61cbd896512a2d19ecdba3932ab09ac67cf48d91aa1a03f24e", "firstIndex": 4898946362},
{"blockNumber": 12706391, "blockId": "0xa7fc3739898f7ff5733896313b248fb0b306a132681609b8b9e2671b42dac3a1", "firstIndex": 4966054748},
{"blockNumber": 12762840, "blockId": "0x5a1048966dfa55a48248517c45683ce342671bf544117d2bac13d8e343563136", "firstIndex": 5033164303},
{"blockNumber": 12816597, "blockId": "0x97f5b87f422870a8ffeff70c12f4377dde96d248aa912cb64ebfaea51dc086b6", "firstIndex": 5100272541},
{"blockNumber": 12872315, "blockId": "0x432f7df7973fdea4e5abe2c09b634ca3b724e37a19d5ec23ca5810e4854c6b33", "firstIndex": 5167382243},
{"blockNumber": 12929620, "blockId": "0xa4524e3b4c5dadac91df2f29696532cfed377f47469fe7cfabf10baac945eea2", "firstIndex": 5234490748},
{"blockNumber": 12988653, "blockId": "0x9be70e31aeea164c370d8d183a1af45120693b13d360edc78214535c40ef1d63", "firstIndex": 5301600248},
{"blockNumber": 13049074, "blockId": "0x3076857bd17959c2a3df18357eab3ecbb7ca25457785b07074d2e885eb566fc2", "firstIndex": 5368707746},
{"blockNumber": 13108823, "blockId": "0x67b2764f6b53779e3d683def2c7659c6d92814152574073d24f10cd67200a22b", "firstIndex": 5435817680},
{"blockNumber": 13175389, "blockId": "0x0d41e34dbca58dec45e0509b6cbf027b2377220845a85caebbf3a14b8505e8ce", "firstIndex": 5502925085},
{"blockNumber": 13237624, "blockId": "0x723e0818d54ab1b1c53efc2f2f448c16c1f943af2033575c612c3c71228d4ddf", "firstIndex": 5570034948},
{"blockNumber": 13298658, "blockId": "0xd7f9f477b20bcf1379ccaaae701c9658a02b177861e0af9e873c63fe8b854f4b", "firstIndex": 5637144562},
{"blockNumber": 13361166, "blockId": "0x9f9e4130093269d59045f91344a13eb2786023e02099f53dd44854d50a7f5ffb", "firstIndex": 5704253260},
{"blockNumber": 13421716, "blockId": "0x28752f35ad2adeba342ef6aa51bca1c0abc757c4f712cf620c8a5e6a8576636e", "firstIndex": 5771361775},
{"blockNumber": 13480516, "blockId": "0x43f4ecf24c17b7a77621e2ca09c88c53b3407fb018f4bb3a032c212015b4e18d", "firstIndex": 5838469616},
{"blockNumber": 13535688, "blockId": "0x99ffcd9982219a4be45fa5f15622b288afe4f85da536956d39dccac1ec416444", "firstIndex": 5905579739},
{"blockNumber": 13590487, "blockId": "0xff7b74464a318037e0670c0be7a5b1b79418d1a42c7a3587c46adc1ef720c819", "firstIndex": 5972688013},
{"blockNumber": 13646596, "blockId": "0xd555ce5f2b65ffeec9b9e3e75a56f5f23d7353a1014bcc861efd7df6e5dc2c92", "firstIndex": 6039797525},
{"blockNumber": 13702902, "blockId": "0x3e8feab590c6631c63931fee83bf7e10234890bdf377b1b374fb70221b9ac9c0", "firstIndex": 6106905641},
{"blockNumber": 13760908, "blockId": "0x18e96fa585c57aad1f2a8dc9639d9f4fe53c3f38894862471b7cec17f97f3cb1", "firstIndex": 6174014330},
{"blockNumber": 13819340, "blockId": "0x25384bf2cef178e30c4d9e0abae59357055241fc715133b2bb078e8278e5296a", "firstIndex": 6241123906},
{"blockNumber": 13877817, "blockId": "0x4a84a0a7035dbfed0e1281fb1df162a195f2c516686a2e2a6ad605e6f2b3cf57", "firstIndex": 6308232690},
{"blockNumber": 13935260, "blockId": "0x6c14a8e563614edbad66698ebe911e997e27844f9d33c7069795379540e71a44", "firstIndex": 6375341148},
{"blockNumber": 13993914, "blockId": "0xc0991d1bc431c5215fb0218b0b541f2b4fae0994775e5eddb016965fe5b4a0c7", "firstIndex": 6442448947},
{"blockNumber": 14051007, "blockId": "0x854803ce364b6fe4446800e74448a5ee883e7d472e8a0347f1ad13bd347fa69f", "firstIndex": 6509558964},
{"blockNumber": 14109063, "blockId": "0xa3fadce7ab18a5fedcfed9e3b1f0cf2fb32151172c362327c602343bc26dde8f", "firstIndex": 6576668017},
{"blockNumber": 14166701, "blockId": "0xaa2443a38e2b0d6b01cf4b73de54d2b7e92c99475bd0802beebd86845c126d1b", "firstIndex": 6643774648},
{"blockNumber": 14222367, "blockId": "0xa27c78134c1fbf432c8caee7c9d2ef323fdde519a065804c9c8da6ca1615e04f", "firstIndex": 6710886238},
{"blockNumber": 14277029, "blockId": "0x5d0ed08b65db01e382c339dcb2715b2927fb472b0c1f4256841c386d785c3f3a", "firstIndex": 6777995247},
{"blockNumber": 14330955, "blockId": "0x3e5014d82c416549acfe2e26b13ff5ddf29ad74943ef5b94a65eca0a4d72d82e", "firstIndex": 6845102969},
{"blockNumber": 14384139, "blockId": "0x720e1fd687292019c3fa1812c39c73d20fde783c0196f43378a51cb4e0b46d65", "firstIndex": 6912212078},
{"blockNumber": 14437534, "blockId": "0xe4342ab6f363066d9c2514a870241ceb63df232d33659fdb3928faf433360bb4", "firstIndex": 6979321493},
{"blockNumber": 14489073, "blockId": "0xeb2f9923d816e773460eccfa42eb4ff2bbfc353b756ba274e98dc85c64ab1536", "firstIndex": 7046428879},
{"blockNumber": 14541559, "blockId": "0x55cc33721fa9022964f47c30f4146fee7cc80f5b4158d7ecea0ff661cbba3487", "firstIndex": 7113538857},
{"blockNumber": 14594432, "blockId": "0x676441bdc1ec430a9fbc05e56a0437a475606a253f9fdffc86f5f6286c159b5d", "firstIndex": 7180647956},
{"blockNumber": 14644942, "blockId": "0x03232a1d3c9b2f1f0c2a4b780c83bce66238d9abdeed71c2a8af9e97a53011b0", "firstIndex": 7247754981},
{"blockNumber": 14695793, "blockId": "0x6017f6d1d79188692e1a1a18de5f93b83b97905c8ff5bed5cef4b2e9f4d4d41b", "firstIndex": 7314863646},
{"blockNumber": 14746186, "blockId": "0x332c8ac0b2960fb1627407d45443201a0f6de7421b52af077b32fa6e018e5401", "firstIndex": 7381974499},
{"blockNumber": 14798403, "blockId": "0x4507ae1d9f27a3eada874dcf04112a5d4ee7155701afd3fb857fe58201a2dede", "firstIndex": 7449082752},
{"blockNumber": 14848093, "blockId": "0xff98c820c66c3beaaa3b195158c94f099148573a7f032295ba7156c13f78eaa8", "firstIndex": 7516191978},
{"blockNumber": 14897508, "blockId": "0x49c31d3433b197585f733d7c382ae16d9d78bf59c333a9b76362cc6c5fed270c", "firstIndex": 7583300663},
{"blockNumber": 14950638, "blockId": "0xf1a1609206788977841927271808fe1c42d059026385d025bf4cea0c0a05966b", "firstIndex": 7650408992},
{"blockNumber": 15003958, "blockId": "0xf9e6b41108d907672f29ba1df97ed83aca2ba933335e6b5ccfb5d4001807b33f", "firstIndex": 7717519105},
{"blockNumber": 15056783, "blockId": "0x126a5a5c22e878a4c42bea890d48cfe00edf7fbc27d805c88a5269540bc94a42", "firstIndex": 7784628173},
{"blockNumber": 15108163, "blockId": "0x5f8494ccacefce9fef8a95a52247f2670ed30bb5a3b3deb85c645e72c115d506", "firstIndex": 7851735261},
{"blockNumber": 15159385, "blockId": "0x0fed6c5aafb53e9cdb0d8620c3eef9b619fb3ae9559296618f634eddae33f0c6", "firstIndex": 7918844299},
{"blockNumber": 15210872, "blockId": "0x37bdb94b57044b67fbd9b9a7cdc728fcfd2066cf1904cab478fd42d41ec6e1f0", "firstIndex": 7985954119},
{"blockNumber": 15264226, "blockId": "0xc173ad8a791e2293efd8e0b2c9f088fac49233aabb1e72db6a045aa9904f4728", "firstIndex": 8053061840},
{"blockNumber": 15315693, "blockId": "0x985cf97bf28f0becb2b16abfe629a5f520596e45fc23f4936bc2f6298b8145f9", "firstIndex": 8120172048},
{"blockNumber": 15364606, "blockId": "0x593362ef8a82140f3dd3ff13d0ce50287dcb0b8b3d31ffcc7813c6be16f7438d", "firstIndex": 8187281275},
{"blockNumber": 15412727, "blockId": "0x5a6ff10acc48e9e8bcf963b042012ea094628b9488b8bf569783b3297c67eedd", "firstIndex": 8254388575},
{"blockNumber": 15462642, "blockId": "0x137f15c5d112b0fdc6e4c84393032030646122c3b0d86286143a1ab8bfacbac4", "firstIndex": 8321497214},
{"blockNumber": 15509781, "blockId": "0xe58a8ca223b7c03755d046a6212bc33fa9814f73efe52804d8eea7ae4b4bd7f9", "firstIndex": 8388606496},
{"blockNumber": 15558625, "blockId": "0x4127d527b58c7685a1237d228508ea77d40f33dbeffb5cc30e47e83a4b6152a5", "firstIndex": 8455714910},
{"blockNumber": 15604075, "blockId": "0x7d1c1bccfe5c23a1edd81246e6e55277f271cffef30bde22c9d1202f764dd96f", "firstIndex": 8522823324},
{"blockNumber": 15651742, "blockId": "0x732e5203a504fd97510325c1fb876476ca132831e78182367cd4b4efae1232c3", "firstIndex": 8589934155},
{"blockNumber": 15700810, "blockId": "0xe031decb5e42a69dafcc814441227085508cf51c9c77f35d06487db19bd05a6a", "firstIndex": 8657041374},
{"blockNumber": 15762855, "blockId": "0xbdff9b1e193c44f35bf38945d2523c74ef59c4f352efc93fad02ef34eef1c5f0", "firstIndex": 8724150924},
{"blockNumber": 15814304, "blockId": "0x524a226d9f48e37db00e252c7f6a1350d1829d38fe83c756021cc11c56d3b2dd", "firstIndex": 8791259373},
{"blockNumber": 15865514, "blockId": "0x4e4ad46f3ae7fb2a2e7cb94aa804929cb7353c79c78cc2cbcee710da5f0ec4b5", "firstIndex": 8858369571},
{"blockNumber": 15920411, "blockId": "0xa39e4cbd8e84b427e44b8d06b5b1149652f3af5fa88633b802665b8677772bc5", "firstIndex": 8925477782},
{"blockNumber": 15974193, "blockId": "0x4a3f82935e6fcda6ce7636001989b65b79422d362e60833e01185a7960b2fa55", "firstIndex": 8992587369},
{"blockNumber": 16032739, "blockId": "0x7f50bb76dfd01125e0a4a7595106e43329eb8bd02e819be2ade4248a403c714a", "firstIndex": 9059696611},
{"blockNumber": 16090896, "blockId": "0xdc4db130973ef793d2223844e35122a2477e642d86c69616fb3604d2a79805c4", "firstIndex": 9126804425},
{"blockNumber": 16150783, "blockId": "0xb46294fe1cbcbbdff7309ebf0bf481e78e2589d386b95fa4c781c437e02e0aec", "firstIndex": 9193912139},
{"blockNumber": 16207278, "blockId": "0x8a3af0387572b56989bb6c98d8860e07b5cdefb2ebe7b1dc58be07c24a53e056", "firstIndex": 9261022215},
{"blockNumber": 16262389, "blockId": "0xec5a03d88cfc1bf8c278ddce7ace5032c3f7ed5032b4575df54779ec1abca131", "firstIndex": 9328130680},
{"blockNumber": 16319525, "blockId": "0x74ae702375e242205fe4986448b0ee494d4a38ccc813981489c4b0488e58a949", "firstIndex": 9395240779},
{"blockNumber": 16373441, "blockId": "0x5f404555cf45c547ef0fc044f2701bc7a28ce1e432de06e0264dbfd1c17447d0", "firstIndex": 9462347671},
{"blockNumber": 16423343, "blockId": "0xe4af890b6c4dd3b10af2e6c08f14563bd2a3121b22a3cc6a0db29d438c1dcc59", "firstIndex": 9529457704},
{"blockNumber": 16474687, "blockId": "0x019a8b374cf150fcda1c8af2a3e7ba63603ae81744d6343897c4b17ee58411d5", "firstIndex": 9596566169},
{"blockNumber": 16525504, "blockId": "0xad394ce7e84e1c904cc5dd6dabd51b1a575c0b2fb006f10f272875b7f921904e", "firstIndex": 9663675287},
{"blockNumber": 16574051, "blockId": "0x098e2dca2eda0d700aa3127bdfb679ce16e3ca297a305978c6ecfff044269511", "firstIndex": 9730784563},
{"blockNumber": 16622457, "blockId": "0x8627f1237731c1127116898724f1b6420b8e7bcd8e18a0ea568f192bbd2cd55d", "firstIndex": 9797893241},
{"blockNumber": 16672410, "blockId": "0x3e32b46faa670b191b89bea84a68022d5ea8b90008b388a948ff27f871d17292", "firstIndex": 9865002964},
{"blockNumber": 16719964, "blockId": "0x1668c3ea6cb256f306e93851cc2c00612f85df5cf7b5527ffce14c89088d7fdf", "firstIndex": 9932111708},
{"blockNumber": 16769014, "blockId": "0xed9a6f88e47b5bfe4d5bc92e1c1bfeb252f716f0990ffa1f907655c0f5e3a365", "firstIndex": 9999220312},
{"blockNumber": 16818134, "blockId": "0x93504ba617500698a1b46f3110cd26827b8efe468190b2a91d3595784e34a1b6", "firstIndex": 10066329167},
{"blockNumber": 16869369, "blockId": "0x94ca4b395a5c36c4e9d07cbb08271f0680455af4e4659906b647e16abebd7d4e", "firstIndex": 10133438415},
{"blockNumber": 16921428, "blockId": "0x66650775bff4cb95de56099c10b06f3dfe302a731b391f406d1aa551121308ba", "firstIndex": 10200547148},
{"blockNumber": 16974677, "blockId": "0x58d5fc4cd45047dac10d55a036e0a4448faacc1041b29c4c7fb41a6e9394c3ff", "firstIndex": 10267655623},
{"blockNumber": 17031050, "blockId": "0x9d15b831979e95fc4149ecae46478cac1e54470b277d57d7102d19a72f28eeba", "firstIndex": 10334764441},
{"blockNumber": 17086163, "blockId": "0x978ff48863fcdb50b274ea4220e7635c13c41bae91912db9e24205fb917fde85", "firstIndex": 10401873176},
{"blockNumber": 17141086, "blockId": "0x4084d442499893361731f17534944d260cccd8e07d41c52d7160910555770b90", "firstIndex": 10468982462},
{"blockNumber": 17190735, "blockId": "0x42ed86fb42def23870df6ec23204f557669ec7cd8de59c9f3ef9ab90704e4b11", "firstIndex": 10536090726},
{"blockNumber": 17237026, "blockId": "0x9a49d943f82a7335f8049df4d953080f25920e3b098f6a2d872520166b225961", "firstIndex": 10603199938},
{"blockNumber": 17287007, "blockId": "0xac3d189676d24eadc62adcf97d8730a001a7766fe3ac27b18e745ca9727f6c46", "firstIndex": 10670308186},
{"blockNumber": 17338498, "blockId": "0xe50e251aa0a3209fa8a0030b237cf8357b08eeb80492a432c8aa5969a820ee42", "firstIndex": 10737417623},
{"blockNumber": 17389178, "blockId": "0x1e18cedcfa6c81bc845c9136fc92f9f27ca8fff8847289bb139f374f0f6ba733", "firstIndex": 10804526769},
{"blockNumber": 17442038, "blockId": "0x07f47ebbee99938fb96beb1ffadf8266f71d0d73f393172f0e24facd40d45c97", "firstIndex": 10871635940},
{"blockNumber": 17497589, "blockId": "0x247a5e852e36e46f7897ad1875e93d810ff4434bb835b469afae3a3d3d4a9c9a", "firstIndex": 10938744251},
{"blockNumber": 17554111, "blockId": "0x420e420a1b599e4dc3bd115652b4d6f5eedc391086ef064f97a9cbffc02b62f9", "firstIndex": 11005853686},
{"blockNumber": 17608282, "blockId": "0xb2ba644d3593d3c832f932f14ffec532e26e9e1cdcd8fbe7477f46e238c6bffd", "firstIndex": 11072962303},
{"blockNumber": 17664091, "blockId": "0x6d0bfff8145652a617bfe03ab87e3af09219cdb822c9c214b42b16f20742c1eb", "firstIndex": 11140070260},
{"blockNumber": 17714917, "blockId": "0x61509544b5174d3ff5d48816ff54f2929c8a400ef21454116056baa136b607b6", "firstIndex": 11207179013},
{"blockNumber": 17763850, "blockId": "0x8da83012621f42670601406c902ce6e48e77c6285dcc6983e108e69b3a8d5d2d", "firstIndex": 11274288732},
{"blockNumber": 17814186, "blockId": "0xc37bda200856589c220dc206e31186d6c12761fd193f5bc7123cf331fbbed7f5", "firstIndex": 11341397343},
{"blockNumber": 17864461, "blockId": "0x99af43893dc9f54c55abbdbd7b000e27248145371e9b00db116dfb961b36749c", "firstIndex": 11408506122},
{"blockNumber": 17913212, "blockId": "0x9b642236d63d756fcd3404713bc6abe97804b46a9e665ae5f1d9aa48e693fe78", "firstIndex": 11475614347},
{"blockNumber": 17961497, "blockId": "0xaec9b1e851cf51646673ec7698e1d5964d44eb04729ad11385f6ae84d497070a", "firstIndex": 11542723713},
{"blockNumber": 18010859, "blockId": "0xb15fb7fc308516cad5f1dadfb7dd92fd3fe612308f67154cf2a241990aa15bc4", "firstIndex": 11609833204},
{"blockNumber": 18061052, "blockId": "0xd8ea94b8b8a7380d21dc433abb69973f3de2eeb339f4aaffdb79f85a5d86929d", "firstIndex": 11676938565},
{"blockNumber": 18111466, "blockId": "0x5b5cc6d182bffaf06692a7efda8cad5d4742fdbce5c1146b3be3c5be10ea046a", "firstIndex": 11744048539},
{"blockNumber": 18166001, "blockId": "0x901d8401c795542c75069cd917781fa587ad416da0a5298f144fae2f14305705", "firstIndex": 11811159949},
{"blockNumber": 18218505, "blockId": "0x5894285023f34ec6fd501cd90b605e118b9688ca7c01082289bf69a6987142bc", "firstIndex": 11878268395},
{"blockNumber": 18271037, "blockId": "0x3f0fce315fd993d966b6c599ebd360462466e67046d512dc430ac07f75243bae", "firstIndex": 11945377050},
{"blockNumber": 18322778, "blockId": "0xfa6536b3b06e7b579e53a8d01368182808c950586e39e12485be3145c3acbf43", "firstIndex": 12012486036},
{"blockNumber": 18372240, "blockId": "0x5531859d5b8a4093020e2fb971808d0eeaddd7d61dc58d70f34923081f97336f", "firstIndex": 12079595358},
{"blockNumber": 18421630, "blockId": "0xfeadff676fe8f04d0362b77ea2d7c3ea59b05d68cc6b2cc95cfe751558b1bfa7", "firstIndex": 12146703872},
{"blockNumber": 18471472, "blockId": "0x352b0a17e4160d334b876eb180d686fd67c7ddc40a78219b0bc4b59f5ae5912d", "firstIndex": 12213813174},
{"blockNumber": 18522088, "blockId": "0xf7d408309246089531b47c787049c7ffd2820c1b87b4804033a458bc193a3432", "firstIndex": 12280921181},
{"blockNumber": 18572725, "blockId": "0x44126d4a5b43594b09d26e0b0a5dc859c074865cb9b11ab78c80f28406fd0240", "firstIndex": 12348029615},
{"blockNumber": 18623223, "blockId": "0xd3aec1f98cea30e870d0f530194e9a72e225eb45eefe435e38da5967eb2fa470", "firstIndex": 12415139522},
{"blockNumber": 18675228, "blockId": "0x24deaef2508038bddf8e9020f95128ae34eb1786f786c3f98df0bdf95f3e97cc", "firstIndex": 12482248123},
{"blockNumber": 18725495, "blockId": "0x2e5a7edd7337b20e862741e116680a472089426dbc2b0eb5f1c984f8d6da935b", "firstIndex": 12549357121},
{"blockNumber": 18778174, "blockId": "0xe734a1084fa5eeff427bba8e8d3623181a30dd9bad128fc22de9b15dffd0e549", "firstIndex": 12616466131},
{"blockNumber": 18834807, "blockId": "0x0924a55ef4f9c2605261433c5b236d47645484efa44b34dc9f15b7c2e6b47a74", "firstIndex": 12683575000},
{"blockNumber": 18883054, "blockId": "0xef638ec491a1cf685b22bf4072e41710440d6133a53e31250762825247d3eb64", "firstIndex": 12750683712},
{"blockNumber": 18933405, "blockId": "0x6e9eefc2e121d13e6c04544ea370fe8543afeb36fef3a8e2ea68afb079738c8c", "firstIndex": 12817792510},
{"blockNumber": 18988033, "blockId": "0x4c98c0dbb6a7217609c4b277fe7ff9e2c32c8d000ee628a3f37b2b296e1ca421", "firstIndex": 12884901389},
{"blockNumber": 19041096, "blockId": "0x61a219a23d111cd07861ff47b89d4b2840c0ab5421d5d55a6f1fe43da3f33b98", "firstIndex": 12952010147},
{"blockNumber": 19088914, "blockId": "0xefb26ba58447e50e9b820e0b0d16c08da45793a824af12a2e1f38b6f46b7e36a", "firstIndex": 13019118891},
{"blockNumber": 19140413, "blockId": "0x91dbb13623fc4d5a4e462dfa0a03c92bf5301948dd4b2c3e5fddd2bc88856d47", "firstIndex": 13086228137},
{"blockNumber": 19191876, "blockId": "0xdd7f97af7ad206b2636656ed4722fe1c47c6ba9fa5531066785acbe4f8d4c4ad", "firstIndex": 13153336551},
{"blockNumber": 19237651, "blockId": "0xf1e875f314314abe3654b667c6443c772dd67c525c52877b06515ea8fc5a391b", "firstIndex": 13220444812},
{"blockNumber": 19291032, "blockId": "0x9eda269ac47957b619d2a8c408b5efc8cfb168a46bfde351f43325fa337fce47", "firstIndex": 13287553967},
{"blockNumber": 19344225, "blockId": "0x62859b78d45f589f679fc6a3bf266a2e0aa906fd61d787755ec51cfadc05eaf6", "firstIndex": 13354663338},
{"blockNumber": 19394709, "blockId": "0x7cda3d193bcfe8ddbc89b85e98c814dbbe31a54ccab52ab7738ecd003061f3c8", "firstIndex": 13421772549},
{"blockNumber": 19442850, "blockId": "0xd95f3f8bab5b91c3e1f848c9455936850bf92cf6a5086b817f6f81031bc0f81e", "firstIndex": 13488880704},
{"blockNumber": 19488177, "blockId": "0x3a1a9623ee332d6decada9359cf84eeafb81fffa997d6cf4f3efcc92e98e9310", "firstIndex": 13555989570},
{"blockNumber": 19534365, "blockId": "0x5c7a8ee7cbba4894ea412cc335895a9a7d3369d35779bcdc97a6c24b6a81a7ba", "firstIndex": 13623097446},
{"blockNumber": 19579409, "blockId": "0x4280f5f76cb0f9947f9da1cb9de6c4b6e65b37b3a3c174f5fb29d360cf389ffb", "firstIndex": 13690206790},
{"blockNumber": 19621406, "blockId": "0x0a98ca72e97fd295c7c73263b17185d74dbe427f41c616a5d5454faa5a1c63a0", "firstIndex": 13757316653},
{"blockNumber": 19664868, "blockId": "0xc8c53d89b7a849861f3daaf9503e6397a0340cecb085850c3636bee6ea2b7189", "firstIndex": 13824424608},
{"blockNumber": 19709011, "blockId": "0xf65babc88ef6a9eb2ffce9a6844adf895995f65a2facd1956f5df17ca68dbf94", "firstIndex": 13891534141},
{"blockNumber": 19755170, "blockId": "0xf903882449eeb86d69af33b6797099275e82d8787334a02a42a2168dbd739b95", "firstIndex": 13958642627},
{"blockNumber": 19803666, "blockId": "0xa8a975f570db082a3771790bb1fe405667dde4674ff49e31092687fa92727e36", "firstIndex": 14025752433},
{"blockNumber": 19847730, "blockId": "0x89d479a4b4d9faa7af3e3f61d13a7a9c80a11c4adf7cd018859eb0ddba30bdd5", "firstIndex": 14092860598},
{"blockNumber": 19894138, "blockId": "0x740278e6dc79cfd2b030780868a1b8e1520dbae0e8d3ee0a3ef43c84c83c6e0a", "firstIndex": 14159952129},
{"blockNumber": 19938334, "blockId": "0x05ae7ce9813235c19baa57658bc5c8e160130328a1fd2cc58d2baf2d9a4ae097", "firstIndex": 14227078393},
{"blockNumber": 19985138, "blockId": "0x511c8876a93a77fa473ae0d29e25d8163162c09b7212f506be0ea73840ea3971", "firstIndex": 14294187397},
{"blockNumber": 20028221, "blockId": "0x64379baedf429cfa65df076142a7d9909d7e369d6aa6b8e161bb686944d169ff", "firstIndex": 14361296446},
{"blockNumber": 20071555, "blockId": "0x4f43d8b1091d2bd0c6b080ce51bce08aae6be605e00889be3cdd17cb80614ff8", "firstIndex": 14428405409},
{"blockNumber": 20113611, "blockId": "0xebbf87409ef6732a53513414926b0aa998a7269f9e9717a3acbe96a33c2de937", "firstIndex": 14495514255},
{"blockNumber": 20156628, "blockId": "0x70218108c0669f42c4ac1afff80947cf4a0a2fd8e9300b31d45bbe330a5159c9", "firstIndex": 14562622257},
{"blockNumber": 20199886, "blockId": "0x1d6aaa4b041eb0cc9caed34347bb943e50a139caf8998d781ff4f06d4aa727dc", "firstIndex": 14629732017},
{"blockNumber": 20244148, "blockId": "0x37fed57b6945a815ec56c84c6261e90cf9f3883d9cbffc62b70d9722c1f0f363", "firstIndex": 14696839730},
{"blockNumber": 20288249, "blockId": "0x1d9fe664431e093c4ef6139ad35fea767a2e66825e605cfea9e91d1738665619", "firstIndex": 14763948803},
{"blockNumber": 20333358, "blockId": "0x1ac7c1362791507057b7e87d7118b4f61c2919c8ebdb498ecf7d64a59ba3d2e2", "firstIndex": 14831058814},
{"blockNumber": 20376855, "blockId": "0xf6a19eac8a402f591079a4aa7ff57349c350ec96d683a031e4d85c05bc0541b2", "firstIndex": 14898167076},
{"blockNumber": 20421474, "blockId": "0xe8767aa7516b632dc8c0fed3a366273ef14bd7b8c2212998b7105fb80b4c0d9b", "firstIndex": 14965276285},
{"blockNumber": 20466974, "blockId": "0xafa2d2e6c9594762c163c339069cb15aa9fd27d301268892e12f99a3f2912ade", "firstIndex": 15032384674},
{"blockNumber": 20512183, "blockId": "0xc1c51f9372d781d7ac0a1a7aa052503bbde3e1706bf65f225b0f6a6eed93d055", "firstIndex": 15099493832},
{"blockNumber": 20557251, "blockId": "0xa905087c15e2b12b19cd23f20089b471cd9815dcff73842d2043e13a276fcc55", "firstIndex": 15166601921},
{"blockNumber": 20595654, "blockId": "0x9e7772dd946f65e7569f34f6896d8e597ac8ff88700b871de21ff578c0c995e6", "firstIndex": 15233711474},
{"blockNumber": 20638340, "blockId": "0xc08b46af83db6f0e5d1acad7020bacd59d0f18df72651d32c9b60edd0351a886", "firstIndex": 15300820584},
{"blockNumber": 20683359, "blockId": "0x7ec93a1f1d211c24d93ce7e344634efef680a6388374cf3831d250ecd63711f9", "firstIndex": 15367929168},
{"blockNumber": 20728401, "blockId": "0x4ad4124c2595abcf4477cac4eb70c0c9372bbd48f2066c7ac5b8e9f94817ab48", "firstIndex": 15435038303},
{"blockNumber": 20771272, "blockId": "0x68fd6b12fdb4add8b2957105a21b571c7dcb114c6c2ca02f5fa40730a108f864", "firstIndex": 15502146509},
{"blockNumber": 20814862, "blockId": "0x4e7f640bf3c135b2442ca4c744023021af163e8d45e8d776fb7051c49d79fcff", "firstIndex": 15569256173},
{"blockNumber": 20857627, "blockId": "0x9b023bffed12530b5a68db263bb621a147e0cf02b30ec39a82b82a1fdeb7ca36", "firstIndex": 15636364955},
{"blockNumber": 20896662, "blockId": "0x8c3adf694b8520ed1edd38282b2f58cf383426176628b5be6330a56882e2a07e", "firstIndex": 15703473445},
{"blockNumber": 20939156, "blockId": "0x0649e2559f767da945a674c9d6c700628109e18d0757ad33496a228a827413a4", "firstIndex": 15770581979},
{"blockNumber": 20981054, "blockId": "0xebf0ecca671b833f77df70106cf1171d4dd3fa0f062607e4845569073997f1cd", "firstIndex": 15837691319},
{"blockNumber": 21022980, "blockId": "0x07479520c08a727c42e3616aca7fc4435773f4ef277d84369190263159606d71", "firstIndex": 15904799391},
{"blockNumber": 21068139, "blockId": "0x669cfd03b293a61815588cb9cef928a512d328c1fe9191fe6b281573f429f8b8", "firstIndex": 15971908604},
{"blockNumber": 21112368, "blockId": "0x98957aedaf8b0483be208e8d16a3e6d73ff217e732f3649650671c92c8fb57a5", "firstIndex": 16039007815},
{"blockNumber": 21155756, "blockId": "0xc301e40988fbe19b1254ab3abe2f78fe0cd35259898be02476179a041926f777", "firstIndex": 16106127130},
{"blockNumber": 21200717, "blockId": "0x8d35b199ee65c5ccf4ca0fc22b50282850cd88b72580679f1568eea67d3431e8", "firstIndex": 16173235593},
{"blockNumber": 21244396, "blockId": "0xe7ec084eb8a7125fe26c4ef64fdbb3e0a727b2d882fb1e4fdad1d7fa3dc55fd2", "firstIndex": 16240343803},
{"blockNumber": 21290031, "blockId": "0x8bb31b883d30875b7ed8a01c35a6f51e603b8d974c4e74af20fb9c723e5ac5df", "firstIndex": 16307452772},
{"blockNumber": 21335893, "blockId": "0xd0976b7c946a7556e607e47a02d2cfcccd259508e2828b9c4faa5c7105a597a8", "firstIndex": 16374562657},
{"blockNumber": 21378611, "blockId": "0x72d42953f013aeb7c84bb22c9e07828b92f73885b20c208af612874df8fe85ed", "firstIndex": 16441671192},
{"blockNumber": 21421343, "blockId": "0x663b3cddcf3f7cfa59ed29b4894e43fa784c07576cfec897ff1a3838235ecb50", "firstIndex": 16508779766},
{"blockNumber": 21466598, "blockId": "0xe04ebd60289dbcc09a27e053e7b240a59e928e2a76a85a6fa4044ee387c46d18", "firstIndex": 16575888836},
{"blockNumber": 21511717, "blockId": "0x5ea9863a0b504f381010ff383d60a56390ba9cef162d9d5a077b1884d2adac98", "firstIndex": 16642998234},
{"blockNumber": 21550050, "blockId": "0x2a7b3e635c80521887ca3ebda04541090ec720dd817cd06fcba61e7ae0f78a4f", "firstIndex": 16710106733},
{"blockNumber": 21592440, "blockId": "0xc7095268d0d53cb5a069c2ea7107cc42172bb1b817f04154938e6b52bdbb369c", "firstIndex": 16777215261},
{"blockNumber": 21636007, "blockId": "0x55f463b932053ebfc02728a12c2cdfb2cb4700a115cde686c9ab22bb4c336ecb", "firstIndex": 16844324844},
{"blockNumber": 21680799, "blockId": "0x4521d473aa7f5150f920c66fe639b20f30cb373b32e425fcb7dcce9803b506be", "firstIndex": 16911431255},
{"blockNumber": 21725330, "blockId": "0x793876007ce3594df603628766b139a02273ceee43d5b3ca2d0ca2f13a5fc5d9", "firstIndex": 16978541626},
{"blockNumber": 21771310, "blockId": "0xd0db23bb28d6e39917348b1879dd560c73354ea6d7d0bbee0b7a8fcbcede1091", "firstIndex": 17045650902},
{"blockNumber": 21811348, "blockId": "0x8165948c08db34aeb8cc2a3e331a00af2d09fbff9c466019fb2f2892ca3f7b71", "firstIndex": 17112760024},
{"blockNumber": 21851615, "blockId": "0x3083e8f4fbb4d726ef19b1a5bce20e639a0418eac384f9967f24c7697a7e2dfc", "firstIndex": 17179868870},
{"blockNumber": 21894041, "blockId": "0x9b0776a020374e4985f6a7e1199596bedd387ff085f65224d724ab11955c56fe", "firstIndex": 17246977342},
{"blockNumber": 21932218, "blockId": "0xe9c50eddc21759d3ab790e2babba56f4b8d38eb739f1b561bc337a1ba6e15393", "firstIndex": 17314085541},
{"blockNumber": 21959188, "blockId": "0x6b9c482cc4822af2ad62a359b923062556ab6a046d1a39a8243b764848690601", "firstIndex": 17381193886},
{"blockNumber": 21994911, "blockId": "0x0d99ef9dd42dd1c62fc5249eb4f618e7672ab93fa0ba7545c77360371cf972e5", "firstIndex": 17448302795},
{"blockNumber": 22026007, "blockId": "0xf7cdc7f6694f2c2ed31fa8a607f65cfa59d0dd7d7424ab5c017f959ae2982c71", "firstIndex": 17515413036},
{"blockNumber": 22059890, "blockId": "0x82b768a0dddefda2eefd3a33596ea2be075312f1dd4b01f6b0d377faca2af98b", "firstIndex": 17582521768},
{"blockNumber": 22090784, "blockId": "0xf97c2eaf9a550360ac24000c0ff17ffa388a2bdd6f73f2f36718e332edfa107a", "firstIndex": 17649630983},
{"blockNumber": 22121157, "blockId": "0xa790025235db782e899f23d8b09663ec2d74ec149e4125d62989f98829b08e2d", "firstIndex": 17716724973},
{"blockNumber": 22148056, "blockId": "0xbe25ac4f1bdd89a7db5782bf1157e6c4378d80220d67a029397853ef16cd1a4c", "firstIndex": 17783838366},
{"blockNumber": 22168652, "blockId": "0x6ae43618c915e636794e2cc2d75dde9992766881c7405fe6479c045ed4bee57e", "firstIndex": 17850956277},
{"blockNumber": 22190975, "blockId": "0x9437121647899a4b7b84d67fbea7cc6ff967481c2eab4328ccd86e2cefe19420", "firstIndex": 17918066140},
{"blockNumber": 22234357, "blockId": "0x036030830134f9224160d5a0b62da35ec7813dc8855d554bd22e9d38545243ed", "firstIndex": 17985175075},
{"blockNumber": 22276736, "blockId": "0x5ceb96d98aa1b4c1c2f2fa253ae9cdb1b04e0420c11bf795400e8762c0a1635c", "firstIndex": 18052284344},
{"blockNumber": 22321282, "blockId": "0x8a601ebf6a757020c6d43a978f0bd2c150c4acc1ffdd50c7ee88afc78b0c11f8", "firstIndex": 18119392242},
{"blockNumber": 22349231, "blockId": "0xb751c026a92ba5be95ad7ea4e2729a175b0d0e11a4c108f47cab232b4715d1a2", "firstIndex": 18186501218},
{"blockNumber": 22377469, "blockId": "0xa47916860a22f7e26761ec2d7f717410791bd3ed0237b2f6266750214c7bbf08", "firstIndex": 18253610249},
{"blockNumber": 22422685, "blockId": "0x8beaee39603af55fad222730f556c840c41cd76a5eef0bad367ac94d3b86c7aa", "firstIndex": 18320716377},
{"blockNumber": 22462378, "blockId": "0x6dba9c5d2949f5a6a072267b590e8b15e6fb157a0fc22719387f1fd6bfcd8d5d", "firstIndex": 18387828426},
{"blockNumber": 22500185, "blockId": "0x2484c380df0a8f7edfdf8d917570d23fab8499aea80c35b6cf4e5fe1e34106e9", "firstIndex": 18454936227},
{"blockNumber": 22539624, "blockId": "0xd418071906803d25afc3842a6a6468ad3b5fea27107b314ce4e2ccf08b478acf", "firstIndex": 18522044531},
{"blockNumber": 22577021, "blockId": "0xff222982693f3ff60d2097822171f80a6ddd979080aeb7e995bfb1b973497c84", "firstIndex": 18589154438},
{"blockNumber": 22614525, "blockId": "0x9868da1fea2ffca3f67e35570f02eb5707b27f6967ea4a109eb4ddbf24566efd", "firstIndex": 18656264174},
{"blockNumber": 22652848, "blockId": "0x060a911da11ab0f1dda307f5196e622d23901d198925749e70ab58a439477c5a", "firstIndex": 18723372617},
{"blockNumber": 22692432, "blockId": "0x6a937f2c283aba8c778c1f5ef340b225fd820f8a7dfa6f24f5fe541994f32f2d", "firstIndex": 18790480232},
{"blockNumber": 22731200, "blockId": "0x00d57a9e7a2dad252436fe9f0382c6a8860d301a9f9ffe6d7ac64c82b95300f8", "firstIndex": 18857590076},
{"blockNumber": 22769000, "blockId": "0xa48db20307c19c373ef2d31d85088ea14b8df0450491c31982504c87b04edbc0", "firstIndex": 18924699130},
{"blockNumber": 22808126, "blockId": "0x1419c64ff003edca0586f1c8ec3063da5c54c57ff826cfb34bc866cc18949653", "firstIndex": 18991807807},
{"blockNumber": 22845231, "blockId": "0x691f87217e61c5d7ae9ad53a44d30e1ab6b1cc3f2b689b9fbf7c38fbacacfe3e", "firstIndex": 19058917062},
{"blockNumber": 22884189, "blockId": "0x7f102d44c0ea7803f5b0e1a98a6abf0e8383eb99fb114d6f7b4591753ce8bba3", "firstIndex": 19126024122},
{"blockNumber": 22920923, "blockId": "0x04fe6179495016fc3fe56d8ef5311c360a5761a898262173849c3494fdd73d92", "firstIndex": 19193134595},
{"blockNumber": 22958100, "blockId": "0xe38e0ff7b0c4065ca42ea577bc32f2566ca46f2ddeedcc4bc1f8fb00e7f26329", "firstIndex": 19260242424},
{"blockNumber": 22988600, "blockId": "0x04ca74758b22e0ea54b8c992022ff21c16a2af9c45144c3b0f80de921a7eee82", "firstIndex": 19327351273},
{"blockNumber": 23018392, "blockId": "0x61cc979b00bc97b48356f986a5b9ec997d674bc904c2a2e4b0f17de08e50b3bb", "firstIndex": 19394459627},
{"blockNumber": 23048524, "blockId": "0x489de15d95739ede4ab15e8b5151d80d4dc85ae10e7be800b1a4723094a678df", "firstIndex": 19461570073}
{"blockNumber": 4166212, "blockHash": "0xd94b724fc1c7dceb3251b51b81f7a0f3220ae5b9add1e62917004f14f2a3532c", "firstIndex": 67108840},
{"blockNumber": 4513996, "blockHash": "0xc0fd25fef5888609d05fa8c5620d8e18c31cdd675dd4ee926ff966668f0e5d76", "firstIndex": 134217480},
{"blockNumber": 4817399, "blockHash": "0x8573c3166fb7f716e97cf6109d382f86441238e0b85828944a072ae8583a6cfa", "firstIndex": 201326547},
{"blockNumber": 5087706, "blockHash": "0xcd2dfae45901e299b25faa04c10df60983d5c0a3d0e478e789c7aeec980cf691", "firstIndex": 268435422},
{"blockNumber": 5306085, "blockHash": "0xe8026984c85873f2b24018a7e0bdf8c2df136b2fc49666b4addf0d2e067890da", "firstIndex": 335544018},
{"blockNumber": 5509898, "blockHash": "0x41bc63dc8184f9a7d4b9a5a554e00eee32fee60ccfa2339264be11270ac28de1", "firstIndex": 402652789},
{"blockNumber": 5670367, "blockHash": "0xe61c2ba463f2f458817ed1bf0ff9fb9d07e9d7354f46b48175b2d784b817bf3b", "firstIndex": 469761896},
{"blockNumber": 5826113, "blockHash": "0xfbbb1fc5e03a5562bf6b7f1799d7a572d9ef66c7fd0e0b9f4fed7262767b5c86", "firstIndex": 536870852},
{"blockNumber": 5953008, "blockHash": "0x6bbdccd094928e846de3c615a33f2952e3259afaa1929f4ee241a56907d0593b", "firstIndex": 603979150},
{"blockNumber": 6102812, "blockHash": "0x2403f2502088a1fa26c1294f5245032fe3b7f3a8bb14c12d0ba8d577156bbc1b", "firstIndex": 671087962},
{"blockNumber": 6276672, "blockHash": "0xeaa05a0e0574fbb164244628a7d14d9d47f1692941098aa737059d44104401df", "firstIndex": 738197399},
{"blockNumber": 6448662, "blockHash": "0xe6f097fa18a2cef425514e863659e42e1aa8d74df49cb0bed5877e82a08d1f84", "firstIndex": 805306056},
{"blockNumber": 6655925, "blockHash": "0xee5c9d87b06dc0c2e850fead07bcd13f85d45ad6b5a6e7ebee53cc8618772786", "firstIndex": 872415229},
{"blockNumber": 6873878, "blockHash": "0x9a0ea103146da21fa1d9cab7ff609ec2c5e7f1856288a1f744239588e33904c2", "firstIndex": 939524080},
{"blockNumber": 7080893, "blockHash": "0x35ad45055d7a1a3ef94efeb05e4122757bcc126ff1bc2b4ac72dc78ab3c0fd75", "firstIndex": 1006632327},
{"blockNumber": 7266955, "blockHash": "0x7b80f70520f2c16eb5890f4963af8f60446f50af82b589ceaf009d09e704b302", "firstIndex": 1073741608},
{"blockNumber": 7466649, "blockHash": "0xea8980b5c692c729013f9cc4c9f66e94d5ef2c5caad23f370be0953b6d85f32c", "firstIndex": 1140850335},
{"blockNumber": 7661736, "blockHash": "0x615cdb03dbf29a22d59bbc769c03f3647c2c9abb7d1767f3f6529708209e7073", "firstIndex": 1207959232},
{"blockNumber": 7834494, "blockHash": "0x05b7dd13e2eb8a833e4b278f9151ca01b294ae5fc5617769422ec85ffb446215", "firstIndex": 1275068098},
{"blockNumber": 7989998, "blockHash": "0x4b5026add7f2fc2c02993b82c61bceba1b75cfcb8a78c5112cc2832bc0413be6", "firstIndex": 1342177025},
{"blockNumber": 8142950, "blockHash": "0x9f56b7753a0e9964973f3e096e1385215e3aef232b448359e5bec05046b016a3", "firstIndex": 1409285545},
{"blockNumber": 8297598, "blockHash": "0x2ddfeb161fe34ff73c4a995bb94256c3b522b39348169edb97bb3d876ccdf77d", "firstIndex": 1476394898},
{"blockNumber": 8465061, "blockHash": "0xfdbe5435da5ae9fb34a9154e5ebe515afac9a128e383f798f1e69952be404fcd", "firstIndex": 1543503805},
{"blockNumber": 8655740, "blockHash": "0x305137325a39a44aba997e214c05f32d965658f7e3fe20322f2e57e72f239977", "firstIndex": 1610612406},
{"blockNumber": 8807102, "blockHash": "0xc32f9fd9a43d2b502dbcc4a683ffe324fca9a871753478bcb1f4fb9573da96a5", "firstIndex": 1677721343},
{"blockNumber": 8911110, "blockHash": "0x806eabee7dc429b57afcb6c0b72b61728b4f13c62ee9942f1813790463f9ab5b", "firstIndex": 1744830172},
{"blockNumber": 8960262, "blockHash": "0xa69f24032e9d5761565e726d484926a0932c6ca7fdb4c04a55bfc5e8f5879c4d", "firstIndex": 1811938729},
{"blockNumber": 9085906, "blockHash": "0x922b4d6aeca64517f5048fa1ffa98d3e813bc415716a1763d07cf2bdea236896", "firstIndex": 1879048062},
{"blockNumber": 9230838, "blockHash": "0xd892f513c48c95a859ae59c4e00ec4be0d684c549456c90e9c36669c11092f50", "firstIndex": 1946156580},
{"blockNumber": 9390449, "blockHash": "0xcc0ced78fa66b570f338d129794d574560a23958e28d0d5288003d4542cec734", "firstIndex": 2013264376},
{"blockNumber": 9515554, "blockHash": "0xc06d04737813c3e2c8e910b2d4543c5c684c7bfe235bb501445f68fd0663f3d9", "firstIndex": 2080374749},
{"blockNumber": 9659367, "blockHash": "0x765fa33d9418b1ed648e88b497b4bf4aa66990e6413dfc155525cda61e3cc813", "firstIndex": 2147483344},
{"blockNumber": 9793940, "blockHash": "0x9527dcbd85a100c51aee4ba90828fee23312f5f041859f3416a54fd87a437ce2", "firstIndex": 2214591669},
{"blockNumber": 9923756, "blockHash": "0xc4da5fdd2de077459fe84f721d7fc01ef69683e03e634b0c5443186486792246", "firstIndex": 2281700949},
{"blockNumber": 10042325, "blockHash": "0x6f554f195d20e4ba1262f73cf2c1cd44b14a71befe5ef0a5318fc309f8cde9fa", "firstIndex": 2348810131},
{"blockNumber": 10168768, "blockHash": "0xdcdf88ec2c197d552343c80d01375a35d6462aff39c5bd976160f7176d2add64", "firstIndex": 2415918723},
{"blockNumber": 10289467, "blockHash": "0x5ece23babcf6b8838695b891b044c3dcf8120cdc15acd83018e96ffc590f6cd0", "firstIndex": 2483027720},
{"blockNumber": 10386601, "blockHash": "0xd398ee129aea1247019fa39dc2f5083819687d8f5ad17ec39ba1803d178fbcda", "firstIndex": 2550136043},
{"blockNumber": 10479586, "blockHash": "0x61c1cccd60a546eacc29f2ae2a6facab402b4f35e2986c6629b302e6b1ebca3e", "firstIndex": 2617245577},
{"blockNumber": 10562579, "blockHash": "0x14fd6a079050e3e110140372605c050945d6db03746fa5b84a8718cb2a0e9b86", "firstIndex": 2684353802},
{"blockNumber": 10641438, "blockHash": "0x1998dcae09111b2c1bf6d0592e2f0b4a0143fe7db7115ff94b2b73b74542cee4", "firstIndex": 2751463334},
{"blockNumber": 10717094, "blockHash": "0x9ce45a150bb6bd13687d0692c86716911ffc6aff5831a03f4015bdffb85f5093", "firstIndex": 2818571144},
{"blockNumber": 10784494, "blockHash": "0xf65c936cf578e673696477f7db7189b47222fdf7ed13096564b9d47b15c86eb9", "firstIndex": 2885680709},
{"blockNumber": 10848591, "blockHash": "0x84ea8e9d5789302ebe88888115488f91cfb75fd89e282dfbd29ba9d6225ab0ca", "firstIndex": 2952789491},
{"blockNumber": 10909102, "blockHash": "0x22275047f643f4a3b82c349274a53d56a00da25beac89655877fb8cbfa3182c8", "firstIndex": 3019898024},
{"blockNumber": 10972828, "blockHash": "0x38e8acfec19bb53ab9fe70dd3577ccd8bd7c41b4d468b8f099a19bf06ce56518", "firstIndex": 3087006561},
{"blockNumber": 11036532, "blockHash": "0x478dac03f520e987a6e845cac1b3a756d69e4f5257f20758b8fc38456e7b6884", "firstIndex": 3154116463},
{"blockNumber": 11102457, "blockHash": "0x1a36616b03351ebe481330fc86578f57d9da5c0dda59f4406d49b53a2fa9b6f6", "firstIndex": 3221224961},
{"blockNumber": 11168092, "blockHash": "0xb23af483ca6dbc2fb8d2fada1f8189de2e155758ed9ecbbb20c234a30c7e093c", "firstIndex": 3288333510},
{"blockNumber": 11233640, "blockHash": "0xdc4c80d8370823773425b31e70b3af59582b556514857d1e9a9b4c53b880a424", "firstIndex": 3355442901},
{"blockNumber": 11300454, "blockHash": "0xdbd4d8e9c1756093037f431330db4604e5a3e67bb74f425f667633d5298fe34c", "firstIndex": 3422551962},
{"blockNumber": 11367550, "blockHash": "0x9e00816bd5b39353657a99421d278091599a39f246bbf8c0898f83e6d70b7c7c", "firstIndex": 3489660743},
{"blockNumber": 11431820, "blockHash": "0xd32d5e5871ce71acc9dcc7cf9d8cb9d964646182529c327a0ddf617814e27d74", "firstIndex": 3556769150},
{"blockNumber": 11497031, "blockHash": "0xed823054d4b293294e3301d3131e2ae9f0ce63c9f07d2d80fc144e42d535d61d", "firstIndex": 3623878513},
{"blockNumber": 11560027, "blockHash": "0x7098177efe77cfb5912dc63069fe366033111041ca5c539907a89e439c861d8f", "firstIndex": 3690987092},
{"blockNumber": 11625049, "blockHash": "0x61fccb47104a076b905d390498a1b38c317348bb1567e69aa1b670bed7a6a13f", "firstIndex": 3758096043},
{"blockNumber": 11690298, "blockHash": "0xaf6f9e9c92db9e155f1ffb35eb68765c09de1206c353c9a56a5d5fa53e2d88d2", "firstIndex": 3825204430},
{"blockNumber": 11755003, "blockHash": "0xf9efb54f08a2b59086a607541005928f1bda54fa2c46fcc48352fa6aa461c03a", "firstIndex": 3892313541},
{"blockNumber": 11819591, "blockHash": "0xa551deb3bd2ec0ca2683cb76adb8fb295d171a3787aa5ff071bd1fa94d3af82e", "firstIndex": 3959422322},
{"blockNumber": 11883278, "blockHash": "0x60c7a359d4aeb09b63b64487fa7794aee87c41b11446f69ba83e24d5bce6947f", "firstIndex": 4026531239},
{"blockNumber": 11948436, "blockHash": "0x1c46fb22653e5f9eb9503fe532151519845d05b121291766715973e027eb9c0c", "firstIndex": 4093639741},
{"blockNumber": 12013084, "blockHash": "0x0b8d1da9e70c4494ba61190390c891230846e69d9b0be605df8b04c4199ffd1b", "firstIndex": 4160749561},
{"blockNumber": 12078612, "blockHash": "0xb049af260be25c6153b67c227689b0556d8fb7d6033c3f61b8218f5c4ac5917a", "firstIndex": 4227857310},
{"blockNumber": 12143549, "blockHash": "0x685dce41efbe2bab0ac9a4b9026d9e5faa3e01ae4637c8181cd2f1eac326fe6a", "firstIndex": 4294966232},
{"blockNumber": 12207918, "blockHash": "0x53644ea8f781c6270f1136ed4bbcb956c98f12a4793c2f56f49b3105282b13ea", "firstIndex": 4362075410},
{"blockNumber": 12272370, "blockHash": "0x503523e1a01cb7d68be0eb22f36280c0ee7a1fc802061a704e326fbb8efcfe27", "firstIndex": 4429184831},
{"blockNumber": 12329342, "blockHash": "0xe6a1bf78dd827def078e23c768cf6e96666ee06b5fdc113380a4769d85f4bd33", "firstIndex": 4496293151},
{"blockNumber": 12382311, "blockHash": "0x352fe793c16b1d8aea195af86ee9aec3dbaa83e7a32872a0d222a77e96fc0e57", "firstIndex": 4563402486},
{"blockNumber": 12436970, "blockHash": "0x90276349edf0941bb3327dc1d749d0371fe550eaddafb5141fcacdd281de0976", "firstIndex": 4630510187},
{"blockNumber": 12489954, "blockHash": "0x71894b1e1608aeba777662a6055e601ce6f17950dbe3844c3601e0f88cbd5848", "firstIndex": 4697619586},
{"blockNumber": 12541655, "blockHash": "0xc34c7299041df72a81c4b86e81639d2dfcebfae603e287fc143881a636ce1188", "firstIndex": 4764728948},
{"blockNumber": 12597319, "blockHash": "0x4c4dc71d8d2f1223ad0469ddfe09fa8934f64e3712ed83708d6e1b9c64eceebb", "firstIndex": 4831837390},
{"blockNumber": 12651863, "blockHash": "0x406a642c8364fb61cbd896512a2d19ecdba3932ab09ac67cf48d91aa1a03f24e", "firstIndex": 4898946362},
{"blockNumber": 12706391, "blockHash": "0xa7fc3739898f7ff5733896313b248fb0b306a132681609b8b9e2671b42dac3a1", "firstIndex": 4966054748},
{"blockNumber": 12762840, "blockHash": "0x5a1048966dfa55a48248517c45683ce342671bf544117d2bac13d8e343563136", "firstIndex": 5033164303},
{"blockNumber": 12816597, "blockHash": "0x97f5b87f422870a8ffeff70c12f4377dde96d248aa912cb64ebfaea51dc086b6", "firstIndex": 5100272541},
{"blockNumber": 12872315, "blockHash": "0x432f7df7973fdea4e5abe2c09b634ca3b724e37a19d5ec23ca5810e4854c6b33", "firstIndex": 5167382243},
{"blockNumber": 12929620, "blockHash": "0xa4524e3b4c5dadac91df2f29696532cfed377f47469fe7cfabf10baac945eea2", "firstIndex": 5234490748},
{"blockNumber": 12988653, "blockHash": "0x9be70e31aeea164c370d8d183a1af45120693b13d360edc78214535c40ef1d63", "firstIndex": 5301600248},
{"blockNumber": 13049074, "blockHash": "0x3076857bd17959c2a3df18357eab3ecbb7ca25457785b07074d2e885eb566fc2", "firstIndex": 5368707746},
{"blockNumber": 13108823, "blockHash": "0x67b2764f6b53779e3d683def2c7659c6d92814152574073d24f10cd67200a22b", "firstIndex": 5435817680},
{"blockNumber": 13175389, "blockHash": "0x0d41e34dbca58dec45e0509b6cbf027b2377220845a85caebbf3a14b8505e8ce", "firstIndex": 5502925085},
{"blockNumber": 13237624, "blockHash": "0x723e0818d54ab1b1c53efc2f2f448c16c1f943af2033575c612c3c71228d4ddf", "firstIndex": 5570034948},
{"blockNumber": 13298658, "blockHash": "0xd7f9f477b20bcf1379ccaaae701c9658a02b177861e0af9e873c63fe8b854f4b", "firstIndex": 5637144562},
{"blockNumber": 13361166, "blockHash": "0x9f9e4130093269d59045f91344a13eb2786023e02099f53dd44854d50a7f5ffb", "firstIndex": 5704253260},
{"blockNumber": 13421716, "blockHash": "0x28752f35ad2adeba342ef6aa51bca1c0abc757c4f712cf620c8a5e6a8576636e", "firstIndex": 5771361775},
{"blockNumber": 13480516, "blockHash": "0x43f4ecf24c17b7a77621e2ca09c88c53b3407fb018f4bb3a032c212015b4e18d", "firstIndex": 5838469616},
{"blockNumber": 13535688, "blockHash": "0x99ffcd9982219a4be45fa5f15622b288afe4f85da536956d39dccac1ec416444", "firstIndex": 5905579739},
{"blockNumber": 13590487, "blockHash": "0xff7b74464a318037e0670c0be7a5b1b79418d1a42c7a3587c46adc1ef720c819", "firstIndex": 5972688013},
{"blockNumber": 13646596, "blockHash": "0xd555ce5f2b65ffeec9b9e3e75a56f5f23d7353a1014bcc861efd7df6e5dc2c92", "firstIndex": 6039797525},
{"blockNumber": 13702902, "blockHash": "0x3e8feab590c6631c63931fee83bf7e10234890bdf377b1b374fb70221b9ac9c0", "firstIndex": 6106905641},
{"blockNumber": 13760908, "blockHash": "0x18e96fa585c57aad1f2a8dc9639d9f4fe53c3f38894862471b7cec17f97f3cb1", "firstIndex": 6174014330},
{"blockNumber": 13819340, "blockHash": "0x25384bf2cef178e30c4d9e0abae59357055241fc715133b2bb078e8278e5296a", "firstIndex": 6241123906},
{"blockNumber": 13877817, "blockHash": "0x4a84a0a7035dbfed0e1281fb1df162a195f2c516686a2e2a6ad605e6f2b3cf57", "firstIndex": 6308232690},
{"blockNumber": 13935260, "blockHash": "0x6c14a8e563614edbad66698ebe911e997e27844f9d33c7069795379540e71a44", "firstIndex": 6375341148},
{"blockNumber": 13993914, "blockHash": "0xc0991d1bc431c5215fb0218b0b541f2b4fae0994775e5eddb016965fe5b4a0c7", "firstIndex": 6442448947},
{"blockNumber": 14051007, "blockHash": "0x854803ce364b6fe4446800e74448a5ee883e7d472e8a0347f1ad13bd347fa69f", "firstIndex": 6509558964},
{"blockNumber": 14109063, "blockHash": "0xa3fadce7ab18a5fedcfed9e3b1f0cf2fb32151172c362327c602343bc26dde8f", "firstIndex": 6576668017},
{"blockNumber": 14166701, "blockHash": "0xaa2443a38e2b0d6b01cf4b73de54d2b7e92c99475bd0802beebd86845c126d1b", "firstIndex": 6643774648},
{"blockNumber": 14222367, "blockHash": "0xa27c78134c1fbf432c8caee7c9d2ef323fdde519a065804c9c8da6ca1615e04f", "firstIndex": 6710886238},
{"blockNumber": 14277029, "blockHash": "0x5d0ed08b65db01e382c339dcb2715b2927fb472b0c1f4256841c386d785c3f3a", "firstIndex": 6777995247},
{"blockNumber": 14330955, "blockHash": "0x3e5014d82c416549acfe2e26b13ff5ddf29ad74943ef5b94a65eca0a4d72d82e", "firstIndex": 6845102969},
{"blockNumber": 14384139, "blockHash": "0x720e1fd687292019c3fa1812c39c73d20fde783c0196f43378a51cb4e0b46d65", "firstIndex": 6912212078},
{"blockNumber": 14437534, "blockHash": "0xe4342ab6f363066d9c2514a870241ceb63df232d33659fdb3928faf433360bb4", "firstIndex": 6979321493},
{"blockNumber": 14489073, "blockHash": "0xeb2f9923d816e773460eccfa42eb4ff2bbfc353b756ba274e98dc85c64ab1536", "firstIndex": 7046428879},
{"blockNumber": 14541559, "blockHash": "0x55cc33721fa9022964f47c30f4146fee7cc80f5b4158d7ecea0ff661cbba3487", "firstIndex": 7113538857},
{"blockNumber": 14594432, "blockHash": "0x676441bdc1ec430a9fbc05e56a0437a475606a253f9fdffc86f5f6286c159b5d", "firstIndex": 7180647956},
{"blockNumber": 14644942, "blockHash": "0x03232a1d3c9b2f1f0c2a4b780c83bce66238d9abdeed71c2a8af9e97a53011b0", "firstIndex": 7247754981},
{"blockNumber": 14695793, "blockHash": "0x6017f6d1d79188692e1a1a18de5f93b83b97905c8ff5bed5cef4b2e9f4d4d41b", "firstIndex": 7314863646},
{"blockNumber": 14746186, "blockHash": "0x332c8ac0b2960fb1627407d45443201a0f6de7421b52af077b32fa6e018e5401", "firstIndex": 7381974499},
{"blockNumber": 14798403, "blockHash": "0x4507ae1d9f27a3eada874dcf04112a5d4ee7155701afd3fb857fe58201a2dede", "firstIndex": 7449082752},
{"blockNumber": 14848093, "blockHash": "0xff98c820c66c3beaaa3b195158c94f099148573a7f032295ba7156c13f78eaa8", "firstIndex": 7516191978},
{"blockNumber": 14897508, "blockHash": "0x49c31d3433b197585f733d7c382ae16d9d78bf59c333a9b76362cc6c5fed270c", "firstIndex": 7583300663},
{"blockNumber": 14950638, "blockHash": "0xf1a1609206788977841927271808fe1c42d059026385d025bf4cea0c0a05966b", "firstIndex": 7650408992},
{"blockNumber": 15003958, "blockHash": "0xf9e6b41108d907672f29ba1df97ed83aca2ba933335e6b5ccfb5d4001807b33f", "firstIndex": 7717519105},
{"blockNumber": 15056783, "blockHash": "0x126a5a5c22e878a4c42bea890d48cfe00edf7fbc27d805c88a5269540bc94a42", "firstIndex": 7784628173},
{"blockNumber": 15108163, "blockHash": "0x5f8494ccacefce9fef8a95a52247f2670ed30bb5a3b3deb85c645e72c115d506", "firstIndex": 7851735261},
{"blockNumber": 15159385, "blockHash": "0x0fed6c5aafb53e9cdb0d8620c3eef9b619fb3ae9559296618f634eddae33f0c6", "firstIndex": 7918844299},
{"blockNumber": 15210872, "blockHash": "0x37bdb94b57044b67fbd9b9a7cdc728fcfd2066cf1904cab478fd42d41ec6e1f0", "firstIndex": 7985954119},
{"blockNumber": 15264226, "blockHash": "0xc173ad8a791e2293efd8e0b2c9f088fac49233aabb1e72db6a045aa9904f4728", "firstIndex": 8053061840},
{"blockNumber": 15315693, "blockHash": "0x985cf97bf28f0becb2b16abfe629a5f520596e45fc23f4936bc2f6298b8145f9", "firstIndex": 8120172048},
{"blockNumber": 15364606, "blockHash": "0x593362ef8a82140f3dd3ff13d0ce50287dcb0b8b3d31ffcc7813c6be16f7438d", "firstIndex": 8187281275},
{"blockNumber": 15412727, "blockHash": "0x5a6ff10acc48e9e8bcf963b042012ea094628b9488b8bf569783b3297c67eedd", "firstIndex": 8254388575},
{"blockNumber": 15462642, "blockHash": "0x137f15c5d112b0fdc6e4c84393032030646122c3b0d86286143a1ab8bfacbac4", "firstIndex": 8321497214},
{"blockNumber": 15509781, "blockHash": "0xe58a8ca223b7c03755d046a6212bc33fa9814f73efe52804d8eea7ae4b4bd7f9", "firstIndex": 8388606496},
{"blockNumber": 15558625, "blockHash": "0x4127d527b58c7685a1237d228508ea77d40f33dbeffb5cc30e47e83a4b6152a5", "firstIndex": 8455714910},
{"blockNumber": 15604075, "blockHash": "0x7d1c1bccfe5c23a1edd81246e6e55277f271cffef30bde22c9d1202f764dd96f", "firstIndex": 8522823324},
{"blockNumber": 15651742, "blockHash": "0x732e5203a504fd97510325c1fb876476ca132831e78182367cd4b4efae1232c3", "firstIndex": 8589934155},
{"blockNumber": 15700810, "blockHash": "0xe031decb5e42a69dafcc814441227085508cf51c9c77f35d06487db19bd05a6a", "firstIndex": 8657041374},
{"blockNumber": 15762855, "blockHash": "0xbdff9b1e193c44f35bf38945d2523c74ef59c4f352efc93fad02ef34eef1c5f0", "firstIndex": 8724150924},
{"blockNumber": 15814304, "blockHash": "0x524a226d9f48e37db00e252c7f6a1350d1829d38fe83c756021cc11c56d3b2dd", "firstIndex": 8791259373},
{"blockNumber": 15865514, "blockHash": "0x4e4ad46f3ae7fb2a2e7cb94aa804929cb7353c79c78cc2cbcee710da5f0ec4b5", "firstIndex": 8858369571},
{"blockNumber": 15920411, "blockHash": "0xa39e4cbd8e84b427e44b8d06b5b1149652f3af5fa88633b802665b8677772bc5", "firstIndex": 8925477782},
{"blockNumber": 15974193, "blockHash": "0x4a3f82935e6fcda6ce7636001989b65b79422d362e60833e01185a7960b2fa55", "firstIndex": 8992587369},
{"blockNumber": 16032739, "blockHash": "0x7f50bb76dfd01125e0a4a7595106e43329eb8bd02e819be2ade4248a403c714a", "firstIndex": 9059696611},
{"blockNumber": 16090896, "blockHash": "0xdc4db130973ef793d2223844e35122a2477e642d86c69616fb3604d2a79805c4", "firstIndex": 9126804425},
{"blockNumber": 16150783, "blockHash": "0xb46294fe1cbcbbdff7309ebf0bf481e78e2589d386b95fa4c781c437e02e0aec", "firstIndex": 9193912139},
{"blockNumber": 16207278, "blockHash": "0x8a3af0387572b56989bb6c98d8860e07b5cdefb2ebe7b1dc58be07c24a53e056", "firstIndex": 9261022215},
{"blockNumber": 16262389, "blockHash": "0xec5a03d88cfc1bf8c278ddce7ace5032c3f7ed5032b4575df54779ec1abca131", "firstIndex": 9328130680},
{"blockNumber": 16319525, "blockHash": "0x74ae702375e242205fe4986448b0ee494d4a38ccc813981489c4b0488e58a949", "firstIndex": 9395240779},
{"blockNumber": 16373441, "blockHash": "0x5f404555cf45c547ef0fc044f2701bc7a28ce1e432de06e0264dbfd1c17447d0", "firstIndex": 9462347671},
{"blockNumber": 16423343, "blockHash": "0xe4af890b6c4dd3b10af2e6c08f14563bd2a3121b22a3cc6a0db29d438c1dcc59", "firstIndex": 9529457704},
{"blockNumber": 16474687, "blockHash": "0x019a8b374cf150fcda1c8af2a3e7ba63603ae81744d6343897c4b17ee58411d5", "firstIndex": 9596566169},
{"blockNumber": 16525504, "blockHash": "0xad394ce7e84e1c904cc5dd6dabd51b1a575c0b2fb006f10f272875b7f921904e", "firstIndex": 9663675287},
{"blockNumber": 16574051, "blockHash": "0x098e2dca2eda0d700aa3127bdfb679ce16e3ca297a305978c6ecfff044269511", "firstIndex": 9730784563},
{"blockNumber": 16622457, "blockHash": "0x8627f1237731c1127116898724f1b6420b8e7bcd8e18a0ea568f192bbd2cd55d", "firstIndex": 9797893241},
{"blockNumber": 16672410, "blockHash": "0x3e32b46faa670b191b89bea84a68022d5ea8b90008b388a948ff27f871d17292", "firstIndex": 9865002964},
{"blockNumber": 16719964, "blockHash": "0x1668c3ea6cb256f306e93851cc2c00612f85df5cf7b5527ffce14c89088d7fdf", "firstIndex": 9932111708},
{"blockNumber": 16769014, "blockHash": "0xed9a6f88e47b5bfe4d5bc92e1c1bfeb252f716f0990ffa1f907655c0f5e3a365", "firstIndex": 9999220312},
{"blockNumber": 16818134, "blockHash": "0x93504ba617500698a1b46f3110cd26827b8efe468190b2a91d3595784e34a1b6", "firstIndex": 10066329167},
{"blockNumber": 16869369, "blockHash": "0x94ca4b395a5c36c4e9d07cbb08271f0680455af4e4659906b647e16abebd7d4e", "firstIndex": 10133438415},
{"blockNumber": 16921428, "blockHash": "0x66650775bff4cb95de56099c10b06f3dfe302a731b391f406d1aa551121308ba", "firstIndex": 10200547148},
{"blockNumber": 16974677, "blockHash": "0x58d5fc4cd45047dac10d55a036e0a4448faacc1041b29c4c7fb41a6e9394c3ff", "firstIndex": 10267655623},
{"blockNumber": 17031050, "blockHash": "0x9d15b831979e95fc4149ecae46478cac1e54470b277d57d7102d19a72f28eeba", "firstIndex": 10334764441},
{"blockNumber": 17086163, "blockHash": "0x978ff48863fcdb50b274ea4220e7635c13c41bae91912db9e24205fb917fde85", "firstIndex": 10401873176},
{"blockNumber": 17141086, "blockHash": "0x4084d442499893361731f17534944d260cccd8e07d41c52d7160910555770b90", "firstIndex": 10468982462},
{"blockNumber": 17190735, "blockHash": "0x42ed86fb42def23870df6ec23204f557669ec7cd8de59c9f3ef9ab90704e4b11", "firstIndex": 10536090726},
{"blockNumber": 17237026, "blockHash": "0x9a49d943f82a7335f8049df4d953080f25920e3b098f6a2d872520166b225961", "firstIndex": 10603199938},
{"blockNumber": 17287007, "blockHash": "0xac3d189676d24eadc62adcf97d8730a001a7766fe3ac27b18e745ca9727f6c46", "firstIndex": 10670308186},
{"blockNumber": 17338498, "blockHash": "0xe50e251aa0a3209fa8a0030b237cf8357b08eeb80492a432c8aa5969a820ee42", "firstIndex": 10737417623},
{"blockNumber": 17389178, "blockHash": "0x1e18cedcfa6c81bc845c9136fc92f9f27ca8fff8847289bb139f374f0f6ba733", "firstIndex": 10804526769},
{"blockNumber": 17442038, "blockHash": "0x07f47ebbee99938fb96beb1ffadf8266f71d0d73f393172f0e24facd40d45c97", "firstIndex": 10871635940},
{"blockNumber": 17497589, "blockHash": "0x247a5e852e36e46f7897ad1875e93d810ff4434bb835b469afae3a3d3d4a9c9a", "firstIndex": 10938744251},
{"blockNumber": 17554111, "blockHash": "0x420e420a1b599e4dc3bd115652b4d6f5eedc391086ef064f97a9cbffc02b62f9", "firstIndex": 11005853686},
{"blockNumber": 17608282, "blockHash": "0xb2ba644d3593d3c832f932f14ffec532e26e9e1cdcd8fbe7477f46e238c6bffd", "firstIndex": 11072962303},
{"blockNumber": 17664091, "blockHash": "0x6d0bfff8145652a617bfe03ab87e3af09219cdb822c9c214b42b16f20742c1eb", "firstIndex": 11140070260},
{"blockNumber": 17714917, "blockHash": "0x61509544b5174d3ff5d48816ff54f2929c8a400ef21454116056baa136b607b6", "firstIndex": 11207179013},
{"blockNumber": 17763850, "blockHash": "0x8da83012621f42670601406c902ce6e48e77c6285dcc6983e108e69b3a8d5d2d", "firstIndex": 11274288732},
{"blockNumber": 17814186, "blockHash": "0xc37bda200856589c220dc206e31186d6c12761fd193f5bc7123cf331fbbed7f5", "firstIndex": 11341397343},
{"blockNumber": 17864461, "blockHash": "0x99af43893dc9f54c55abbdbd7b000e27248145371e9b00db116dfb961b36749c", "firstIndex": 11408506122},
{"blockNumber": 17913212, "blockHash": "0x9b642236d63d756fcd3404713bc6abe97804b46a9e665ae5f1d9aa48e693fe78", "firstIndex": 11475614347},
{"blockNumber": 17961497, "blockHash": "0xaec9b1e851cf51646673ec7698e1d5964d44eb04729ad11385f6ae84d497070a", "firstIndex": 11542723713},
{"blockNumber": 18010859, "blockHash": "0xb15fb7fc308516cad5f1dadfb7dd92fd3fe612308f67154cf2a241990aa15bc4", "firstIndex": 11609833204},
{"blockNumber": 18061052, "blockHash": "0xd8ea94b8b8a7380d21dc433abb69973f3de2eeb339f4aaffdb79f85a5d86929d", "firstIndex": 11676938565},
{"blockNumber": 18111466, "blockHash": "0x5b5cc6d182bffaf06692a7efda8cad5d4742fdbce5c1146b3be3c5be10ea046a", "firstIndex": 11744048539},
{"blockNumber": 18166001, "blockHash": "0x901d8401c795542c75069cd917781fa587ad416da0a5298f144fae2f14305705", "firstIndex": 11811159949},
{"blockNumber": 18218505, "blockHash": "0x5894285023f34ec6fd501cd90b605e118b9688ca7c01082289bf69a6987142bc", "firstIndex": 11878268395},
{"blockNumber": 18271037, "blockHash": "0x3f0fce315fd993d966b6c599ebd360462466e67046d512dc430ac07f75243bae", "firstIndex": 11945377050},
{"blockNumber": 18322778, "blockHash": "0xfa6536b3b06e7b579e53a8d01368182808c950586e39e12485be3145c3acbf43", "firstIndex": 12012486036},
{"blockNumber": 18372240, "blockHash": "0x5531859d5b8a4093020e2fb971808d0eeaddd7d61dc58d70f34923081f97336f", "firstIndex": 12079595358},
{"blockNumber": 18421630, "blockHash": "0xfeadff676fe8f04d0362b77ea2d7c3ea59b05d68cc6b2cc95cfe751558b1bfa7", "firstIndex": 12146703872},
{"blockNumber": 18471472, "blockHash": "0x352b0a17e4160d334b876eb180d686fd67c7ddc40a78219b0bc4b59f5ae5912d", "firstIndex": 12213813174},
{"blockNumber": 18522088, "blockHash": "0xf7d408309246089531b47c787049c7ffd2820c1b87b4804033a458bc193a3432", "firstIndex": 12280921181},
{"blockNumber": 18572725, "blockHash": "0x44126d4a5b43594b09d26e0b0a5dc859c074865cb9b11ab78c80f28406fd0240", "firstIndex": 12348029615},
{"blockNumber": 18623223, "blockHash": "0xd3aec1f98cea30e870d0f530194e9a72e225eb45eefe435e38da5967eb2fa470", "firstIndex": 12415139522},
{"blockNumber": 18675228, "blockHash": "0x24deaef2508038bddf8e9020f95128ae34eb1786f786c3f98df0bdf95f3e97cc", "firstIndex": 12482248123},
{"blockNumber": 18725495, "blockHash": "0x2e5a7edd7337b20e862741e116680a472089426dbc2b0eb5f1c984f8d6da935b", "firstIndex": 12549357121},
{"blockNumber": 18778174, "blockHash": "0xe734a1084fa5eeff427bba8e8d3623181a30dd9bad128fc22de9b15dffd0e549", "firstIndex": 12616466131},
{"blockNumber": 18834807, "blockHash": "0x0924a55ef4f9c2605261433c5b236d47645484efa44b34dc9f15b7c2e6b47a74", "firstIndex": 12683575000},
{"blockNumber": 18883054, "blockHash": "0xef638ec491a1cf685b22bf4072e41710440d6133a53e31250762825247d3eb64", "firstIndex": 12750683712},
{"blockNumber": 18933405, "blockHash": "0x6e9eefc2e121d13e6c04544ea370fe8543afeb36fef3a8e2ea68afb079738c8c", "firstIndex": 12817792510},
{"blockNumber": 18988033, "blockHash": "0x4c98c0dbb6a7217609c4b277fe7ff9e2c32c8d000ee628a3f37b2b296e1ca421", "firstIndex": 12884901389},
{"blockNumber": 19041096, "blockHash": "0x61a219a23d111cd07861ff47b89d4b2840c0ab5421d5d55a6f1fe43da3f33b98", "firstIndex": 12952010147},
{"blockNumber": 19088914, "blockHash": "0xefb26ba58447e50e9b820e0b0d16c08da45793a824af12a2e1f38b6f46b7e36a", "firstIndex": 13019118891},
{"blockNumber": 19140413, "blockHash": "0x91dbb13623fc4d5a4e462dfa0a03c92bf5301948dd4b2c3e5fddd2bc88856d47", "firstIndex": 13086228137},
{"blockNumber": 19191876, "blockHash": "0xdd7f97af7ad206b2636656ed4722fe1c47c6ba9fa5531066785acbe4f8d4c4ad", "firstIndex": 13153336551},
{"blockNumber": 19237651, "blockHash": "0xf1e875f314314abe3654b667c6443c772dd67c525c52877b06515ea8fc5a391b", "firstIndex": 13220444812},
{"blockNumber": 19291032, "blockHash": "0x9eda269ac47957b619d2a8c408b5efc8cfb168a46bfde351f43325fa337fce47", "firstIndex": 13287553967},
{"blockNumber": 19344225, "blockHash": "0x62859b78d45f589f679fc6a3bf266a2e0aa906fd61d787755ec51cfadc05eaf6", "firstIndex": 13354663338},
{"blockNumber": 19394709, "blockHash": "0x7cda3d193bcfe8ddbc89b85e98c814dbbe31a54ccab52ab7738ecd003061f3c8", "firstIndex": 13421772549},
{"blockNumber": 19442850, "blockHash": "0xd95f3f8bab5b91c3e1f848c9455936850bf92cf6a5086b817f6f81031bc0f81e", "firstIndex": 13488880704},
{"blockNumber": 19488177, "blockHash": "0x3a1a9623ee332d6decada9359cf84eeafb81fffa997d6cf4f3efcc92e98e9310", "firstIndex": 13555989570},
{"blockNumber": 19534365, "blockHash": "0x5c7a8ee7cbba4894ea412cc335895a9a7d3369d35779bcdc97a6c24b6a81a7ba", "firstIndex": 13623097446},
{"blockNumber": 19579409, "blockHash": "0x4280f5f76cb0f9947f9da1cb9de6c4b6e65b37b3a3c174f5fb29d360cf389ffb", "firstIndex": 13690206790},
{"blockNumber": 19621406, "blockHash": "0x0a98ca72e97fd295c7c73263b17185d74dbe427f41c616a5d5454faa5a1c63a0", "firstIndex": 13757316653},
{"blockNumber": 19664868, "blockHash": "0xc8c53d89b7a849861f3daaf9503e6397a0340cecb085850c3636bee6ea2b7189", "firstIndex": 13824424608},
{"blockNumber": 19709011, "blockHash": "0xf65babc88ef6a9eb2ffce9a6844adf895995f65a2facd1956f5df17ca68dbf94", "firstIndex": 13891534141},
{"blockNumber": 19755170, "blockHash": "0xf903882449eeb86d69af33b6797099275e82d8787334a02a42a2168dbd739b95", "firstIndex": 13958642627},
{"blockNumber": 19803666, "blockHash": "0xa8a975f570db082a3771790bb1fe405667dde4674ff49e31092687fa92727e36", "firstIndex": 14025752433},
{"blockNumber": 19847730, "blockHash": "0x89d479a4b4d9faa7af3e3f61d13a7a9c80a11c4adf7cd018859eb0ddba30bdd5", "firstIndex": 14092860598},
{"blockNumber": 19894138, "blockHash": "0x740278e6dc79cfd2b030780868a1b8e1520dbae0e8d3ee0a3ef43c84c83c6e0a", "firstIndex": 14159952129},
{"blockNumber": 19938334, "blockHash": "0x05ae7ce9813235c19baa57658bc5c8e160130328a1fd2cc58d2baf2d9a4ae097", "firstIndex": 14227078393},
{"blockNumber": 19985138, "blockHash": "0x511c8876a93a77fa473ae0d29e25d8163162c09b7212f506be0ea73840ea3971", "firstIndex": 14294187397},
{"blockNumber": 20028221, "blockHash": "0x64379baedf429cfa65df076142a7d9909d7e369d6aa6b8e161bb686944d169ff", "firstIndex": 14361296446},
{"blockNumber": 20071555, "blockHash": "0x4f43d8b1091d2bd0c6b080ce51bce08aae6be605e00889be3cdd17cb80614ff8", "firstIndex": 14428405409},
{"blockNumber": 20113611, "blockHash": "0xebbf87409ef6732a53513414926b0aa998a7269f9e9717a3acbe96a33c2de937", "firstIndex": 14495514255},
{"blockNumber": 20156628, "blockHash": "0x70218108c0669f42c4ac1afff80947cf4a0a2fd8e9300b31d45bbe330a5159c9", "firstIndex": 14562622257},
{"blockNumber": 20199886, "blockHash": "0x1d6aaa4b041eb0cc9caed34347bb943e50a139caf8998d781ff4f06d4aa727dc", "firstIndex": 14629732017},
{"blockNumber": 20244148, "blockHash": "0x37fed57b6945a815ec56c84c6261e90cf9f3883d9cbffc62b70d9722c1f0f363", "firstIndex": 14696839730},
{"blockNumber": 20288249, "blockHash": "0x1d9fe664431e093c4ef6139ad35fea767a2e66825e605cfea9e91d1738665619", "firstIndex": 14763948803},
{"blockNumber": 20333358, "blockHash": "0x1ac7c1362791507057b7e87d7118b4f61c2919c8ebdb498ecf7d64a59ba3d2e2", "firstIndex": 14831058814},
{"blockNumber": 20376855, "blockHash": "0xf6a19eac8a402f591079a4aa7ff57349c350ec96d683a031e4d85c05bc0541b2", "firstIndex": 14898167076},
{"blockNumber": 20421474, "blockHash": "0xe8767aa7516b632dc8c0fed3a366273ef14bd7b8c2212998b7105fb80b4c0d9b", "firstIndex": 14965276285},
{"blockNumber": 20466974, "blockHash": "0xafa2d2e6c9594762c163c339069cb15aa9fd27d301268892e12f99a3f2912ade", "firstIndex": 15032384674},
{"blockNumber": 20512183, "blockHash": "0xc1c51f9372d781d7ac0a1a7aa052503bbde3e1706bf65f225b0f6a6eed93d055", "firstIndex": 15099493832},
{"blockNumber": 20557251, "blockHash": "0xa905087c15e2b12b19cd23f20089b471cd9815dcff73842d2043e13a276fcc55", "firstIndex": 15166601921},
{"blockNumber": 20595654, "blockHash": "0x9e7772dd946f65e7569f34f6896d8e597ac8ff88700b871de21ff578c0c995e6", "firstIndex": 15233711474},
{"blockNumber": 20638340, "blockHash": "0xc08b46af83db6f0e5d1acad7020bacd59d0f18df72651d32c9b60edd0351a886", "firstIndex": 15300820584},
{"blockNumber": 20683359, "blockHash": "0x7ec93a1f1d211c24d93ce7e344634efef680a6388374cf3831d250ecd63711f9", "firstIndex": 15367929168},
{"blockNumber": 20728401, "blockHash": "0x4ad4124c2595abcf4477cac4eb70c0c9372bbd48f2066c7ac5b8e9f94817ab48", "firstIndex": 15435038303},
{"blockNumber": 20771272, "blockHash": "0x68fd6b12fdb4add8b2957105a21b571c7dcb114c6c2ca02f5fa40730a108f864", "firstIndex": 15502146509},
{"blockNumber": 20814862, "blockHash": "0x4e7f640bf3c135b2442ca4c744023021af163e8d45e8d776fb7051c49d79fcff", "firstIndex": 15569256173},
{"blockNumber": 20857627, "blockHash": "0x9b023bffed12530b5a68db263bb621a147e0cf02b30ec39a82b82a1fdeb7ca36", "firstIndex": 15636364955},
{"blockNumber": 20896662, "blockHash": "0x8c3adf694b8520ed1edd38282b2f58cf383426176628b5be6330a56882e2a07e", "firstIndex": 15703473445},
{"blockNumber": 20939156, "blockHash": "0x0649e2559f767da945a674c9d6c700628109e18d0757ad33496a228a827413a4", "firstIndex": 15770581979},
{"blockNumber": 20981054, "blockHash": "0xebf0ecca671b833f77df70106cf1171d4dd3fa0f062607e4845569073997f1cd", "firstIndex": 15837691319},
{"blockNumber": 21022980, "blockHash": "0x07479520c08a727c42e3616aca7fc4435773f4ef277d84369190263159606d71", "firstIndex": 15904799391},
{"blockNumber": 21068139, "blockHash": "0x669cfd03b293a61815588cb9cef928a512d328c1fe9191fe6b281573f429f8b8", "firstIndex": 15971908604},
{"blockNumber": 21112368, "blockHash": "0x98957aedaf8b0483be208e8d16a3e6d73ff217e732f3649650671c92c8fb57a5", "firstIndex": 16039007815},
{"blockNumber": 21155756, "blockHash": "0xc301e40988fbe19b1254ab3abe2f78fe0cd35259898be02476179a041926f777", "firstIndex": 16106127130},
{"blockNumber": 21200717, "blockHash": "0x8d35b199ee65c5ccf4ca0fc22b50282850cd88b72580679f1568eea67d3431e8", "firstIndex": 16173235593},
{"blockNumber": 21244396, "blockHash": "0xe7ec084eb8a7125fe26c4ef64fdbb3e0a727b2d882fb1e4fdad1d7fa3dc55fd2", "firstIndex": 16240343803},
{"blockNumber": 21290031, "blockHash": "0x8bb31b883d30875b7ed8a01c35a6f51e603b8d974c4e74af20fb9c723e5ac5df", "firstIndex": 16307452772},
{"blockNumber": 21335893, "blockHash": "0xd0976b7c946a7556e607e47a02d2cfcccd259508e2828b9c4faa5c7105a597a8", "firstIndex": 16374562657},
{"blockNumber": 21378611, "blockHash": "0x72d42953f013aeb7c84bb22c9e07828b92f73885b20c208af612874df8fe85ed", "firstIndex": 16441671192},
{"blockNumber": 21421343, "blockHash": "0x663b3cddcf3f7cfa59ed29b4894e43fa784c07576cfec897ff1a3838235ecb50", "firstIndex": 16508779766},
{"blockNumber": 21466598, "blockHash": "0xe04ebd60289dbcc09a27e053e7b240a59e928e2a76a85a6fa4044ee387c46d18", "firstIndex": 16575888836},
{"blockNumber": 21511717, "blockHash": "0x5ea9863a0b504f381010ff383d60a56390ba9cef162d9d5a077b1884d2adac98", "firstIndex": 16642998234},
{"blockNumber": 21550050, "blockHash": "0x2a7b3e635c80521887ca3ebda04541090ec720dd817cd06fcba61e7ae0f78a4f", "firstIndex": 16710106733},
{"blockNumber": 21592440, "blockHash": "0xc7095268d0d53cb5a069c2ea7107cc42172bb1b817f04154938e6b52bdbb369c", "firstIndex": 16777215261},
{"blockNumber": 21636007, "blockHash": "0x55f463b932053ebfc02728a12c2cdfb2cb4700a115cde686c9ab22bb4c336ecb", "firstIndex": 16844324844},
{"blockNumber": 21680799, "blockHash": "0x4521d473aa7f5150f920c66fe639b20f30cb373b32e425fcb7dcce9803b506be", "firstIndex": 16911431255},
{"blockNumber": 21725330, "blockHash": "0x793876007ce3594df603628766b139a02273ceee43d5b3ca2d0ca2f13a5fc5d9", "firstIndex": 16978541626},
{"blockNumber": 21771310, "blockHash": "0xd0db23bb28d6e39917348b1879dd560c73354ea6d7d0bbee0b7a8fcbcede1091", "firstIndex": 17045650902},
{"blockNumber": 21811348, "blockHash": "0x8165948c08db34aeb8cc2a3e331a00af2d09fbff9c466019fb2f2892ca3f7b71", "firstIndex": 17112760024},
{"blockNumber": 21851615, "blockHash": "0x3083e8f4fbb4d726ef19b1a5bce20e639a0418eac384f9967f24c7697a7e2dfc", "firstIndex": 17179868870},
{"blockNumber": 21894041, "blockHash": "0x9b0776a020374e4985f6a7e1199596bedd387ff085f65224d724ab11955c56fe", "firstIndex": 17246977342},
{"blockNumber": 21932218, "blockHash": "0xe9c50eddc21759d3ab790e2babba56f4b8d38eb739f1b561bc337a1ba6e15393", "firstIndex": 17314085541},
{"blockNumber": 21959188, "blockHash": "0x6b9c482cc4822af2ad62a359b923062556ab6a046d1a39a8243b764848690601", "firstIndex": 17381193886},
{"blockNumber": 21994911, "blockHash": "0x0d99ef9dd42dd1c62fc5249eb4f618e7672ab93fa0ba7545c77360371cf972e5", "firstIndex": 17448302795},
{"blockNumber": 22026007, "blockHash": "0xf7cdc7f6694f2c2ed31fa8a607f65cfa59d0dd7d7424ab5c017f959ae2982c71", "firstIndex": 17515413036},
{"blockNumber": 22059890, "blockHash": "0x82b768a0dddefda2eefd3a33596ea2be075312f1dd4b01f6b0d377faca2af98b", "firstIndex": 17582521768},
{"blockNumber": 22090784, "blockHash": "0xf97c2eaf9a550360ac24000c0ff17ffa388a2bdd6f73f2f36718e332edfa107a", "firstIndex": 17649630983},
{"blockNumber": 22121157, "blockHash": "0xa790025235db782e899f23d8b09663ec2d74ec149e4125d62989f98829b08e2d", "firstIndex": 17716724973},
{"blockNumber": 22148056, "blockHash": "0xbe25ac4f1bdd89a7db5782bf1157e6c4378d80220d67a029397853ef16cd1a4c", "firstIndex": 17783838366},
{"blockNumber": 22168652, "blockHash": "0x6ae43618c915e636794e2cc2d75dde9992766881c7405fe6479c045ed4bee57e", "firstIndex": 17850956277},
{"blockNumber": 22190975, "blockHash": "0x9437121647899a4b7b84d67fbea7cc6ff967481c2eab4328ccd86e2cefe19420", "firstIndex": 17918066140},
{"blockNumber": 22234357, "blockHash": "0x036030830134f9224160d5a0b62da35ec7813dc8855d554bd22e9d38545243ed", "firstIndex": 17985175075},
{"blockNumber": 22276736, "blockHash": "0x5ceb96d98aa1b4c1c2f2fa253ae9cdb1b04e0420c11bf795400e8762c0a1635c", "firstIndex": 18052284344},
{"blockNumber": 22321282, "blockHash": "0x8a601ebf6a757020c6d43a978f0bd2c150c4acc1ffdd50c7ee88afc78b0c11f8", "firstIndex": 18119392242},
{"blockNumber": 22349231, "blockHash": "0xb751c026a92ba5be95ad7ea4e2729a175b0d0e11a4c108f47cab232b4715d1a2", "firstIndex": 18186501218},
{"blockNumber": 22377469, "blockHash": "0xa47916860a22f7e26761ec2d7f717410791bd3ed0237b2f6266750214c7bbf08", "firstIndex": 18253610249},
{"blockNumber": 22422685, "blockHash": "0x8beaee39603af55fad222730f556c840c41cd76a5eef0bad367ac94d3b86c7aa", "firstIndex": 18320716377},
{"blockNumber": 22462378, "blockHash": "0x6dba9c5d2949f5a6a072267b590e8b15e6fb157a0fc22719387f1fd6bfcd8d5d", "firstIndex": 18387828426},
{"blockNumber": 22500185, "blockHash": "0x2484c380df0a8f7edfdf8d917570d23fab8499aea80c35b6cf4e5fe1e34106e9", "firstIndex": 18454936227},
{"blockNumber": 22539624, "blockHash": "0xd418071906803d25afc3842a6a6468ad3b5fea27107b314ce4e2ccf08b478acf", "firstIndex": 18522044531},
{"blockNumber": 22577021, "blockHash": "0xff222982693f3ff60d2097822171f80a6ddd979080aeb7e995bfb1b973497c84", "firstIndex": 18589154438},
{"blockNumber": 22614525, "blockHash": "0x9868da1fea2ffca3f67e35570f02eb5707b27f6967ea4a109eb4ddbf24566efd", "firstIndex": 18656264174},
{"blockNumber": 22652848, "blockHash": "0x060a911da11ab0f1dda307f5196e622d23901d198925749e70ab58a439477c5a", "firstIndex": 18723372617},
{"blockNumber": 22692432, "blockHash": "0x6a937f2c283aba8c778c1f5ef340b225fd820f8a7dfa6f24f5fe541994f32f2d", "firstIndex": 18790480232},
{"blockNumber": 22731200, "blockHash": "0x00d57a9e7a2dad252436fe9f0382c6a8860d301a9f9ffe6d7ac64c82b95300f8", "firstIndex": 18857590076},
{"blockNumber": 22769000, "blockHash": "0xa48db20307c19c373ef2d31d85088ea14b8df0450491c31982504c87b04edbc0", "firstIndex": 18924699130},
{"blockNumber": 22808126, "blockHash": "0x1419c64ff003edca0586f1c8ec3063da5c54c57ff826cfb34bc866cc18949653", "firstIndex": 18991807807},
{"blockNumber": 22845231, "blockHash": "0x691f87217e61c5d7ae9ad53a44d30e1ab6b1cc3f2b689b9fbf7c38fbacacfe3e", "firstIndex": 19058917062},
{"blockNumber": 22884189, "blockHash": "0x7f102d44c0ea7803f5b0e1a98a6abf0e8383eb99fb114d6f7b4591753ce8bba3", "firstIndex": 19126024122},
{"blockNumber": 22920923, "blockHash": "0x04fe6179495016fc3fe56d8ef5311c360a5761a898262173849c3494fdd73d92", "firstIndex": 19193134595},
{"blockNumber": 22958100, "blockHash": "0xe38e0ff7b0c4065ca42ea577bc32f2566ca46f2ddeedcc4bc1f8fb00e7f26329", "firstIndex": 19260242424},
{"blockNumber": 22988600, "blockHash": "0x04ca74758b22e0ea54b8c992022ff21c16a2af9c45144c3b0f80de921a7eee82", "firstIndex": 19327351273},
{"blockNumber": 23018392, "blockHash": "0x61cc979b00bc97b48356f986a5b9ec997d674bc904c2a2e4b0f17de08e50b3bb", "firstIndex": 19394459627},
{"blockNumber": 23048524, "blockHash": "0x489de15d95739ede4ab15e8b5151d80d4dc85ae10e7be800b1a4723094a678df", "firstIndex": 19461570073}
]

194
core/filtermaps/checkpoints_sepolia.json
View file

@ -1,99 +1,99 @@
[
{"blockNumber": 3246675, "blockId": "0x36bf7de9e1f151963088ca3efa206b6e78411d699d2f64f3bf86895294275e0b", "firstIndex": 67108765},
{"blockNumber": 3575560, "blockId": "0x4652e3bee440f946aeaa3358c9a62b9bc4d41f663737ab00fded00c2662bfa29", "firstIndex": 134217644},
{"blockNumber": 3694262, "blockId": "0x433573f97e563ca04fa53e0d0eb019d0ffe9dd032a05ee5b33ce7705aedfd34d", "firstIndex": 201316990},
{"blockNumber": 3725628, "blockId": "0x1e25ebd76b9e2a2007bfdd7c82433cce7dae8beb190d249c82bf83d0cd177735", "firstIndex": 268426577},
{"blockNumber": 3795378, "blockId": "0xca999083e01d5947212fb878d529abecdd18a6012997b0d5326afcf6908f17e5", "firstIndex": 335543438},
{"blockNumber": 3856681, "blockId": "0x48148cd35de3de6c626f76ce5d97cfc6acd88772af69c0aef3d640c29ba82096", "firstIndex": 402641717},
{"blockNumber": 3869367, "blockId": "0x08bafccf9ece7b72e4e069fc3c7493fbf25732881758917df2e8f9a51fc2c75f", "firstIndex": 469761225},
{"blockNumber": 3938346, "blockId": "0xc638eb890c0e65ce1169ef0bb6b74dede2f4f6144ff8a5739c4894ff41a22fa2", "firstIndex": 536865235},
{"blockNumber": 3984892, "blockId": "0x6a7782f0765b0f0c406b3cb107fd8bb14c6906293d9bdc5cc00e081bfd1ae59f", "firstIndex": 603972697},
{"blockNumber": 4002660, "blockId": "0xda473de053602bc42fcbb074a32c908303f7f6db2ed8c38192665dce80276702", "firstIndex": 671087183},
{"blockNumber": 4113149, "blockId": "0xd1782f677262226194725028858da83e672748e409fe1c21bdd77c07be0662c8", "firstIndex": 738197365},
{"blockNumber": 4260733, "blockId": "0xc356600819e49d311963312157c481fd9be43a36ad040a0512fd17559c78d394", "firstIndex": 805305957},
{"blockNumber": 4391073, "blockId": "0x06f903dc765b4b0ca1a40847eb74d30b0b51904a70c46f2e836dc5b3ea3dd8d3", "firstIndex": 872415067},
{"blockNumber": 4515567, "blockId": "0xe99d3849755a1b4ec9a65d003260c111d4886e29ad6243fc750c72dbc79dbe4c", "firstIndex": 939523746},
{"blockNumber": 4634815, "blockId": "0x9f281e749b192ef58242ff9fccd496a9075c30bd28c61ed4ce6487fc2d167e5c", "firstIndex": 1006629595},
{"blockNumber": 4718286, "blockId": "0x23d2a40ec3ae059a45006ba651a70f308b6b282829b9cfdf2534e4311e34050f", "firstIndex": 1073731435},
{"blockNumber": 4753427, "blockId": "0x6df7295f9ccb45a95e6cc72a8ee56bda992513b3bf4baa3d62219871a0b5a912", "firstIndex": 1140843105},
{"blockNumber": 4786511, "blockId": "0x705c79435411c1af81385a1eca34343427eceb3f6af72e7f665552d6b0eceb6c", "firstIndex": 1207957185},
{"blockNumber": 4811696, "blockId": "0x69a6e60d1b958b469feafb1dca97f38f97620a90c21ed92f90080f6443ee6c78", "firstIndex": 1275061401},
{"blockNumber": 4841770, "blockId": "0x0a0e9d2d3d177601a478428297b5b9a42124166fefe42f2221c7a612c31e7ca1", "firstIndex": 1342174442},
{"blockNumber": 4914785, "blockId": "0x5ca0e12230aeacbce9064dff15dafb9f24e20b4a737f7d04df8549008b829fa3", "firstIndex": 1409284680},
{"blockNumber": 4992516, "blockId": "0xb9cb1700d8b65615fc3a019b2e9b2ece3b4f1e0e2fcfc01ff40bcf53e7291b10", "firstIndex": 1476387066},
{"blockNumber": 5088617, "blockId": "0x33122f693a264fb2ab626232fac5e2714010fd65bdceb772109511f7c9497333", "firstIndex": 1543503849},
{"blockNumber": 5155024, "blockId": "0x187716822dd6ab9c21138ae5a8e763197a5e7ec48bf0e756c5e6fa2c9f2f24bf", "firstIndex": 1610604247},
{"blockNumber": 5204373, "blockId": "0x753b6495979ca90a1b915edad6af33c58bbabe68b45f95c7645113ec1d35d0e6", "firstIndex": 1677721200},
{"blockNumber": 5269952, "blockId": "0xcbb0409ed824631120d6951b55537143d9a1c82ca6dd6255d2eb31c57844685c", "firstIndex": 1744829248},
{"blockNumber": 5337623, "blockId": "0x0b460658511c4ae5205c038430789f9b31f3c08ba4acc06d273b7a13a149cf8b", "firstIndex": 1811939255},
{"blockNumber": 5399044, "blockId": "0x9d07e37321e2b019f4837cc4f95f103da87ecdcb77e131e9d42d8643b0446524", "firstIndex": 1879041131},
{"blockNumber": 5422692, "blockId": "0x9d9e27614635989788e32f15cd7a0abf7470dccf3336b77547baeb3423b629ca", "firstIndex": 1946154968},
{"blockNumber": 5454259, "blockId": "0x01e694443d471d3411fe0b52720fd4a189e687b14cc8ede116f5e93f0ad96bb6", "firstIndex": 2013265148},
{"blockNumber": 5498863, "blockId": "0xc4cb906c96d9f80b830b8add25318c38b3ff857c1bd577cd9cae0a324a068ffc", "firstIndex": 2080373677},
{"blockNumber": 5554788, "blockId": "0x9b934b2d66723559d82e0f4cd49a2497234fe892248cb2f46c13f6a9585cfddd", "firstIndex": 2147475372},
{"blockNumber": 5594711, "blockId": "0x840c0b7889b8c882cfa0eceb516dd73043b51c0e5a0c1be849a0e2a5dda7e842", "firstIndex": 2214592267},
{"blockNumber": 5645179, "blockId": "0x188873a43506fee33349bc33df4a55059401e98dc7f0a5d488a0bc5c87dfd431", "firstIndex": 2281695869},
{"blockNumber": 5687634, "blockId": "0x7ea8d7a2afccf795a2a02eb00746d89db57f13f8936cf7ab0f0a7db1b1ceb341", "firstIndex": 2348806824},
{"blockNumber": 5727808, "blockId": "0x3f3eaf36c8e0271403737676907eac15312927c1d60ce814337654b7d7f2b2bd", "firstIndex": 2415915517},
{"blockNumber": 5784480, "blockId": "0x10ec5adf01449c29550f5117f09ecafc49fc614d7da5886cf01619950827f850", "firstIndex": 2483023750},
{"blockNumber": 5843906, "blockId": "0x9e6e14b3fba395a9ae7b3f1295f33a12d0f8b9f10e1aaab1649a90a56a323676", "firstIndex": 2550136398},
{"blockNumber": 5906280, "blockId": "0x2188869a164e9c19232f6702a5233dc27a2304839159dd7230cc93ebdfc6c048", "firstIndex": 2617245012},
{"blockNumber": 5977929, "blockId": "0xb247181ed597d9473c2c6e5762ae45a05f328d8df215fdac83d19ae4c028109a", "firstIndex": 2684347033},
{"blockNumber": 6051479, "blockId": "0x0e8634c95dac31b7c835b44d0a14290e9212b31b9c3f621997ad1cca2d8ab0ba", "firstIndex": 2751463372},
{"blockNumber": 6118422, "blockId": "0xa642c4457d4b63d68a5502bdba2355616e34495bf3ab2d8518a45ded54a16be4", "firstIndex": 2818563217},
{"blockNumber": 6174274, "blockId": "0xbb9808bc56b5f14636e654bdd6d50d35b31345255926341ef3ddb54840095bcc", "firstIndex": 2885680274},
{"blockNumber": 6276207, "blockId": "0x7ae027a18889e1b60a46ea3e5f2759035f3d9041de68eca0b154068ec533db59", "firstIndex": 2952789656},
{"blockNumber": 6368344, "blockId": "0x8b4401f89589d9e99259e6c05c18b1780e96b65027caf9d83edf80390d317f66", "firstIndex": 3019898378},
{"blockNumber": 6470718, "blockId": "0x0e2f8de093392c38750581c41e93c2fb1c0f6676b65c9af52792f3c56e894457", "firstIndex": 3087007589},
{"blockNumber": 6553237, "blockId": "0xa6c4f3a1fe3b4e116c367fcbb1ae98e0f25570fdb2f1eadee2681deb39462c1e", "firstIndex": 3154116356},
{"blockNumber": 6663750, "blockId": "0x9049748ddf655c7cc2ccbe2d36010b3a67ae011d114d32b1c812a7a53102f27f", "firstIndex": 3221225375},
{"blockNumber": 6766894, "blockId": "0xc98188fb313b1beb7fc4f415b4a74c363903b3b5e0172437d79446bc011238b5", "firstIndex": 3288334172},
{"blockNumber": 6886576, "blockId": "0x4ae8237a94198c82c266f01e472fdcd3ae150c84ecf714205a9be672f1e705b8", "firstIndex": 3355443040},
{"blockNumber": 6978746, "blockId": "0xb2b69ce29c5c9c46c5cbc6e96260c34313d8b0b4a739cf4091c98f37093aa732", "firstIndex": 3422552017},
{"blockNumber": 7098871, "blockId": "0x9ab2b681feeb38287a1b689fa64a50efd3407b6331fd6896187cad98a29ce107", "firstIndex": 3489660879},
{"blockNumber": 7203023, "blockId": "0xdedfff5985c1c93d476120d2afbaa61967bae4edabb9711c7794a1b1cd453aa1", "firstIndex": 3556769365},
{"blockNumber": 7256709, "blockId": "0xc078041cffc1802b6342fd2ba5a92ff8076ac0f5b72b4a77ba75cd486fe1b3c9", "firstIndex": 3623876563},
{"blockNumber": 7307744, "blockId": "0xdf8dcdd4af9213a79587eb2f4d91f4111da74f99207ee05e4ae50bcb4f8d5435", "firstIndex": 3690986225},
{"blockNumber": 7369268, "blockId": "0x62315823621a38a1f138e193b85259988fcc15f4a74d0cf19a412920bcdcad98", "firstIndex": 3758096030},
{"blockNumber": 7445109, "blockId": "0x8b668ac1274a8dcc9526eb115c2f67bd56f6657b5072c7542da2252daca10fef", "firstIndex": 3825204921},
{"blockNumber": 7511512, "blockId": "0x056ad7411652aa64849c67705a86e42221cb88f7fb6a7012ab7d3d5b9a30eb76", "firstIndex": 3892313524},
{"blockNumber": 7557259, "blockId": "0xef660b97661f073b7c7319b69451e16d26ecaed54f9237e209999d137da2bfd4", "firstIndex": 3959415096},
{"blockNumber": 7606260, "blockId": "0x9f5f06ad381166261780864bf17ad4d3cdd475f87ba51fa7f9ba621bf0ac0acb", "firstIndex": 4026521703},
{"blockNumber": 7654244, "blockId": "0xefe5b537435001203741bf445ca26466baf7ad3f63c2bba94f0f658aed8c244b", "firstIndex": 4093627258},
{"blockNumber": 7700433, "blockId": "0x89af6e8c5aa934a9139944b3de7a15232069479dba387e0fdd043ad36156c8bf", "firstIndex": 4160749200},
{"blockNumber": 7730079, "blockId": "0x08e0d511a93e2a9c004b3456d77d687ae86247417ada1cdd1f85332a62dfed1d", "firstIndex": 4227846795},
{"blockNumber": 7777515, "blockId": "0xeba8faed2e12bb9ed5e7fad19dd82662f15f6f4f512a0d232eedf1e676712428", "firstIndex": 4294966082},
{"blockNumber": 7828860, "blockId": "0x62c054bd24be351dc4777460ee922a75fdcea5c9830455cbac6fa29552719c85", "firstIndex": 4362076087},
{"blockNumber": 7869890, "blockId": "0x5664bcfa6151f798781e22bea4f9a2c796d097402350cb131e4e3c78e13e461a", "firstIndex": 4429183267},
{"blockNumber": 7911722, "blockId": "0x9a85e48e3135c97c51fc1786f2af0596c802e021b6c53cfca65a129cafcd23ed", "firstIndex": 4496287265},
{"blockNumber": 7960147, "blockId": "0xc9359cc76d7090e1c8a031108f0ab7a8935d971efd4325fe53612a1d99562f6f", "firstIndex": 4563402388},
{"blockNumber": 8030418, "blockId": "0x21867e68cd8327aed2da2601399d60f7f9e41dca4a4f2f9be982e5a2b9304a88", "firstIndex": 4630511616},
{"blockNumber": 8087701, "blockId": "0x0fa8c8d7549cc9a8d308262706fe248efe759f8b63511efb1e7f3926e9af2dcb", "firstIndex": 4697614758},
{"blockNumber": 8149130, "blockId": "0x655ea638fd9e35cc25f4332f260d7bf98f4f6fa9a72e1bff861209f18659e94c", "firstIndex": 4764727744},
{"blockNumber": 8208672, "blockId": "0xb5847a670dc3b6181f9e2e40e4218548048366d237a0d12e938b9879bc8cf800", "firstIndex": 4831837882},
{"blockNumber": 8271345, "blockId": "0x96797214946f29093883b877ccb0f2a9f771a9a3db3794a642b5dcb781c4d194", "firstIndex": 4898942160},
{"blockNumber": 8302858, "blockId": "0x6a5977b3382ca69a9e0412333f97b911c1f69f857d8f31dd0fc930980e24f2fc", "firstIndex": 4966054626},
{"blockNumber": 8333618, "blockId": "0x2547294aa23b67c42adbdddfcf424b17a95c4ff0f352a6a2442c529cfb0c892a", "firstIndex": 5033163605},
{"blockNumber": 8360582, "blockId": "0xf34f5dceb0ef22e0f782b56c12790472acc675997b9c45075bd4e18a9dacd03c", "firstIndex": 5100273631},
{"blockNumber": 8387230, "blockId": "0x0fbea42e87620b5beeb76b67febc173847c54333d7dce9fa2f8f2a3fa9c8c22a", "firstIndex": 5167381673},
{"blockNumber": 8414795, "blockId": "0x6c9c000cf5e35da3a7e9e1cd56147c8ce9b43a76d6de945675efd9dc03b628c9", "firstIndex": 5234477010},
{"blockNumber": 8444749, "blockId": "0xba85f8c9abaddc34e2113eb49385667ba4b008168ae701f46aa7a7ce78c633a1", "firstIndex": 5301598562},
{"blockNumber": 8474551, "blockId": "0x720866a40242f087dd25b6f0dd79224884f435b114a39e60c5669f5c942c78c1", "firstIndex": 5368707262},
{"blockNumber": 8501310, "blockId": "0x2b6da233532c701202fb5ac67e005f7d3eb71f88a9fac10c25d24dd11ada05e5", "firstIndex": 5435803858},
{"blockNumber": 8526970, "blockId": "0x005f9bbad0a10234129d09894d7fcf04bf1398d326510eedb4195808c282802d", "firstIndex": 5502926509},
{"blockNumber": 8550412, "blockId": "0x37c9f3efc9f33cf62f590087c8c9ac70011883f75e648647a6fd0fec00ca627c", "firstIndex": 5570034950},
{"blockNumber": 8573540, "blockId": "0x81cfb46a07be7c70bb8a0f76b03a4cd502f92032bea68ad7ba10e26351673000", "firstIndex": 5637137662},
{"blockNumber": 8590416, "blockId": "0x5c223d58ef22d7b0dd8c498e8498da4787b5dc706681c2bc83849441f5d0922d", "firstIndex": 5704252906},
{"blockNumber": 8616793, "blockId": "0x9043ce02742fb5ec43a696602867b7ce6003a95b36cd28a37eeb9785a46ad49f", "firstIndex": 5771357264},
{"blockNumber": 8647290, "blockId": "0xd90115193764b0a33f3f2a719381b3ddbce2532607c72fb287a864eb391eeada", "firstIndex": 5838466144},
{"blockNumber": 8673192, "blockId": "0x9bc92d340cccaf4c8c03372efc24eb92c5159106729de8d2e9e064f5568d082b", "firstIndex": 5905577457},
{"blockNumber": 8700694, "blockId": "0xb3d656a173b962bc6825198e94a4974289db06a8998060bd0f5ee2044a7a7deb", "firstIndex": 5972679345},
{"blockNumber": 8724533, "blockId": "0x253ffc6d77b88fe18736e4c313e9930341c444bc87b2ee22b26cfe8d9d0b178d", "firstIndex": 6039795829},
{"blockNumber": 8743948, "blockId": "0x04eb66d0261705d31e629193148d0685058d7759ba5f95d2d38e412dbadb8256", "firstIndex": 6106901747},
{"blockNumber": 8758378, "blockId": "0x64adf54e662d11db716610157da672c3d8b45f001dbce40a269871b86a84d026", "firstIndex": 6174011544},
{"blockNumber": 8777722, "blockId": "0x0a7f9a956024b404c915e70b42221aa027b2dd715b0697f099dccefae0b9af97", "firstIndex": 6241124215},
{"blockNumber": 8800154, "blockId": "0x411f90dc18f2bca31fa63615c2866c907bbac1fae8c06782cabfaf788efba665", "firstIndex": 6308233216},
{"blockNumber": 8829725, "blockId": "0x5686f3a5eec1b070d0113c588f8f4a560d57ad96b8045cedb5c08bbadaa0273e", "firstIndex": 6375340033},
{"blockNumber": 8858036, "blockId": "0x4f9b5d9fac9c6f6e2224f613cda12e8ab95d636774ce87489dce8a9f805ee2e5", "firstIndex": 6442450330},
{"blockNumber": 8884811, "blockId": "0x9cf74f978872683802c065e72b5a5326fdad95f19733c34d927b575cd85fd0bd", "firstIndex": 6509559380}
{"blockNumber": 3246675, "blockHash": "0x36bf7de9e1f151963088ca3efa206b6e78411d699d2f64f3bf86895294275e0b", "firstIndex": 67108765},
{"blockNumber": 3575560, "blockHash": "0x4652e3bee440f946aeaa3358c9a62b9bc4d41f663737ab00fded00c2662bfa29", "firstIndex": 134217644},
{"blockNumber": 3694262, "blockHash": "0x433573f97e563ca04fa53e0d0eb019d0ffe9dd032a05ee5b33ce7705aedfd34d", "firstIndex": 201316990},
{"blockNumber": 3725628, "blockHash": "0x1e25ebd76b9e2a2007bfdd7c82433cce7dae8beb190d249c82bf83d0cd177735", "firstIndex": 268426577},
{"blockNumber": 3795378, "blockHash": "0xca999083e01d5947212fb878d529abecdd18a6012997b0d5326afcf6908f17e5", "firstIndex": 335543438},
{"blockNumber": 3856681, "blockHash": "0x48148cd35de3de6c626f76ce5d97cfc6acd88772af69c0aef3d640c29ba82096", "firstIndex": 402641717},
{"blockNumber": 3869367, "blockHash": "0x08bafccf9ece7b72e4e069fc3c7493fbf25732881758917df2e8f9a51fc2c75f", "firstIndex": 469761225},
{"blockNumber": 3938346, "blockHash": "0xc638eb890c0e65ce1169ef0bb6b74dede2f4f6144ff8a5739c4894ff41a22fa2", "firstIndex": 536865235},
{"blockNumber": 3984892, "blockHash": "0x6a7782f0765b0f0c406b3cb107fd8bb14c6906293d9bdc5cc00e081bfd1ae59f", "firstIndex": 603972697},
{"blockNumber": 4002660, "blockHash": "0xda473de053602bc42fcbb074a32c908303f7f6db2ed8c38192665dce80276702", "firstIndex": 671087183},
{"blockNumber": 4113149, "blockHash": "0xd1782f677262226194725028858da83e672748e409fe1c21bdd77c07be0662c8", "firstIndex": 738197365},
{"blockNumber": 4260733, "blockHash": "0xc356600819e49d311963312157c481fd9be43a36ad040a0512fd17559c78d394", "firstIndex": 805305957},
{"blockNumber": 4391073, "blockHash": "0x06f903dc765b4b0ca1a40847eb74d30b0b51904a70c46f2e836dc5b3ea3dd8d3", "firstIndex": 872415067},
{"blockNumber": 4515567, "blockHash": "0xe99d3849755a1b4ec9a65d003260c111d4886e29ad6243fc750c72dbc79dbe4c", "firstIndex": 939523746},
{"blockNumber": 4634815, "blockHash": "0x9f281e749b192ef58242ff9fccd496a9075c30bd28c61ed4ce6487fc2d167e5c", "firstIndex": 1006629595},
{"blockNumber": 4718286, "blockHash": "0x23d2a40ec3ae059a45006ba651a70f308b6b282829b9cfdf2534e4311e34050f", "firstIndex": 1073731435},
{"blockNumber": 4753427, "blockHash": "0x6df7295f9ccb45a95e6cc72a8ee56bda992513b3bf4baa3d62219871a0b5a912", "firstIndex": 1140843105},
{"blockNumber": 4786511, "blockHash": "0x705c79435411c1af81385a1eca34343427eceb3f6af72e7f665552d6b0eceb6c", "firstIndex": 1207957185},
{"blockNumber": 4811696, "blockHash": "0x69a6e60d1b958b469feafb1dca97f38f97620a90c21ed92f90080f6443ee6c78", "firstIndex": 1275061401},
{"blockNumber": 4841770, "blockHash": "0x0a0e9d2d3d177601a478428297b5b9a42124166fefe42f2221c7a612c31e7ca1", "firstIndex": 1342174442},
{"blockNumber": 4914785, "blockHash": "0x5ca0e12230aeacbce9064dff15dafb9f24e20b4a737f7d04df8549008b829fa3", "firstIndex": 1409284680},
{"blockNumber": 4992516, "blockHash": "0xb9cb1700d8b65615fc3a019b2e9b2ece3b4f1e0e2fcfc01ff40bcf53e7291b10", "firstIndex": 1476387066},
{"blockNumber": 5088617, "blockHash": "0x33122f693a264fb2ab626232fac5e2714010fd65bdceb772109511f7c9497333", "firstIndex": 1543503849},
{"blockNumber": 5155024, "blockHash": "0x187716822dd6ab9c21138ae5a8e763197a5e7ec48bf0e756c5e6fa2c9f2f24bf", "firstIndex": 1610604247},
{"blockNumber": 5204373, "blockHash": "0x753b6495979ca90a1b915edad6af33c58bbabe68b45f95c7645113ec1d35d0e6", "firstIndex": 1677721200},
{"blockNumber": 5269952, "blockHash": "0xcbb0409ed824631120d6951b55537143d9a1c82ca6dd6255d2eb31c57844685c", "firstIndex": 1744829248},
{"blockNumber": 5337623, "blockHash": "0x0b460658511c4ae5205c038430789f9b31f3c08ba4acc06d273b7a13a149cf8b", "firstIndex": 1811939255},
{"blockNumber": 5399044, "blockHash": "0x9d07e37321e2b019f4837cc4f95f103da87ecdcb77e131e9d42d8643b0446524", "firstIndex": 1879041131},
{"blockNumber": 5422692, "blockHash": "0x9d9e27614635989788e32f15cd7a0abf7470dccf3336b77547baeb3423b629ca", "firstIndex": 1946154968},
{"blockNumber": 5454259, "blockHash": "0x01e694443d471d3411fe0b52720fd4a189e687b14cc8ede116f5e93f0ad96bb6", "firstIndex": 2013265148},
{"blockNumber": 5498863, "blockHash": "0xc4cb906c96d9f80b830b8add25318c38b3ff857c1bd577cd9cae0a324a068ffc", "firstIndex": 2080373677},
{"blockNumber": 5554788, "blockHash": "0x9b934b2d66723559d82e0f4cd49a2497234fe892248cb2f46c13f6a9585cfddd", "firstIndex": 2147475372},
{"blockNumber": 5594711, "blockHash": "0x840c0b7889b8c882cfa0eceb516dd73043b51c0e5a0c1be849a0e2a5dda7e842", "firstIndex": 2214592267},
{"blockNumber": 5645179, "blockHash": "0x188873a43506fee33349bc33df4a55059401e98dc7f0a5d488a0bc5c87dfd431", "firstIndex": 2281695869},
{"blockNumber": 5687634, "blockHash": "0x7ea8d7a2afccf795a2a02eb00746d89db57f13f8936cf7ab0f0a7db1b1ceb341", "firstIndex": 2348806824},
{"blockNumber": 5727808, "blockHash": "0x3f3eaf36c8e0271403737676907eac15312927c1d60ce814337654b7d7f2b2bd", "firstIndex": 2415915517},
{"blockNumber": 5784480, "blockHash": "0x10ec5adf01449c29550f5117f09ecafc49fc614d7da5886cf01619950827f850", "firstIndex": 2483023750},
{"blockNumber": 5843906, "blockHash": "0x9e6e14b3fba395a9ae7b3f1295f33a12d0f8b9f10e1aaab1649a90a56a323676", "firstIndex": 2550136398},
{"blockNumber": 5906280, "blockHash": "0x2188869a164e9c19232f6702a5233dc27a2304839159dd7230cc93ebdfc6c048", "firstIndex": 2617245012},
{"blockNumber": 5977929, "blockHash": "0xb247181ed597d9473c2c6e5762ae45a05f328d8df215fdac83d19ae4c028109a", "firstIndex": 2684347033},
{"blockNumber": 6051479, "blockHash": "0x0e8634c95dac31b7c835b44d0a14290e9212b31b9c3f621997ad1cca2d8ab0ba", "firstIndex": 2751463372},
{"blockNumber": 6118422, "blockHash": "0xa642c4457d4b63d68a5502bdba2355616e34495bf3ab2d8518a45ded54a16be4", "firstIndex": 2818563217},
{"blockNumber": 6174274, "blockHash": "0xbb9808bc56b5f14636e654bdd6d50d35b31345255926341ef3ddb54840095bcc", "firstIndex": 2885680274},
{"blockNumber": 6276207, "blockHash": "0x7ae027a18889e1b60a46ea3e5f2759035f3d9041de68eca0b154068ec533db59", "firstIndex": 2952789656},
{"blockNumber": 6368344, "blockHash": "0x8b4401f89589d9e99259e6c05c18b1780e96b65027caf9d83edf80390d317f66", "firstIndex": 3019898378},
{"blockNumber": 6470718, "blockHash": "0x0e2f8de093392c38750581c41e93c2fb1c0f6676b65c9af52792f3c56e894457", "firstIndex": 3087007589},
{"blockNumber": 6553237, "blockHash": "0xa6c4f3a1fe3b4e116c367fcbb1ae98e0f25570fdb2f1eadee2681deb39462c1e", "firstIndex": 3154116356},
{"blockNumber": 6663750, "blockHash": "0x9049748ddf655c7cc2ccbe2d36010b3a67ae011d114d32b1c812a7a53102f27f", "firstIndex": 3221225375},
{"blockNumber": 6766894, "blockHash": "0xc98188fb313b1beb7fc4f415b4a74c363903b3b5e0172437d79446bc011238b5", "firstIndex": 3288334172},
{"blockNumber": 6886576, "blockHash": "0x4ae8237a94198c82c266f01e472fdcd3ae150c84ecf714205a9be672f1e705b8", "firstIndex": 3355443040},
{"blockNumber": 6978746, "blockHash": "0xb2b69ce29c5c9c46c5cbc6e96260c34313d8b0b4a739cf4091c98f37093aa732", "firstIndex": 3422552017},
{"blockNumber": 7098871, "blockHash": "0x9ab2b681feeb38287a1b689fa64a50efd3407b6331fd6896187cad98a29ce107", "firstIndex": 3489660879},
{"blockNumber": 7203023, "blockHash": "0xdedfff5985c1c93d476120d2afbaa61967bae4edabb9711c7794a1b1cd453aa1", "firstIndex": 3556769365},
{"blockNumber": 7256709, "blockHash": "0xc078041cffc1802b6342fd2ba5a92ff8076ac0f5b72b4a77ba75cd486fe1b3c9", "firstIndex": 3623876563},
{"blockNumber": 7307744, "blockHash": "0xdf8dcdd4af9213a79587eb2f4d91f4111da74f99207ee05e4ae50bcb4f8d5435", "firstIndex": 3690986225},
{"blockNumber": 7369268, "blockHash": "0x62315823621a38a1f138e193b85259988fcc15f4a74d0cf19a412920bcdcad98", "firstIndex": 3758096030},
{"blockNumber": 7445109, "blockHash": "0x8b668ac1274a8dcc9526eb115c2f67bd56f6657b5072c7542da2252daca10fef", "firstIndex": 3825204921},
{"blockNumber": 7511512, "blockHash": "0x056ad7411652aa64849c67705a86e42221cb88f7fb6a7012ab7d3d5b9a30eb76", "firstIndex": 3892313524},
{"blockNumber": 7557259, "blockHash": "0xef660b97661f073b7c7319b69451e16d26ecaed54f9237e209999d137da2bfd4", "firstIndex": 3959415096},
{"blockNumber": 7606260, "blockHash": "0x9f5f06ad381166261780864bf17ad4d3cdd475f87ba51fa7f9ba621bf0ac0acb", "firstIndex": 4026521703},
{"blockNumber": 7654244, "blockHash": "0xefe5b537435001203741bf445ca26466baf7ad3f63c2bba94f0f658aed8c244b", "firstIndex": 4093627258},
{"blockNumber": 7700433, "blockHash": "0x89af6e8c5aa934a9139944b3de7a15232069479dba387e0fdd043ad36156c8bf", "firstIndex": 4160749200},
{"blockNumber": 7730079, "blockHash": "0x08e0d511a93e2a9c004b3456d77d687ae86247417ada1cdd1f85332a62dfed1d", "firstIndex": 4227846795},
{"blockNumber": 7777515, "blockHash": "0xeba8faed2e12bb9ed5e7fad19dd82662f15f6f4f512a0d232eedf1e676712428", "firstIndex": 4294966082},
{"blockNumber": 7828860, "blockHash": "0x62c054bd24be351dc4777460ee922a75fdcea5c9830455cbac6fa29552719c85", "firstIndex": 4362076087},
{"blockNumber": 7869890, "blockHash": "0x5664bcfa6151f798781e22bea4f9a2c796d097402350cb131e4e3c78e13e461a", "firstIndex": 4429183267},
{"blockNumber": 7911722, "blockHash": "0x9a85e48e3135c97c51fc1786f2af0596c802e021b6c53cfca65a129cafcd23ed", "firstIndex": 4496287265},
{"blockNumber": 7960147, "blockHash": "0xc9359cc76d7090e1c8a031108f0ab7a8935d971efd4325fe53612a1d99562f6f", "firstIndex": 4563402388},
{"blockNumber": 8030418, "blockHash": "0x21867e68cd8327aed2da2601399d60f7f9e41dca4a4f2f9be982e5a2b9304a88", "firstIndex": 4630511616},
{"blockNumber": 8087701, "blockHash": "0x0fa8c8d7549cc9a8d308262706fe248efe759f8b63511efb1e7f3926e9af2dcb", "firstIndex": 4697614758},
{"blockNumber": 8149130, "blockHash": "0x655ea638fd9e35cc25f4332f260d7bf98f4f6fa9a72e1bff861209f18659e94c", "firstIndex": 4764727744},
{"blockNumber": 8208672, "blockHash": "0xb5847a670dc3b6181f9e2e40e4218548048366d237a0d12e938b9879bc8cf800", "firstIndex": 4831837882},
{"blockNumber": 8271345, "blockHash": "0x96797214946f29093883b877ccb0f2a9f771a9a3db3794a642b5dcb781c4d194", "firstIndex": 4898942160},
{"blockNumber": 8302858, "blockHash": "0x6a5977b3382ca69a9e0412333f97b911c1f69f857d8f31dd0fc930980e24f2fc", "firstIndex": 4966054626},
{"blockNumber": 8333618, "blockHash": "0x2547294aa23b67c42adbdddfcf424b17a95c4ff0f352a6a2442c529cfb0c892a", "firstIndex": 5033163605},
{"blockNumber": 8360582, "blockHash": "0xf34f5dceb0ef22e0f782b56c12790472acc675997b9c45075bd4e18a9dacd03c", "firstIndex": 5100273631},
{"blockNumber": 8387230, "blockHash": "0x0fbea42e87620b5beeb76b67febc173847c54333d7dce9fa2f8f2a3fa9c8c22a", "firstIndex": 5167381673},
{"blockNumber": 8414795, "blockHash": "0x6c9c000cf5e35da3a7e9e1cd56147c8ce9b43a76d6de945675efd9dc03b628c9", "firstIndex": 5234477010},
{"blockNumber": 8444749, "blockHash": "0xba85f8c9abaddc34e2113eb49385667ba4b008168ae701f46aa7a7ce78c633a1", "firstIndex": 5301598562},
{"blockNumber": 8474551, "blockHash": "0x720866a40242f087dd25b6f0dd79224884f435b114a39e60c5669f5c942c78c1", "firstIndex": 5368707262},
{"blockNumber": 8501310, "blockHash": "0x2b6da233532c701202fb5ac67e005f7d3eb71f88a9fac10c25d24dd11ada05e5", "firstIndex": 5435803858},
{"blockNumber": 8526970, "blockHash": "0x005f9bbad0a10234129d09894d7fcf04bf1398d326510eedb4195808c282802d", "firstIndex": 5502926509},
{"blockNumber": 8550412, "blockHash": "0x37c9f3efc9f33cf62f590087c8c9ac70011883f75e648647a6fd0fec00ca627c", "firstIndex": 5570034950},
{"blockNumber": 8573540, "blockHash": "0x81cfb46a07be7c70bb8a0f76b03a4cd502f92032bea68ad7ba10e26351673000", "firstIndex": 5637137662},
{"blockNumber": 8590416, "blockHash": "0x5c223d58ef22d7b0dd8c498e8498da4787b5dc706681c2bc83849441f5d0922d", "firstIndex": 5704252906},
{"blockNumber": 8616793, "blockHash": "0x9043ce02742fb5ec43a696602867b7ce6003a95b36cd28a37eeb9785a46ad49f", "firstIndex": 5771357264},
{"blockNumber": 8647290, "blockHash": "0xd90115193764b0a33f3f2a719381b3ddbce2532607c72fb287a864eb391eeada", "firstIndex": 5838466144},
{"blockNumber": 8673192, "blockHash": "0x9bc92d340cccaf4c8c03372efc24eb92c5159106729de8d2e9e064f5568d082b", "firstIndex": 5905577457},
{"blockNumber": 8700694, "blockHash": "0xb3d656a173b962bc6825198e94a4974289db06a8998060bd0f5ee2044a7a7deb", "firstIndex": 5972679345},
{"blockNumber": 8724533, "blockHash": "0x253ffc6d77b88fe18736e4c313e9930341c444bc87b2ee22b26cfe8d9d0b178d", "firstIndex": 6039795829},
{"blockNumber": 8743948, "blockHash": "0x04eb66d0261705d31e629193148d0685058d7759ba5f95d2d38e412dbadb8256", "firstIndex": 6106901747},
{"blockNumber": 8758378, "blockHash": "0x64adf54e662d11db716610157da672c3d8b45f001dbce40a269871b86a84d026", "firstIndex": 6174011544},
{"blockNumber": 8777722, "blockHash": "0x0a7f9a956024b404c915e70b42221aa027b2dd715b0697f099dccefae0b9af97", "firstIndex": 6241124215},
{"blockNumber": 8800154, "blockHash": "0x411f90dc18f2bca31fa63615c2866c907bbac1fae8c06782cabfaf788efba665", "firstIndex": 6308233216},
{"blockNumber": 8829725, "blockHash": "0x5686f3a5eec1b070d0113c588f8f4a560d57ad96b8045cedb5c08bbadaa0273e", "firstIndex": 6375340033},
{"blockNumber": 8858036, "blockHash": "0x4f9b5d9fac9c6f6e2224f613cda12e8ab95d636774ce87489dce8a9f805ee2e5", "firstIndex": 6442450330},
{"blockNumber": 8884811, "blockHash": "0x9cf74f978872683802c065e72b5a5326fdad95f19733c34d927b575cd85fd0bd", "firstIndex": 6509559380}
]

889
core/filtermaps/filtermaps.go
View file

@ -1,889 +0,0 @@
// Copyright 2024 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package filtermaps
import (
"errors"
"fmt"
"os"
"slices"
"sync"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/lru"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
)
var (
mapCountGauge = metrics.NewRegisteredGauge("filtermaps/maps/count", nil) // actual number of rendered maps
mapLogValueMeter = metrics.NewRegisteredMeter("filtermaps/maps/logvalues", nil) // number of log values processed
mapBlockMeter = metrics.NewRegisteredMeter("filtermaps/maps/blocks", nil) // number of block delimiters processed
mapRenderTimer = metrics.NewRegisteredTimer("filtermaps/maps/rendertime", nil) // time elapsed while rendering a single map
mapWriteTimer = metrics.NewRegisteredTimer("filtermaps/maps/writetime", nil) // time elapsed while writing a batch of finished maps to db
matchRequestTimer = metrics.NewRegisteredTimer("filtermaps/match/requesttime", nil) // processing time a matching request in a single epoch
matchEpochTimer = metrics.NewRegisteredTimer("filtermaps/match/epochtime", nil) // total processing time a matching request
matchBaseRowAccessMeter = metrics.NewRegisteredMeter("filtermaps/match/baserowaccess", nil) // number of accessed rows on layer 0
matchBaseRowSizeMeter = metrics.NewRegisteredMeter("filtermaps/match/baserowsize", nil) // size of accessed rows on layer 0
matchExtRowAccessMeter = metrics.NewRegisteredMeter("filtermaps/match/extrowaccess", nil) // number of accessed rows on higher layers
matchExtRowSizeMeter = metrics.NewRegisteredMeter("filtermaps/match/extrowsize", nil) // size of accessed rows on higher layers
matchLogLookup = metrics.NewRegisteredMeter("filtermaps/match/loglookup", nil) // number of log lookups based on potential matches
matchAllMeter = metrics.NewRegisteredMeter("filtermaps/match/matchall", nil) // number of requests returned with ErrMatchAll
)
const (
databaseVersion = 2 // reindexed if database version does not match
cachedLastBlocks = 1000 // last block of map pointers
cachedLvPointers = 1000 // first log value pointer of block pointers
cachedFilterMaps = 3 // complete filter maps (cached by map renderer)
cachedRenderSnapshots = 8 // saved map renderer data at block boundaries
)
// FilterMaps is the in-memory representation of the log index structure that is
// responsible for building and updating the index according to the canonical
// chain.
//
// Note that FilterMaps implements the same data structure as proposed in EIP-7745
// without the tree hashing and consensus changes:
// https://eips.ethereum.org/EIPS/eip-7745
type FilterMaps struct {
// If disabled is set, log indexing is fully disabled.
// This is configured by the --history.logs.disable Geth flag.
// We chose to implement disabling this way because it requires less special
// case logic in eth/filters.
disabled bool
disabledCh chan struct{} // closed by indexer if disabled
closeCh chan struct{}
closeWg sync.WaitGroup
history uint64
hashScheme bool // use hashdb-safe delete range method
exportFileName string
Params
db ethdb.KeyValueStore
// fields written by the indexer and read by matcher backend. Indexer can
// read them without a lock and write them under indexLock write lock.
// Matcher backend can read them under indexLock read lock.
indexLock sync.RWMutex
indexedRange filterMapsRange
indexedView *ChainView // always consistent with the log index
hasTempRange bool
// cleanedEpochsBefore indicates that all unindexed data before this point
// has been cleaned.
//
// This field is only accessed and modified within tryUnindexTail, so no
// explicit locking is required.
cleanedEpochsBefore uint32
// also accessed by indexer and matcher backend but no locking needed.
filterMapCache *lru.Cache[uint32, filterMap]
lastBlockCache *lru.Cache[uint32, lastBlockOfMap]
lvPointerCache *lru.Cache[uint64, uint64]
// the matchers set and the fields of FilterMapsMatcherBackend instances are
// read and written both by exported functions and the indexer.
// Note that if both indexLock and matchersLock needs to be locked then
// indexLock should be locked first.
matchersLock sync.Mutex
matchers map[*FilterMapsMatcherBackend]struct{}
// fields only accessed by the indexer (no mutex required).
renderSnapshots *lru.Cache[uint64, *renderedMap]
startedHeadIndex, startedTailIndex, startedTailUnindex bool
startedHeadIndexAt, startedTailIndexAt, startedTailUnindexAt time.Time
loggedHeadIndex, loggedTailIndex bool
lastLogHeadIndex, lastLogTailIndex time.Time
ptrHeadIndex, ptrTailIndex, ptrTailUnindexBlock uint64
ptrTailUnindexMap uint32
targetView *ChainView
matcherSyncRequests []*FilterMapsMatcherBackend
historyCutoff uint64
finalBlock, lastFinal uint64
lastFinalEpoch uint32
stop bool
targetCh chan targetUpdate
blockProcessingCh chan bool
blockProcessing bool
matcherSyncCh chan *FilterMapsMatcherBackend
waitIdleCh chan chan bool
tailRenderer *mapRenderer
// test hooks
testDisableSnapshots, testSnapshotUsed bool
testProcessEventsHook func()
}
// filterMap is a full or partial in-memory representation of a filter map where
// rows are allowed to have a nil value meaning the row is not stored in the
// structure. Note that therefore a known empty row should be represented with
// a zero-length slice.
// It can be used as a memory cache or an overlay while preparing a batch of
// changes to the structure. In either case a nil value should be interpreted
// as transparent (uncached/unchanged).
type filterMap []FilterRow
// fastCopy returns a copy of the given filter map. Note that the row slices are
// copied but their contents are not. This permits appending to the rows further
// (which happens during map rendering) without affecting the validity of
// copies made for snapshots during rendering.
// Appending to the rows of both the original map and the fast copy, or two fast
// copies of the same map would result in data corruption, therefore a fast copy
// should always be used in a read only way.
func (fm filterMap) fastCopy() filterMap {
return slices.Clone(fm)
}
// fullCopy returns a copy of the given filter map, also making a copy of each
// individual filter row, ensuring that a modification to either one will never
// affect the other.
func (fm filterMap) fullCopy() filterMap {
c := make(filterMap, len(fm))
for i, row := range fm {
c[i] = slices.Clone(row)
}
return c
}
// FilterRow encodes a single row of a filter map as a list of column indices.
// Note that the values are always stored in the same order as they were added
// and if the same column index is added twice, it is also stored twice.
// Order of column indices and potential duplications do not matter when searching
// for a value but leaving the original order makes reverting to a previous state
// simpler.
type FilterRow []uint32
// Equal returns true if the given filter rows are equivalent.
func (a FilterRow) Equal(b FilterRow) bool {
return slices.Equal(a, b)
}
// filterMapsRange describes the rendered range of filter maps and the range
// of fully rendered blocks.
type filterMapsRange struct {
initialized bool
headIndexed bool
headDelimiter uint64 // zero if headIndexed is false
// if initialized then all maps are rendered in the maps range
maps common.Range[uint32]
// if tailPartialEpoch > 0 then maps between firstRenderedMap-mapsPerEpoch and
// firstRenderedMap-mapsPerEpoch+tailPartialEpoch-1 are rendered
tailPartialEpoch uint32
// if initialized then all log values in the blocks range are fully
// rendered
// blockLvPointers are available in the blocks range
blocks common.Range[uint64]
}
// hasIndexedBlocks returns true if the range has at least one fully indexed block.
func (fmr *filterMapsRange) hasIndexedBlocks() bool {
return fmr.initialized && !fmr.blocks.IsEmpty() && !fmr.maps.IsEmpty()
}
// lastBlockOfMap is used for caching the (number, id) pairs belonging to the
// last block of each map.
type lastBlockOfMap struct {
number uint64
id common.Hash
}
// Config contains the configuration options for NewFilterMaps.
type Config struct {
History uint64 // number of historical blocks to index
Disabled bool // disables indexing completely
// This option enables the checkpoint JSON file generator.
// If set, the given file will be updated with checkpoint information.
ExportFileName string
// expect trie nodes of hash based state scheme in the filtermaps key range;
// use safe iterator based implementation of DeleteRange that skips them
HashScheme bool
}
// NewFilterMaps creates a new FilterMaps and starts the indexer.
func NewFilterMaps(db ethdb.KeyValueStore, initView *ChainView, historyCutoff, finalBlock uint64, params Params, config Config) (*FilterMaps, error) {
rs, initialized, err := rawdb.ReadFilterMapsRange(db)
if err != nil || (initialized && rs.Version != databaseVersion) {
rs, initialized = rawdb.FilterMapsRange{}, false
log.Warn("Invalid log index database version; resetting log index")
}
if err := params.sanitize(); err != nil {
return nil, err
}
f := &FilterMaps{
db: db,
closeCh: make(chan struct{}),
waitIdleCh: make(chan chan bool),
targetCh: make(chan targetUpdate, 1),
blockProcessingCh: make(chan bool, 1),
history: config.History,
disabled: config.Disabled,
hashScheme: config.HashScheme,
disabledCh: make(chan struct{}),
exportFileName: config.ExportFileName,
Params: params,
targetView: initView,
indexedView: initView,
indexedRange: filterMapsRange{
initialized: initialized,
headIndexed: rs.HeadIndexed,
headDelimiter: rs.HeadDelimiter,
blocks: common.NewRange(rs.BlocksFirst, rs.BlocksAfterLast-rs.BlocksFirst),
maps: common.NewRange(rs.MapsFirst, rs.MapsAfterLast-rs.MapsFirst),
tailPartialEpoch: rs.TailPartialEpoch,
},
// deleting last unindexed epoch might have been interrupted by shutdown
cleanedEpochsBefore: max(rs.MapsFirst>>params.logMapsPerEpoch, 1) - 1,
historyCutoff: historyCutoff,
finalBlock: finalBlock,
matcherSyncCh: make(chan *FilterMapsMatcherBackend),
matchers: make(map[*FilterMapsMatcherBackend]struct{}),
filterMapCache: lru.NewCache[uint32, filterMap](cachedFilterMaps),
lastBlockCache: lru.NewCache[uint32, lastBlockOfMap](cachedLastBlocks),
lvPointerCache: lru.NewCache[uint64, uint64](cachedLvPointers),
renderSnapshots: lru.NewCache[uint64, *renderedMap](cachedRenderSnapshots),
}
f.checkRevertRange() // revert maps that are inconsistent with the current chain view
if f.indexedRange.hasIndexedBlocks() {
log.Info("Initialized log indexer",
"firstblock", f.indexedRange.blocks.First(), "lastblock", f.indexedRange.blocks.Last(),
"firstmap", f.indexedRange.maps.First(), "lastmap", f.indexedRange.maps.Last(),
"headindexed", f.indexedRange.headIndexed)
}
return f, nil
}
// Start starts the indexer.
func (f *FilterMaps) Start() {
if !f.testDisableSnapshots && f.indexedRange.hasIndexedBlocks() && f.indexedRange.headIndexed {
// previous target head rendered; load last map as snapshot
if err := f.loadHeadSnapshot(); err != nil {
log.Error("Could not load head filter map snapshot", "error", err)
}
}
f.closeWg.Add(2)
go f.removeBloomBits()
go f.indexerLoop()
}
// Stop ensures that the indexer is fully stopped before returning.
func (f *FilterMaps) Stop() {
close(f.closeCh)
f.closeWg.Wait()
}
// checkRevertRange checks whether the existing index is consistent with the
// current indexed view and reverts inconsistent maps if necessary.
func (f *FilterMaps) checkRevertRange() {
if f.indexedRange.maps.Count() == 0 {
return
}
lastMap := f.indexedRange.maps.Last()
lastBlockNumber, lastBlockId, err := f.getLastBlockOfMap(lastMap)
if err != nil {
log.Error("Error initializing log index database; resetting log index", "error", err)
f.reset()
return
}
for lastBlockNumber > f.indexedView.HeadNumber() || f.indexedView.BlockId(lastBlockNumber) != lastBlockId {
// revert last map
if f.indexedRange.maps.Count() == 1 {
f.reset() // reset database if no rendered maps remained
return
}
lastMap--
newRange := f.indexedRange
newRange.maps.SetLast(lastMap)
lastBlockNumber, lastBlockId, err = f.getLastBlockOfMap(lastMap)
if err != nil {
log.Error("Error initializing log index database; resetting log index", "error", err)
f.reset()
return
}
newRange.blocks.SetAfterLast(lastBlockNumber) // lastBlockNumber is probably partially indexed
newRange.headIndexed = false
newRange.headDelimiter = 0
// only shorten range and leave map data; next head render will overwrite it
f.setRange(f.db, f.indexedView, newRange, false)
}
}
// reset un-initializes the FilterMaps structure and removes all related data from
// the database.
// Note that in case of leveldb database the fallback implementation of DeleteRange
// might take a long time to finish and deleting the entire database may be
// interrupted by a shutdown. Deleting the filterMapsRange entry first does
// guarantee though that the next init() will not return successfully until the
// entire database has been cleaned.
func (f *FilterMaps) reset() {
f.indexLock.Lock()
f.indexedRange = filterMapsRange{}
f.indexedView = nil
f.filterMapCache.Purge()
f.renderSnapshots.Purge()
f.lastBlockCache.Purge()
f.lvPointerCache.Purge()
f.indexLock.Unlock()
// deleting the range first ensures that resetDb will be called again at next
// startup and any leftover data will be removed even if it cannot finish now.
rawdb.DeleteFilterMapsRange(f.db)
f.safeDeleteWithLogs(rawdb.DeleteFilterMapsDb, "Resetting log index database", f.isShuttingDown)
}
// isShuttingDown returns true if FilterMaps is shutting down.
func (f *FilterMaps) isShuttingDown() bool {
select {
case <-f.closeCh:
return true
default:
return false
}
}
// init initializes an empty log index according to the current targetView.
func (f *FilterMaps) init() error {
// ensure that there is no remaining data in the filter maps key range
if err := f.safeDeleteWithLogs(rawdb.DeleteFilterMapsDb, "Resetting log index database", f.isShuttingDown); err != nil {
return err
}
f.indexLock.Lock()
defer f.indexLock.Unlock()
var bestIdx, bestLen int
for idx, checkpointList := range checkpoints {
// binary search for the last matching epoch head
min, max := 0, len(checkpointList)
for min < max {
mid := (min + max + 1) / 2
cp := checkpointList[mid-1]
if cp.BlockNumber <= f.targetView.HeadNumber() && f.targetView.BlockId(cp.BlockNumber) == cp.BlockId {
min = mid
} else {
max = mid - 1
}
}
if max > bestLen {
bestIdx, bestLen = idx, max
}
}
var initBlockNumber uint64
if bestLen > 0 {
initBlockNumber = checkpoints[bestIdx][bestLen-1].BlockNumber
}
if initBlockNumber < f.historyCutoff {
return errors.New("cannot start indexing before history cutoff point")
}
batch := f.db.NewBatch()
for epoch := range bestLen {
cp := checkpoints[bestIdx][epoch]
f.storeLastBlockOfMap(batch, f.lastEpochMap(uint32(epoch)), cp.BlockNumber, cp.BlockId)
f.storeBlockLvPointer(batch, cp.BlockNumber, cp.FirstIndex)
}
fmr := filterMapsRange{
initialized: true,
}
if bestLen > 0 {
cp := checkpoints[bestIdx][bestLen-1]
fmr.blocks = common.NewRange(cp.BlockNumber+1, 0)
fmr.maps = common.NewRange(f.firstEpochMap(uint32(bestLen)), 0)
}
f.setRange(batch, f.targetView, fmr, false)
return batch.Write()
}
// removeBloomBits removes old bloom bits data from the database.
func (f *FilterMaps) removeBloomBits() {
f.safeDeleteWithLogs(rawdb.DeleteBloomBitsDb, "Removing old bloom bits database", f.isShuttingDown)
f.closeWg.Done()
}
// safeDeleteWithLogs is a wrapper for a function that performs a safe range
// delete operation using rawdb.SafeDeleteRange. It emits log messages if the
// process takes long enough to call the stop callback.
func (f *FilterMaps) safeDeleteWithLogs(deleteFn func(db ethdb.KeyValueStore, hashScheme bool, stopCb func(bool) bool) error, action string, stopCb func() bool) error {
var (
start = time.Now()
logPrinted bool
lastLogPrinted = start
)
switch err := deleteFn(f.db, f.hashScheme, func(deleted bool) bool {
if deleted && !logPrinted || time.Since(lastLogPrinted) > time.Second*10 {
log.Info(action+" in progress...", "elapsed", common.PrettyDuration(time.Since(start)))
logPrinted, lastLogPrinted = true, time.Now()
}
return stopCb()
}); {
case err == nil:
if logPrinted {
log.Info(action+" finished", "elapsed", common.PrettyDuration(time.Since(start)))
}
return nil
case errors.Is(err, rawdb.ErrDeleteRangeInterrupted):
log.Warn(action+" interrupted", "elapsed", common.PrettyDuration(time.Since(start)))
return err
default:
log.Error(action+" failed", "error", err)
return err
}
}
// setRange updates the indexed chain view and covered range and also adds the
// changes to the given batch.
//
// Note that this function assumes that the index write lock is being held.
func (f *FilterMaps) setRange(batch ethdb.KeyValueWriter, newView *ChainView, newRange filterMapsRange, isTempRange bool) {
f.indexedView = newView
f.indexedRange = newRange
f.hasTempRange = isTempRange
f.updateMatchersValidRange()
if newRange.initialized {
rs := rawdb.FilterMapsRange{
Version: databaseVersion,
HeadIndexed: newRange.headIndexed,
HeadDelimiter: newRange.headDelimiter,
BlocksFirst: newRange.blocks.First(),
BlocksAfterLast: newRange.blocks.AfterLast(),
MapsFirst: newRange.maps.First(),
MapsAfterLast: newRange.maps.AfterLast(),
TailPartialEpoch: newRange.tailPartialEpoch,
}
rawdb.WriteFilterMapsRange(batch, rs)
if !isTempRange {
mapCountGauge.Update(int64(newRange.maps.Count() + newRange.tailPartialEpoch))
}
} else {
rawdb.DeleteFilterMapsRange(batch)
mapCountGauge.Update(0)
}
}
// getLogByLvIndex returns the log at the given log value index. If the index does
// not point to the first log value entry of a log then no log and no error are
// returned as this can happen when the log value index was a false positive.
// Note that this function assumes that the log index structure is consistent
// with the canonical chain at the point where the given log value index points.
// If this is not the case then an invalid result or an error may be returned.
//
// Note that this function assumes that the indexer read lock is being held when
// called from outside the indexerLoop goroutine.
func (f *FilterMaps) getLogByLvIndex(lvIndex uint64) (*types.Log, error) {
mapIndex := uint32(lvIndex >> f.logValuesPerMap)
if !f.indexedRange.maps.Includes(mapIndex) {
return nil, nil
}
// find possible block range based on map to block pointers
lastBlockNumber, _, err := f.getLastBlockOfMap(mapIndex)
if err != nil {
return nil, fmt.Errorf("failed to retrieve last block of map %d containing searched log value index %d: %v", mapIndex, lvIndex, err)
}
var firstBlockNumber uint64
if mapIndex > 0 {
firstBlockNumber, _, err = f.getLastBlockOfMap(mapIndex - 1)
if err != nil {
return nil, fmt.Errorf("failed to retrieve last block of map %d before searched log value index %d: %v", mapIndex, lvIndex, err)
}
}
if firstBlockNumber < f.indexedRange.blocks.First() {
firstBlockNumber = f.indexedRange.blocks.First()
}
// find block with binary search based on block to log value index pointers
for firstBlockNumber < lastBlockNumber {
midBlockNumber := (firstBlockNumber + lastBlockNumber + 1) / 2
midLvPointer, err := f.getBlockLvPointer(midBlockNumber)
if err != nil {
return nil, fmt.Errorf("failed to retrieve log value pointer of block %d while binary searching log value index %d: %v", midBlockNumber, lvIndex, err)
}
if lvIndex < midLvPointer {
lastBlockNumber = midBlockNumber - 1
} else {
firstBlockNumber = midBlockNumber
}
}
// get block receipts
receipts := f.indexedView.Receipts(firstBlockNumber)
if receipts == nil {
return nil, fmt.Errorf("failed to retrieve receipts for block %d containing searched log value index %d: %v", firstBlockNumber, lvIndex, err)
}
lvPointer, err := f.getBlockLvPointer(firstBlockNumber)
if err != nil {
return nil, fmt.Errorf("failed to retrieve log value pointer of block %d containing searched log value index %d: %v", firstBlockNumber, lvIndex, err)
}
// iterate through receipts to find the exact log starting at lvIndex
for _, receipt := range receipts {
for _, log := range receipt.Logs {
l := uint64(len(log.Topics) + 1)
r := f.valuesPerMap - lvPointer%f.valuesPerMap
if l > r {
lvPointer += r // skip to map boundary
}
if lvPointer > lvIndex {
// lvIndex does not point to the first log value (address value)
// generated by a log as true matches should always do, so it
// is considered a false positive (no log and no error returned).
return nil, nil
}
if lvPointer == lvIndex {
return log, nil // potential match
}
lvPointer += l
}
}
return nil, nil
}
// getFilterMap fetches an entire filter map from the database.
func (f *FilterMaps) getFilterMap(mapIndex uint32) (filterMap, error) {
if fm, ok := f.filterMapCache.Get(mapIndex); ok {
return fm, nil
}
fm := make(filterMap, f.mapHeight)
for rowIndex := range fm {
rows, err := f.getFilterMapRows([]uint32{mapIndex}, uint32(rowIndex), false)
if err != nil {
return nil, fmt.Errorf("failed to load filter map %d from database: %v", mapIndex, err)
}
fm[rowIndex] = rows[0]
}
f.filterMapCache.Add(mapIndex, fm)
return fm, nil
}
// getFilterMapRows fetches a set of filter map rows at the corresponding map
// indices and a shared row index. If baseLayerOnly is true then only the first
// baseRowLength entries are returned.
func (f *FilterMaps) getFilterMapRows(mapIndices []uint32, rowIndex uint32, baseLayerOnly bool) ([]FilterRow, error) {
rows := make([]FilterRow, len(mapIndices))
var ptr int
for len(mapIndices) > ptr {
var (
groupIndex = f.mapGroupIndex(mapIndices[ptr])
groupLength = 1
)
for ptr+groupLength < len(mapIndices) && f.mapGroupIndex(mapIndices[ptr+groupLength]) == groupIndex {
groupLength++
}
if err := f.getFilterMapRowsOfGroup(rows[ptr:ptr+groupLength], mapIndices[ptr:ptr+groupLength], rowIndex, baseLayerOnly); err != nil {
return nil, err
}
ptr += groupLength
}
return rows, nil
}
// getFilterMapRowsOfGroup fetches a set of filter map rows at map indices
// belonging to the same base row group.
func (f *FilterMaps) getFilterMapRowsOfGroup(target []FilterRow, mapIndices []uint32, rowIndex uint32, baseLayerOnly bool) error {
var (
groupIndex = f.mapGroupIndex(mapIndices[0])
mapRowIndex = f.mapRowIndex(groupIndex, rowIndex)
)
baseRows, err := rawdb.ReadFilterMapBaseRows(f.db, mapRowIndex, f.baseRowGroupSize, f.logMapWidth)
if err != nil {
return fmt.Errorf("failed to retrieve base row group %d of row %d: %v", groupIndex, rowIndex, err)
}
for i, mapIndex := range mapIndices {
if f.mapGroupIndex(mapIndex) != groupIndex {
return fmt.Errorf("maps are not in the same base row group, index: %d, group: %d", mapIndex, groupIndex)
}
row := baseRows[f.mapGroupOffset(mapIndex)]
if !baseLayerOnly {
extRow, err := rawdb.ReadFilterMapExtRow(f.db, f.mapRowIndex(mapIndex, rowIndex), f.logMapWidth)
if err != nil {
return fmt.Errorf("failed to retrieve filter map %d extended row %d: %v", mapIndex, rowIndex, err)
}
row = append(row, extRow...)
}
target[i] = row
}
return nil
}
// storeFilterMapRows stores a set of filter map rows at the corresponding map
// indices and a shared row index.
func (f *FilterMaps) storeFilterMapRows(batch ethdb.Batch, mapIndices []uint32, rowIndex uint32, rows []FilterRow) error {
for len(mapIndices) > 0 {
var (
pos = 1
groupIndex = f.mapGroupIndex(mapIndices[0])
)
for pos < len(mapIndices) && f.mapGroupIndex(mapIndices[pos]) == groupIndex {
pos++
}
if err := f.storeFilterMapRowsOfGroup(batch, mapIndices[:pos], rowIndex, rows[:pos]); err != nil {
return err
}
mapIndices, rows = mapIndices[pos:], rows[pos:]
}
return nil
}
// storeFilterMapRowsOfGroup stores a set of filter map rows at map indices
// belonging to the same base row group.
func (f *FilterMaps) storeFilterMapRowsOfGroup(batch ethdb.Batch, mapIndices []uint32, rowIndex uint32, rows []FilterRow) error {
var (
baseRows [][]uint32
groupIndex = f.mapGroupIndex(mapIndices[0])
mapRowIndex = f.mapRowIndex(groupIndex, rowIndex)
)
if uint32(len(mapIndices)) != f.baseRowGroupSize { // skip base rows read if all rows are replaced
var err error
baseRows, err = rawdb.ReadFilterMapBaseRows(f.db, mapRowIndex, f.baseRowGroupSize, f.logMapWidth)
if err != nil {
return fmt.Errorf("failed to retrieve filter map %d base rows %d for modification: %v", groupIndex, rowIndex, err)
}
} else {
baseRows = make([][]uint32, f.baseRowGroupSize)
}
for i, mapIndex := range mapIndices {
if f.mapGroupIndex(mapIndex) != groupIndex {
return fmt.Errorf("maps are not in the same base row group, index: %d, group: %d", mapIndex, groupIndex)
}
baseRow := []uint32(rows[i])
var extRow FilterRow
if uint32(len(rows[i])) > f.baseRowLength {
extRow = baseRow[f.baseRowLength:]
baseRow = baseRow[:f.baseRowLength]
}
baseRows[f.mapGroupOffset(mapIndex)] = baseRow
rawdb.WriteFilterMapExtRow(batch, f.mapRowIndex(mapIndex, rowIndex), extRow, f.logMapWidth)
}
rawdb.WriteFilterMapBaseRows(batch, mapRowIndex, baseRows, f.logMapWidth)
return nil
}
// mapRowIndex calculates the unified storage index where the given row of the
// given map is stored. Note that this indexing scheme is the same as the one
// proposed in EIP-7745 for tree-hashing the filter map structure and for the
// same data proximity reasons it is also suitable for database representation.
// See also:
// https://eips.ethereum.org/EIPS/eip-7745#hash-tree-structure
func (f *FilterMaps) mapRowIndex(mapIndex, rowIndex uint32) uint64 {
epochIndex, mapSubIndex := mapIndex>>f.logMapsPerEpoch, mapIndex&(f.mapsPerEpoch-1)
return (uint64(epochIndex)<<f.logMapHeight+uint64(rowIndex))<<f.logMapsPerEpoch + uint64(mapSubIndex)
}
// getBlockLvPointer returns the starting log value index where the log values
// generated by the given block are located.
//
// Note that this function assumes that the indexer read lock is being held when
// called from outside the indexerLoop goroutine.
func (f *FilterMaps) getBlockLvPointer(blockNumber uint64) (uint64, error) {
if lvPointer, ok := f.lvPointerCache.Get(blockNumber); ok {
return lvPointer, nil
}
lvPointer, err := rawdb.ReadBlockLvPointer(f.db, blockNumber)
if err != nil {
return 0, fmt.Errorf("failed to retrieve log value pointer of block %d: %v", blockNumber, err)
}
f.lvPointerCache.Add(blockNumber, lvPointer)
return lvPointer, nil
}
// storeBlockLvPointer stores the starting log value index where the log values
// generated by the given block are located.
func (f *FilterMaps) storeBlockLvPointer(batch ethdb.Batch, blockNumber, lvPointer uint64) {
f.lvPointerCache.Add(blockNumber, lvPointer)
rawdb.WriteBlockLvPointer(batch, blockNumber, lvPointer)
}
// deleteBlockLvPointer deletes the starting log value index where the log values
// generated by the given block are located.
func (f *FilterMaps) deleteBlockLvPointer(batch ethdb.Batch, blockNumber uint64) {
f.lvPointerCache.Remove(blockNumber)
rawdb.DeleteBlockLvPointer(batch, blockNumber)
}
// getLastBlockOfMap returns the number and id of the block that generated the
// last log value entry of the given map.
func (f *FilterMaps) getLastBlockOfMap(mapIndex uint32) (uint64, common.Hash, error) {
if lastBlock, ok := f.lastBlockCache.Get(mapIndex); ok {
return lastBlock.number, lastBlock.id, nil
}
number, id, err := rawdb.ReadFilterMapLastBlock(f.db, mapIndex)
if err != nil {
return 0, common.Hash{}, fmt.Errorf("failed to retrieve last block of map %d: %v", mapIndex, err)
}
f.lastBlockCache.Add(mapIndex, lastBlockOfMap{number: number, id: id})
return number, id, nil
}
// storeLastBlockOfMap stores the number of the block that generated the last
// log value entry of the given map.
func (f *FilterMaps) storeLastBlockOfMap(batch ethdb.Batch, mapIndex uint32, number uint64, id common.Hash) {
f.lastBlockCache.Add(mapIndex, lastBlockOfMap{number: number, id: id})
rawdb.WriteFilterMapLastBlock(batch, mapIndex, number, id)
}
// deleteLastBlockOfMap deletes the number of the block that generated the last
// log value entry of the given map.
func (f *FilterMaps) deleteLastBlockOfMap(batch ethdb.Batch, mapIndex uint32) {
f.lastBlockCache.Remove(mapIndex)
rawdb.DeleteFilterMapLastBlock(batch, mapIndex)
}
// deleteTailEpoch deletes index data from the specified epoch. The last block
// pointer for the last map of the epoch and the corresponding block log value
// pointer are retained as these are always assumed to be available for each
// epoch as boundary markers.
// The function returns true if all index data related to the epoch (except for
// the boundary markers) has been fully removed.
func (f *FilterMaps) deleteTailEpoch(epoch uint32) (bool, error) {
f.indexLock.Lock()
defer f.indexLock.Unlock()
// determine epoch boundaries
lastBlock, _, err := f.getLastBlockOfMap(f.lastEpochMap(epoch))
if err != nil {
return false, fmt.Errorf("failed to retrieve last block of deleted epoch %d: %v", epoch, err)
}
var firstBlock uint64
firstMap := f.firstEpochMap(epoch)
if epoch > 0 {
firstBlock, _, err = f.getLastBlockOfMap(firstMap - 1)
if err != nil {
return false, fmt.Errorf("failed to retrieve last block before deleted epoch %d: %v", epoch, err)
}
firstBlock++
}
// update rendered range if necessary
var (
fmr = f.indexedRange
firstEpoch = f.mapEpoch(f.indexedRange.maps.First())
afterLastEpoch = f.mapEpoch(f.indexedRange.maps.AfterLast() + f.mapsPerEpoch - 1)
)
if f.indexedRange.tailPartialEpoch != 0 && firstEpoch > 0 {
firstEpoch--
}
switch {
case epoch < firstEpoch:
// cleanup of already unindexed epoch; range not affected
case epoch == firstEpoch && epoch+1 < afterLastEpoch:
// first fully or partially rendered epoch and there is at least one
// rendered map in the next epoch; remove from indexed range
fmr.tailPartialEpoch = 0
fmr.maps.SetFirst(f.firstEpochMap(epoch + 1))
fmr.blocks.SetFirst(lastBlock + 1)
f.setRange(f.db, f.indexedView, fmr, false)
default:
// cannot be cleaned or unindexed; return with error
return false, errors.New("invalid tail epoch number")
}
// remove index data
deleteFn := func(db ethdb.KeyValueStore, hashScheme bool, stopCb func(bool) bool) error {
first := f.mapRowIndex(firstMap, 0)
count := f.mapRowIndex(firstMap+f.mapsPerEpoch, 0) - first
if err := rawdb.DeleteFilterMapRows(f.db, common.NewRange(first, count), hashScheme, stopCb); err != nil {
return err
}
for mapIndex := firstMap; mapIndex < firstMap+f.mapsPerEpoch; mapIndex++ {
f.filterMapCache.Remove(mapIndex)
}
delMapRange := common.NewRange(firstMap, f.mapsPerEpoch-1) // keep last entry
if err := rawdb.DeleteFilterMapLastBlocks(f.db, delMapRange, hashScheme, stopCb); err != nil {
return err
}
for mapIndex := firstMap; mapIndex < firstMap+f.mapsPerEpoch-1; mapIndex++ {
f.lastBlockCache.Remove(mapIndex)
}
delBlockRange := common.NewRange(firstBlock, lastBlock-firstBlock) // keep last entry
if err := rawdb.DeleteBlockLvPointers(f.db, delBlockRange, hashScheme, stopCb); err != nil {
return err
}
for blockNumber := firstBlock; blockNumber < lastBlock; blockNumber++ {
f.lvPointerCache.Remove(blockNumber)
}
return nil
}
action := fmt.Sprintf("Deleting tail epoch #%d", epoch)
stopFn := func() bool {
f.processEvents()
return f.stop || !f.targetHeadIndexed()
}
if err := f.safeDeleteWithLogs(deleteFn, action, stopFn); err == nil {
// everything removed; mark as cleaned and report success
if f.cleanedEpochsBefore == epoch {
f.cleanedEpochsBefore = epoch + 1
}
return true, nil
} else {
// more data left in epoch range; mark as dirty and report unfinished
if f.cleanedEpochsBefore > epoch {
f.cleanedEpochsBefore = epoch
}
if errors.Is(err, rawdb.ErrDeleteRangeInterrupted) {
return false, nil
}
return false, err
}
}
// exportCheckpoints exports epoch checkpoints in the format used by checkpoints.go.
//
// Note: acquiring the indexLock read lock is unnecessary here, as this function
// is always called within the indexLoop.
func (f *FilterMaps) exportCheckpoints() {
finalLvPtr, err := f.getBlockLvPointer(f.finalBlock + 1)
if err != nil {
log.Error("Error fetching log value pointer of finalized block", "block", f.finalBlock, "error", err)
return
}
epochCount := uint32(finalLvPtr >> (f.logValuesPerMap + f.logMapsPerEpoch))
if epochCount == f.lastFinalEpoch {
return
}
w, err := os.Create(f.exportFileName)
if err != nil {
log.Error("Error creating checkpoint export file", "name", f.exportFileName, "error", err)
return
}
defer w.Close()
log.Info("Exporting log index checkpoints", "epochs", epochCount, "file", f.exportFileName)
w.WriteString("[\n")
comma := ","
for epoch := uint32(0); epoch < epochCount; epoch++ {
lastBlock, lastBlockId, err := f.getLastBlockOfMap(f.lastEpochMap(epoch))
if err != nil {
log.Error("Error fetching last block of epoch", "epoch", epoch, "error", err)
return
}
lvPtr, err := f.getBlockLvPointer(lastBlock)
if err != nil {
log.Error("Error fetching log value pointer of last block", "block", lastBlock, "error", err)
return
}
if epoch == epochCount-1 {
comma = ""
}
w.WriteString(fmt.Sprintf("{\"blockNumber\": %d, \"blockId\": \"0x%064x\", \"firstIndex\": %d}%s\n", lastBlock, lastBlockId, lvPtr, comma))
}
w.WriteString("]\n")
f.lastFinalEpoch = epochCount
}

274
core/filtermaps/index_view.go Normal file
View file

@ -0,0 +1,274 @@
// Copyright 2025 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package filtermaps
import (
"errors"
"sync/atomic"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
)
var (
ErrInvalidView = errors.New("index view already invalidated")
)
type IndexView struct {
// if invalid <= 0 then the view is considered invalid and will return error
// on any read operation (storage maps before firstMemoryMap might have changed
// since its creation)
refCount, invalid int32
storage *mapStorage
tailEpoch uint32 //TODO apply to read functions
blockRange common.Range[uint64] //TODO apply to read functions
headBlockHash common.Hash
headLvPointer uint64 // points after head block delimiter
firstMemoryMap uint32
firstMemoryBlock uint64
finishedMaps []*finishedMap
headMapIndex uint32
headMap *memoryMap
}
func (iv *IndexView) Release() {
iv.addRefCount(-1)
}
func (iv *IndexView) addRefCount(add int32) bool {
return atomic.AddInt32(&iv.refCount, add) <= 0
}
func (iv *IndexView) checkReleased() bool {
return atomic.LoadInt32(&iv.refCount) <= 0
}
func (iv *IndexView) invalidate() {
atomic.StoreInt32(&iv.invalid, 1)
}
func (iv *IndexView) checkInvalid() bool {
return atomic.LoadInt32(&iv.invalid) != 0
}
func (iv *IndexView) GetParams() *Params {
return iv.storage.params
}
func (iv *IndexView) BlockRange() common.Range[uint64] {
return iv.blockRange
}
// getBlockLvPointer returns the starting log value index where the log values
// generated by the given block are located.
// Note that the function returns a valid result (the next free log value index)
// for headNumber + 1.
func (iv *IndexView) GetBlockLvPointer(blockNumber uint64) (uint64, error) {
if iv.checkInvalid() {
return 0, ErrInvalidView
}
if blockNumber < iv.firstMemoryBlock {
lvPtr, err := iv.storage.getBlockLvPointer(blockNumber)
if iv.checkInvalid() {
return 0, ErrInvalidView
}
return lvPtr, err
}
if blockNumber == iv.blockRange.AfterLast() {
return iv.headLvPointer, nil
}
if blockNumber > iv.blockRange.AfterLast() {
return 0, ErrOutOfRange
}
for _, fm := range iv.finishedMaps {
if blockNumber >= fm.firstBlock() && blockNumber <= fm.lastBlock.number {
return fm.blockPtrs[blockNumber-fm.firstBlock()], nil
}
}
if blockNumber >= iv.headMap.firstBlock() && blockNumber <= iv.headMap.lastBlock.number {
return iv.headMap.blockPtrs[blockNumber-iv.headMap.firstBlock()], nil
}
panic("IndexView.GetBlockLvPointer: gap in blockLvPtrs")
}
// GetLastBlockOfMap returns the number and hash of the block that generated the
// last log value entry of the given map.
func (iv *IndexView) GetLastBlockOfMap(mapIndex uint32) (uint64, common.Hash, error) {
if iv.checkInvalid() {
return 0, common.Hash{}, ErrInvalidView
}
if mapIndex < iv.firstMemoryMap {
lastNumber, lastHash, err := iv.storage.getLastBlockOfMap(mapIndex)
if iv.checkInvalid() {
return 0, common.Hash{}, ErrInvalidView
}
return lastNumber, lastHash, err
}
if mapIndex > iv.headMapIndex {
return 0, common.Hash{}, ErrOutOfRange
}
if mapIndex == iv.headMapIndex {
return iv.headMap.lastBlock.number, iv.headMap.lastBlock.hash, nil
}
fm := iv.finishedMaps[mapIndex-iv.firstMemoryMap]
return fm.lastBlock.number, fm.lastBlock.hash, nil
}
// GetFilterMapRows returns a batch of filter maps rows from the same row index,
// each truncated to the length limit of the specified mapping layer.
// The function assumes that the map indices are in strictly ascending order.
func (iv *IndexView) GetFilterMapRows(mapIndices []uint32, rowIndex, layerIndex uint32) (rows []FilterRow, err error) {
if iv.checkInvalid() {
return nil, ErrInvalidView
}
dbIndices := len(mapIndices)
for dbIndices > 0 && mapIndices[dbIndices-1] >= iv.firstMemoryMap {
dbIndices--
}
if dbIndices > 0 {
rows, err = iv.storage.getFilterMapRows(mapIndices[:dbIndices], rowIndex, layerIndex)
if iv.checkInvalid() {
return nil, ErrInvalidView
}
if err != nil {
return nil, err
}
}
for i := dbIndices; i < len(mapIndices); i++ {
mapIndex := mapIndices[i]
var row FilterRow
if mapIndex == iv.headMapIndex {
row = iv.headMap.getRow(rowIndex, iv.storage.params.getMaxRowLength(layerIndex+1))
}
if mapIndex < iv.headMapIndex {
row = iv.finishedMaps[mapIndex-iv.firstMemoryMap].getRow(rowIndex, iv.storage.params.getMaxRowLength(layerIndex+1))
}
rows = append(rows, row)
}
return rows, nil
}
type renderState struct {
params *Params
renderRange common.Range[uint32]
lvPointer uint64
mapIndex uint32
currentMap *memoryMap
finishedMaps []*finishedMap
nextBlock uint64
partialBlock bool
partialBlockHash common.Hash
}
func (rs *renderState) checkNextHash(hash common.Hash) bool {
if rs.partialBlock && rs.partialBlockHash != hash {
return false
}
rs.partialBlock = false
return true
}
func (rs *renderState) addReceipts(receipts types.Receipts) {
if rs.partialBlock {
panic("checkNextHash has to be called before adding partially rendered block")
}
if rs.currentMap != nil {
rs.currentMap.blockPtrs = append(rs.currentMap.blockPtrs, rs.lvPointer)
rs.currentMap.lastBlock = lastBlockOfMap{number: rs.nextBlock} // hash will be set by addHeader
}
for _, receipt := range receipts {
//TODO add tx delimiter
for _, log := range receipt.Logs {
mapRemaining := rs.params.valuesPerMap - rs.lvPointer%rs.params.valuesPerMap
if mapRemaining <= uint64(len(log.Topics)) {
rs.advance(mapRemaining)
}
if rs.currentMap != nil {
rs.addValue(addressValue(log.Address))
for _, topic := range log.Topics {
rs.addValue(topicValue(topic))
}
} else {
rs.advance(uint64(len(log.Topics) + 1))
}
if rs.finished() {
return
}
}
}
}
func (rs *renderState) addHeader(header *types.Header) (uint32, []*finishedMap) {
if rs.partialBlock {
panic("checkNextHash has to be called before adding partially rendered block")
}
if rs.nextBlock != header.Number.Uint64() {
panic("wrong block number")
}
if !rs.finished() {
rs.advance(1) //TODO blockValue
}
rs.nextBlock++
lastBlock := lastBlockOfMap{number: header.Number.Uint64(), hash: header.Hash()}
if rs.currentMap != nil {
rs.currentMap.lastBlock = lastBlock
}
for _, fm := range rs.finishedMaps {
fm.lastBlock = lastBlock
}
firstMemoryMap, finishedMaps := rs.mapIndex-uint32(len(rs.finishedMaps)), rs.finishedMaps
rs.finishedMaps = nil
return firstMemoryMap, finishedMaps
}
// assumes currentMap != nil
func (rs *renderState) addValue(logValue common.Hash) {
if rs.renderRange.Includes(rs.mapIndex) {
for layerIndex := uint32(0); ; layerIndex++ {
rowIndex := rs.params.rowIndex(rs.mapIndex, layerIndex, logValue)
if rs.currentMap.rowLength(rowIndex) < rs.params.getMaxRowLength(layerIndex) {
rs.currentMap.addToRow(rowIndex, rs.params.columnIndex(rs.lvPointer, &logValue))
break
}
}
}
rs.advance(1)
}
func (rs *renderState) advance(count uint64) {
rs.lvPointer += count
if uint32(rs.lvPointer>>rs.params.logValuesPerMap) > rs.mapIndex {
if rs.currentMap != nil {
rs.finishedMaps = append(rs.finishedMaps, rs.currentMap.finished())
}
rs.mapIndex++
if rs.renderRange.Includes(rs.mapIndex) {
rs.currentMap = rs.params.newMemoryMap()
} else {
rs.currentMap = nil
}
}
}
func (rs *renderState) finished() bool {
return rs.mapIndex >= rs.renderRange.AfterLast()
}

989
core/filtermaps/indexer.go
View file

File diff suppressed because it is too large Load diff

637
core/filtermaps/indexer_test.go
View file

@ -1,637 +0,0 @@
// Copyright 2024 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package filtermaps
import (
"context"
crand "crypto/rand"
"crypto/sha256"
"encoding/binary"
"math/big"
"math/rand"
"sync"
"testing"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/consensus/ethash"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp"
)
var testParams = Params{
logMapHeight: 2,
logMapWidth: 24,
logMapsPerEpoch: 4,
logValuesPerMap: 4,
baseRowGroupSize: 4,
baseRowLengthRatio: 2,
logLayerDiff: 2,
}
func TestIndexerRandomRange(t *testing.T) {
ts := newTestSetup(t)
defer ts.close()
forks := make([][]common.Hash, 10)
ts.chain.addBlocks(1000, 5, 2, 4, false)
for i := range forks {
if i != 0 {
forkBlock := rand.Intn(1000)
ts.chain.setHead(forkBlock)
ts.chain.addBlocks(1000-forkBlock, 5, 2, 4, false)
}
forks[i] = ts.chain.getCanonicalChain()
}
expspos := func(block uint64) uint64 { // expected position of block start
if block == 0 {
return 0
}
logCount := (block - 1) * 5 * 2
mapIndex := logCount / 3
spos := mapIndex*16 + (logCount%3)*5
if mapIndex == 0 || logCount%3 != 0 {
spos++
}
return spos
}
expdpos := func(block uint64) uint64 { // expected position of delimiter
if block == 0 {
return 0
}
logCount := block * 5 * 2
mapIndex := (logCount - 1) / 3
return mapIndex*16 + (logCount-mapIndex*3)*5
}
ts.setHistory(0, false)
var (
history int
noHistory bool
fork, head = len(forks) - 1, 1000
checkSnapshot bool
)
ts.fm.WaitIdle()
for i := 0; i < 200; i++ {
switch rand.Intn(3) {
case 0:
// change history settings
switch rand.Intn(10) {
case 0:
history, noHistory = 0, false
case 1:
history, noHistory = 0, true
default:
history, noHistory = rand.Intn(1000)+1, false
}
ts.testDisableSnapshots = rand.Intn(2) == 0
ts.setHistory(uint64(history), noHistory)
case 1:
// change head to random position of random fork
fork, head = rand.Intn(len(forks)), rand.Intn(1001)
ts.chain.setCanonicalChain(forks[fork][:head+1])
case 2:
checkSnapshot = false
if head < 1000 {
checkSnapshot = !noHistory && head != 0 // no snapshot generated for block 0
// add blocks after the current head
head += rand.Intn(1000-head) + 1
ts.fm.testSnapshotUsed = false
ts.chain.setCanonicalChain(forks[fork][:head+1])
}
}
ts.fm.WaitIdle()
if checkSnapshot {
if ts.fm.testSnapshotUsed == ts.fm.testDisableSnapshots {
ts.t.Fatalf("Invalid snapshot used state after head extension (used: %v, disabled: %v)", ts.fm.testSnapshotUsed, ts.fm.testDisableSnapshots)
}
checkSnapshot = false
}
if noHistory {
if ts.fm.indexedRange.initialized {
t.Fatalf("filterMapsRange initialized while indexing is disabled")
}
continue
}
if !ts.fm.indexedRange.initialized {
t.Fatalf("filterMapsRange not initialized while indexing is enabled")
}
var tailBlock uint64
if history > 0 && history <= head {
tailBlock = uint64(head + 1 - history)
}
var tailEpoch uint32
if tailBlock > 0 {
tailLvPtr := expspos(tailBlock) - 1
tailEpoch = uint32(tailLvPtr >> (testParams.logValuesPerMap + testParams.logMapsPerEpoch))
}
tailLvPtr := uint64(tailEpoch) << (testParams.logValuesPerMap + testParams.logMapsPerEpoch) // first available lv ptr
var expTailBlock uint64
if tailEpoch > 0 {
for expspos(expTailBlock) <= tailLvPtr {
expTailBlock++
}
}
if ts.fm.indexedRange.blocks.Last() != uint64(head) {
ts.t.Fatalf("Invalid index head (expected #%d, got #%d)", head, ts.fm.indexedRange.blocks.Last())
}
expHeadDelimiter := expdpos(uint64(head))
if ts.fm.indexedRange.headDelimiter != expHeadDelimiter {
ts.t.Fatalf("Invalid index head delimiter pointer (expected %d, got %d)", expHeadDelimiter, ts.fm.indexedRange.headDelimiter)
}
if ts.fm.indexedRange.blocks.First() != expTailBlock {
ts.t.Fatalf("Invalid index tail block (expected #%d, got #%d)", expTailBlock, ts.fm.indexedRange.blocks.First())
}
}
}
func TestIndexerMatcherView(t *testing.T) {
testIndexerMatcherView(t, false)
}
func TestIndexerMatcherViewWithConcurrentRead(t *testing.T) {
testIndexerMatcherView(t, true)
}
func testIndexerMatcherView(t *testing.T, concurrentRead bool) {
ts := newTestSetup(t)
defer ts.close()
forks := make([][]common.Hash, 20)
hashes := make([]common.Hash, 20)
ts.chain.addBlocks(100, 5, 2, 4, true)
ts.setHistory(0, false)
for i := range forks {
if i != 0 {
ts.chain.setHead(100 - i)
ts.chain.addBlocks(i, 5, 2, 4, true)
}
ts.fm.WaitIdle()
forks[i] = ts.chain.getCanonicalChain()
hashes[i] = ts.matcherViewHash()
}
fork := len(forks) - 1
for i := 0; i < 5000; i++ {
oldFork := fork
fork = rand.Intn(len(forks))
stopCh := make(chan chan struct{})
if concurrentRead {
go func() {
for {
ts.matcherViewHash()
select {
case ch := <-stopCh:
close(ch)
return
default:
}
}
}()
}
ts.chain.setCanonicalChain(forks[fork])
ts.fm.WaitIdle()
if concurrentRead {
ch := make(chan struct{})
stopCh <- ch
<-ch
}
hash := ts.matcherViewHash()
if hash != hashes[fork] {
t.Fatalf("Matcher view hash mismatch when switching from for %d to %d", oldFork, fork)
}
}
}
func TestLogsByIndex(t *testing.T) {
ts := newTestSetup(t)
defer func() {
ts.fm.testProcessEventsHook = nil
ts.close()
}()
ts.chain.addBlocks(1000, 10, 3, 4, true)
ts.setHistory(0, false)
ts.fm.WaitIdle()
firstLog := make([]uint64, 1001) // first valid log position per block
lastLog := make([]uint64, 1001) // last valid log position per block
for i := uint64(0); i <= ts.fm.indexedRange.headDelimiter; i++ {
log, err := ts.fm.getLogByLvIndex(i)
if err != nil {
t.Fatalf("Error getting log by index %d: %v", i, err)
}
if log != nil {
if firstLog[log.BlockNumber] == 0 {
firstLog[log.BlockNumber] = i
}
lastLog[log.BlockNumber] = i
}
}
var failed bool
ts.fm.testProcessEventsHook = func() {
if ts.fm.indexedRange.blocks.IsEmpty() {
return
}
if lvi := firstLog[ts.fm.indexedRange.blocks.First()]; lvi != 0 {
log, err := ts.fm.getLogByLvIndex(lvi)
if log == nil || err != nil {
t.Errorf("Error getting first log of indexed block range: %v", err)
failed = true
}
}
if lvi := lastLog[ts.fm.indexedRange.blocks.Last()]; lvi != 0 {
log, err := ts.fm.getLogByLvIndex(lvi)
if log == nil || err != nil {
t.Errorf("Error getting last log of indexed block range: %v", err)
failed = true
}
}
}
chain := ts.chain.getCanonicalChain()
for i := 0; i < 1000 && !failed; i++ {
head := rand.Intn(len(chain))
ts.chain.setCanonicalChain(chain[:head+1])
ts.fm.WaitIdle()
}
}
func TestIndexerCompareDb(t *testing.T) {
ts := newTestSetup(t)
defer ts.close()
ts.chain.addBlocks(500, 10, 3, 4, true)
ts.setHistory(0, false)
ts.fm.WaitIdle()
// revert points are stored after block 500
ts.chain.addBlocks(500, 10, 3, 4, true)
ts.fm.WaitIdle()
chain1 := ts.chain.getCanonicalChain()
ts.storeDbHash("chain 1 [0, 1000]")
ts.chain.setHead(600)
ts.fm.WaitIdle()
ts.storeDbHash("chain 1/2 [0, 600]")
ts.chain.addBlocks(600, 10, 3, 4, true)
ts.fm.WaitIdle()
chain2 := ts.chain.getCanonicalChain()
ts.storeDbHash("chain 2 [0, 1200]")
ts.chain.setHead(600)
ts.fm.WaitIdle()
ts.checkDbHash("chain 1/2 [0, 600]")
ts.setHistory(800, false)
ts.chain.setCanonicalChain(chain1)
ts.fm.WaitIdle()
ts.storeDbHash("chain 1 [201, 1000]")
ts.setHistory(0, false)
ts.fm.WaitIdle()
ts.checkDbHash("chain 1 [0, 1000]")
ts.setHistory(800, false)
ts.chain.setCanonicalChain(chain2)
ts.fm.WaitIdle()
ts.storeDbHash("chain 2 [401, 1200]")
ts.setHistory(0, true)
ts.fm.WaitIdle()
ts.storeDbHash("no index")
ts.chain.setCanonicalChain(chain2[:501])
ts.setHistory(0, false)
ts.fm.WaitIdle()
ts.chain.setCanonicalChain(chain2)
ts.fm.WaitIdle()
ts.checkDbHash("chain 2 [0, 1200]")
ts.chain.setCanonicalChain(chain1)
ts.fm.WaitIdle()
ts.setHistory(800, false)
ts.fm.WaitIdle()
ts.checkDbHash("chain 1 [201, 1000]")
ts.chain.setCanonicalChain(chain2)
ts.fm.WaitIdle()
ts.checkDbHash("chain 2 [401, 1200]")
ts.setHistory(0, true)
ts.fm.WaitIdle()
ts.checkDbHash("no index")
}
type testSetup struct {
t *testing.T
fm *FilterMaps
db ethdb.Database
chain *testChain
params Params
dbHashes map[string]common.Hash
testDisableSnapshots bool
}
func newTestSetup(t *testing.T) *testSetup {
params := testParams
params.deriveFields()
ts := &testSetup{
t: t,
db: rawdb.NewMemoryDatabase(),
params: params,
dbHashes: make(map[string]common.Hash),
}
ts.chain = ts.newTestChain()
return ts
}
func (ts *testSetup) setHistory(history uint64, noHistory bool) {
if ts.fm != nil {
ts.fm.Stop()
}
head := ts.chain.CurrentBlock()
view := NewChainView(ts.chain, head.Number.Uint64(), head.Hash())
config := Config{
History: history,
Disabled: noHistory,
}
ts.fm, _ = NewFilterMaps(ts.db, view, 0, 0, ts.params, config)
ts.fm.testDisableSnapshots = ts.testDisableSnapshots
ts.fm.Start()
}
func (ts *testSetup) storeDbHash(id string) {
dbHash := ts.fmDbHash()
for otherId, otherHash := range ts.dbHashes {
if otherHash == dbHash {
ts.t.Fatalf("Unexpected equal database hashes `%s` and `%s`", id, otherId)
}
}
ts.dbHashes[id] = dbHash
}
func (ts *testSetup) checkDbHash(id string) {
if ts.fmDbHash() != ts.dbHashes[id] {
ts.t.Fatalf("Database `%s` hash mismatch", id)
}
}
func (ts *testSetup) fmDbHash() common.Hash {
hasher := sha256.New()
it := ts.db.NewIterator(nil, nil)
for it.Next() {
hasher.Write(it.Key())
hasher.Write(it.Value())
}
it.Release()
var result common.Hash
hasher.Sum(result[:0])
return result
}
func (ts *testSetup) matcherViewHash() common.Hash {
mb := ts.fm.NewMatcherBackend()
defer mb.Close()
ctx := context.Background()
params := mb.GetParams()
hasher := sha256.New()
var headPtr uint64
for b := uint64(0); ; b++ {
lvptr, err := mb.GetBlockLvPointer(ctx, b)
if err != nil || (b > 0 && lvptr == headPtr) {
break
}
var enc [8]byte
binary.LittleEndian.PutUint64(enc[:], lvptr)
hasher.Write(enc[:])
headPtr = lvptr
}
headMap := uint32(headPtr >> params.logValuesPerMap)
var enc [12]byte
for r := uint32(0); r < params.mapHeight; r++ {
binary.LittleEndian.PutUint32(enc[:4], r)
for m := uint32(0); m <= headMap; m++ {
binary.LittleEndian.PutUint32(enc[4:8], m)
rows, _ := mb.GetFilterMapRows(ctx, []uint32{m}, r, false)
for _, row := range rows {
for _, v := range row {
binary.LittleEndian.PutUint32(enc[8:], v)
hasher.Write(enc[:])
}
}
}
}
var hash common.Hash
hasher.Sum(hash[:0])
for i := 0; i < 50; i++ {
hasher.Reset()
hasher.Write(hash[:])
lvptr := binary.LittleEndian.Uint64(hash[:8]) % headPtr
if log, _ := mb.GetLogByLvIndex(ctx, lvptr); log != nil {
enc, err := rlp.EncodeToBytes(log)
if err != nil {
panic(err)
}
hasher.Write(enc)
}
hasher.Sum(hash[:0])
}
return hash
}
func (ts *testSetup) close() {
if ts.fm != nil {
ts.fm.Stop()
}
ts.db.Close()
ts.chain.db.Close()
}
type testChain struct {
ts *testSetup
db ethdb.Database
lock sync.RWMutex
canonical []common.Hash
blocks map[common.Hash]*types.Block
receipts map[common.Hash]types.Receipts
}
func (ts *testSetup) newTestChain() *testChain {
return &testChain{
ts: ts,
blocks: make(map[common.Hash]*types.Block),
receipts: make(map[common.Hash]types.Receipts),
}
}
func (tc *testChain) CurrentBlock() *types.Header {
tc.lock.RLock()
defer tc.lock.RUnlock()
if len(tc.canonical) == 0 {
return nil
}
return tc.blocks[tc.canonical[len(tc.canonical)-1]].Header()
}
func (tc *testChain) GetHeader(hash common.Hash, number uint64) *types.Header {
tc.lock.RLock()
defer tc.lock.RUnlock()
if block := tc.blocks[hash]; block != nil {
return block.Header()
}
return nil
}
func (tc *testChain) GetCanonicalHash(number uint64) common.Hash {
tc.lock.RLock()
defer tc.lock.RUnlock()
if uint64(len(tc.canonical)) <= number {
return common.Hash{}
}
return tc.canonical[number]
}
func (tc *testChain) GetReceiptsByHash(hash common.Hash) types.Receipts {
tc.lock.RLock()
defer tc.lock.RUnlock()
return tc.receipts[hash]
}
func (tc *testChain) GetRawReceipts(hash common.Hash, number uint64) types.Receipts {
tc.lock.RLock()
defer tc.lock.RUnlock()
return tc.receipts[hash]
}
func (tc *testChain) addBlocks(count, maxTxPerBlock, maxLogsPerReceipt, maxTopicsPerLog int, random bool) {
tc.lock.Lock()
blockGen := func(i int, gen *core.BlockGen) {
var txCount int
if random {
txCount = rand.Intn(maxTxPerBlock + 1)
} else {
txCount = maxTxPerBlock
}
for k := txCount; k > 0; k-- {
receipt := types.NewReceipt(nil, false, 0)
var logCount int
if random {
logCount = rand.Intn(maxLogsPerReceipt + 1)
} else {
logCount = maxLogsPerReceipt
}
receipt.Logs = make([]*types.Log, logCount)
for i := range receipt.Logs {
log := &types.Log{}
receipt.Logs[i] = log
crand.Read(log.Address[:])
var topicCount int
if random {
topicCount = rand.Intn(maxTopicsPerLog + 1)
} else {
topicCount = maxTopicsPerLog
}
log.Topics = make([]common.Hash, topicCount)
for j := range log.Topics {
crand.Read(log.Topics[j][:])
}
}
gen.AddUncheckedReceipt(receipt)
gen.AddUncheckedTx(types.NewTransaction(999, common.HexToAddress("0x999"), big.NewInt(999), 999, gen.BaseFee(), nil))
}
}
var (
blocks []*types.Block
receipts []types.Receipts
engine = ethash.NewFaker()
)
if len(tc.canonical) == 0 {
gspec := &core.Genesis{
Alloc: types.GenesisAlloc{},
BaseFee: big.NewInt(params.InitialBaseFee),
Config: params.TestChainConfig,
}
tc.db, blocks, receipts = core.GenerateChainWithGenesis(gspec, engine, count, blockGen)
gblock := gspec.ToBlock()
ghash := gblock.Hash()
tc.canonical = []common.Hash{ghash}
tc.blocks[ghash] = gblock
tc.receipts[ghash] = types.Receipts{}
} else {
blocks, receipts = core.GenerateChain(params.TestChainConfig, tc.blocks[tc.canonical[len(tc.canonical)-1]], engine, tc.db, count, blockGen)
}
for i, block := range blocks {
num, hash := int(block.NumberU64()), block.Hash()
if len(tc.canonical) != num {
panic("canonical chain length mismatch")
}
tc.canonical = append(tc.canonical, hash)
tc.blocks[hash] = block
if receipts[i] != nil {
tc.receipts[hash] = receipts[i]
} else {
tc.receipts[hash] = types.Receipts{}
}
}
tc.lock.Unlock()
tc.setTargetHead()
}
func (tc *testChain) setHead(headNum int) {
tc.lock.Lock()
tc.canonical = tc.canonical[:headNum+1]
tc.lock.Unlock()
tc.setTargetHead()
}
func (tc *testChain) setTargetHead() {
head := tc.CurrentBlock()
if tc.ts.fm != nil {
if !tc.ts.fm.disabled {
//tc.ts.fm.targetViewCh <- NewChainView(tc, head.Number.Uint64(), head.Hash())
tc.ts.fm.SetTarget(NewChainView(tc, head.Number.Uint64(), head.Hash()), 0, 0)
}
}
}
func (tc *testChain) getCanonicalChain() []common.Hash {
tc.lock.RLock()
defer tc.lock.RUnlock()
cc := make([]common.Hash, len(tc.canonical))
copy(cc, tc.canonical)
return cc
}
// restore an earlier state of the chain
func (tc *testChain) setCanonicalChain(cc []common.Hash) {
tc.lock.Lock()
tc.canonical = make([]common.Hash, len(cc))
copy(tc.canonical, cc)
tc.lock.Unlock()
tc.setTargetHead()
}

575
core/filtermaps/map_database.go Normal file
View file

@ -0,0 +1,575 @@
// Copyright 2025 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package filtermaps
import (
"errors"
"fmt"
"math"
"sort"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/lru"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/log"
)
const (
databaseVersion = 4 // reindexed if database version does not match
cachedLastBlocks = 1000 // last block of map pointers
cachedLvPointers = 1000 // first log value pointer of block pointers
maxWritesPerBatch = 1000000
)
// mapDatabase implements the database storage layer for filter maps and the
// belonging pointers.
type mapDatabase struct {
params *Params
db ethdb.KeyValueStore
hashScheme bool
lastBlockCache *lru.Cache[uint32, lastBlockOfMap]
lvPointerCache *lru.Cache[uint64, uint64]
testReadRows bool
}
// lastBlockOfMap is used for storing and caching the (number, hash) pairs
// belonging to the last block of each map.
type lastBlockOfMap struct {
number uint64
hash common.Hash
}
// newMapDatabase created a new mapDatabase layer.
func newMapDatabase(params *Params, db ethdb.KeyValueStore, hashScheme bool) *mapDatabase {
return &mapDatabase{
params: params,
db: db,
hashScheme: hashScheme,
lastBlockCache: lru.NewCache[uint32, lastBlockOfMap](cachedLastBlocks),
lvPointerCache: lru.NewCache[uint64, uint64](cachedLvPointers),
}
}
// loadMapRange loads the valid and dirty map ranges and known epoch boundaries
// from the database.
func (m *mapDatabase) loadMapRange() (valid, dirty rangeSet[uint32], knownEpochs uint32, found bool) {
fmr, ok, err := rawdb.ReadFilterMapsRange(m.db)
if !ok || err != nil || fmr.Version != databaseVersion {
return
}
return decodeRangeSet32(fmr.ValidMaps), decodeRangeSet32(fmr.DirtyMaps), fmr.KnownEpochs, true
}
// storeMapRange stores the valid and dirty map ranges and known epoch boundaries
// into the database.
func (m *mapDatabase) storeMapRange(valid, dirty rangeSet[uint32], knownEpochs uint32) {
rawdb.WriteFilterMapsRange(m.db, rawdb.FilterMapsRange{
Version: databaseVersion,
ValidMaps: encodeRangeSet32(valid),
DirtyMaps: encodeRangeSet32(dirty),
KnownEpochs: knownEpochs,
})
}
// deleteMapRange removes the map range entry from the database, signaling a
// revert to the non-initialized state.
func (m *mapDatabase) deleteMapRange() {
rawdb.DeleteFilterMapsRange(m.db)
}
func decodeRangeSet32(enc []uint32) rangeSet[uint32] {
rs := make(rangeSet[uint32], len(enc)/2)
for i := range rs {
rs[i] = common.NewRange(enc[i*2], enc[i*2+1])
}
return rs
}
func encodeRangeSet32(rs rangeSet[uint32]) []uint32 {
enc := make([]uint32, len(rs)*2)
for i, r := range rs {
enc[i*2] = r.First()
enc[i*2+1] = r.Count()
}
return enc
}
// getBlockLvPointer returns the starting log value index where the log values
// generated by the given block are located.
func (m *mapDatabase) getBlockLvPointer(blockNumber uint64) (uint64, error) {
if lvPointer, ok := m.lvPointerCache.Get(blockNumber); ok {
return lvPointer, nil
}
lvPointer, err := rawdb.ReadBlockLvPointer(m.db, blockNumber)
if err != nil {
return 0, fmt.Errorf("failed to retrieve log value pointer of block %d: %v", blockNumber, err)
}
m.lvPointerCache.Add(blockNumber, lvPointer)
return lvPointer, nil
}
// storeBlockLvPointer stores the starting log value index where the log values
// generated by the given block are located.
func (m *mapDatabase) storeBlockLvPointer(db ethdb.KeyValueWriter, blockNumber, lvPointer uint64) {
m.lvPointerCache.Add(blockNumber, lvPointer)
rawdb.WriteBlockLvPointer(db, blockNumber, lvPointer)
}
// getLastBlockOfMap returns the number and hash of the block that generated the
// last log value entry of the given map.
func (m *mapDatabase) getLastBlockOfMap(mapIndex uint32) (uint64, common.Hash, error) {
if lastBlock, ok := m.lastBlockCache.Get(mapIndex); ok {
return lastBlock.number, lastBlock.hash, nil
}
number, hash, err := rawdb.ReadFilterMapLastBlock(m.db, mapIndex)
if err != nil {
return 0, common.Hash{}, fmt.Errorf("failed to retrieve last block of map %d: %v", mapIndex, err)
}
m.lastBlockCache.Add(mapIndex, lastBlockOfMap{number: number, hash: hash})
return number, hash, nil
}
// storeLastBlockOfMap stores the number of the block that generated the last
// log value entry of the given map.
func (m *mapDatabase) storeLastBlockOfMap(db ethdb.KeyValueWriter, mapIndex uint32, number uint64, hash common.Hash) {
m.lastBlockCache.Add(mapIndex, lastBlockOfMap{number: number, hash: hash})
rawdb.WriteFilterMapLastBlock(db, mapIndex, number, hash)
}
// deleteEpochRows deletes the map rows from an entire epoch.
func (m *mapDatabase) deleteEpochRows(epoch uint32, stopCallback func() bool) (bool, error) {
deleteFn := func(db ethdb.KeyValueStore, hashScheme bool, stopCb func(bool) bool) error {
first := m.params.mapRowIndex(m.params.firstEpochMap(epoch), 0)
afterLast := m.params.mapRowIndex(m.params.firstEpochMap(epoch+1), 0)
return rawdb.DeleteFilterMapRows(db, common.NewRange(first, afterLast-first), hashScheme, stopCb)
}
action := fmt.Sprintf("Deleting epoch #%d", epoch)
switch err := m.safeDeleteWithLogs(deleteFn, action, stopCallback); err {
case nil:
return true, nil
case rawdb.ErrDeleteRangeInterrupted:
return false, nil
default:
return false, err
}
}
// reset deletes the entire log index database. Note that deleteMapRange should
// be used first in order to prevent trying to use a partially deleted database
// later.
func (m *mapDatabase) reset(stopCallback func() bool) (bool, error) {
rawdb.DeleteFilterMapsRange(m.db)
if err := m.safeDeleteWithLogs(rawdb.DeleteFilterMapsDb, "Resetting log index database", stopCallback); err != rawdb.ErrDeleteRangeInterrupted {
return err == nil, err
}
return false, nil
}
// deletePointers removes the last block of map and block log value pointers
// belonging to the specified map range.
// Note that in order to determine the relevant block range, the last block
// pointer of the map before the specified map range has to be present unless
// the range starts at map 0. The last block pointer of the last map in the range
// also has to be present unless the range ends at MaxUint32-1.
func (m *mapDatabase) deletePointers(deleteMaps common.Range[uint32], stopCallback func() bool) error {
var firstBlock, afterLastBlock uint64
if deleteMaps.First() > 0 {
lb, _, err := m.getLastBlockOfMap(deleteMaps.First() - 1)
if err != nil {
return err
}
firstBlock = lb + 1
}
if deleteMaps.AfterLast() < math.MaxUint32 {
lb, _, err := m.getLastBlockOfMap(deleteMaps.Last())
if err != nil {
return err
}
afterLastBlock = lb
} else {
afterLastBlock = math.MaxUint64
}
if afterLastBlock > firstBlock {
m.lvPointerCache.Purge()
blockRange := common.NewRange[uint64](firstBlock, afterLastBlock-firstBlock) // keep last pointer
if err := rawdb.DeleteBlockLvPointers(m.db, blockRange, m.hashScheme, func(bool) bool { return stopCallback() }); err != nil {
return err
}
}
if deleteMaps.Count() > 1 {
m.lastBlockCache.Purge()
mapRange := common.NewRange[uint32](deleteMaps.First(), deleteMaps.Count()-1) // keep last pointer
if err := rawdb.DeleteFilterMapLastBlocks(m.db, mapRange, m.hashScheme, func(bool) bool { return stopCallback() }); err != nil {
return err
}
}
return nil
}
// writePointers writes the pointers belonging to the give maps to the database.
func (m *mapDatabase) writePointers(writeRange common.Range[uint32], maps []*finishedMap, stopCallback func() bool) (bool, error) {
batch := m.db.NewBatch()
var (
batchCnt uint32
writeErr error
)
checkStopOrCommit := func() bool {
batchCnt++
if batchCnt >= maxWritesPerBatch {
if writeErr = batch.Write(); writeErr != nil {
return true
}
if stopCallback() {
return true
}
batch = m.db.NewBatch()
batchCnt = 0
}
return false
}
for mapIndex := range writeRange.Iter() {
fm := maps[mapIndex-writeRange.First()]
m.storeLastBlockOfMap(batch, mapIndex, fm.lastBlock.number, fm.lastBlock.hash)
if checkStopOrCommit() {
return false, writeErr
}
for i, lvPtr := range fm.blockPtrs {
m.storeBlockLvPointer(batch, fm.firstBlock()+uint64(i), lvPtr)
if checkStopOrCommit() {
return false, writeErr
}
}
}
if err := batch.Write(); err != nil {
return false, err
}
return true, nil
}
// getFilterMapRows returns a batch of filter maps rows from the same row index,
// each truncated to the length limit of the specified mapping layer.
// The function assumes that the map indices are in strictly ascending order.
// Note that the function modifies the mapIndices slice.
func (m *mapDatabase) getFilterMapRows(mapIndices []uint32, rowIndex, layerIndex uint32) ([]FilterRow, error) {
rows := make([]FilterRow, len(mapIndices))
readRows := make([]*FilterRow, len(mapIndices))
for i := range rows {
readRows[i] = &rows[i]
}
for dbLayer := range min(layerIndex+1, uint32(len(m.params.rowGroupSize))) {
if err := m.getFilterMapRowsOfDbLayer(mapIndices, rowIndex, dbLayer, readRows); err != nil {
return nil, err
}
j := 0
maxRowLength := m.params.maxRowLength[dbLayer]
for i, row := range readRows {
if uint32(len(*row)) == maxRowLength {
if j != i {
mapIndices[j] = mapIndices[i]
readRows[j] = readRows[i]
}
j++
}
}
if j == 0 {
break
}
mapIndices = mapIndices[:j]
readRows = readRows[:j]
}
return rows, nil
}
// getFilterMapRowsOfDbLayer loads the part of the given filter rows belonging to
// the specified database layer and appends it to a slice of already existing
// rows. If dbLayer > 0 then it is assumed that the data from all previous database
// layers have been appended to the filter rows.
func (m *mapDatabase) getFilterMapRowsOfDbLayer(mapIndices []uint32, rowIndex, dbLayer uint32, appendTo []*FilterRow) error {
var ptr int
for ptr < len(mapIndices) {
var (
groupIndex = m.params.mapGroupIndex(mapIndices[ptr], dbLayer)
groupLength = 1
)
for ptr+groupLength < len(mapIndices) && m.params.mapGroupIndex(mapIndices[ptr+groupLength], dbLayer) == groupIndex {
groupLength++
}
if err := m.getFilterMapRowsOfDbLayerGroup(mapIndices[ptr:ptr+groupLength], rowIndex, dbLayer, appendTo[ptr:ptr+groupLength]); err != nil {
return err
}
ptr += groupLength
}
return nil
}
// getFilterMapRowsOfDbLayerGroup loads the part of the given filter rows belonging to
// the specified database layer and appends it to a slice of already existing
// rows. The map indices should be located in the same row group according to the
// group size at the given database layer.
func (m *mapDatabase) getFilterMapRowsOfDbLayerGroup(mapIndices []uint32, rowIndex, dbLayer uint32, appendTo []*FilterRow) error {
var (
groupSize = m.params.rowGroupSize[dbLayer]
groupIndex = m.params.mapGroupIndex(mapIndices[0], dbLayer)
mapRowIndex = m.params.mapRowIndex(groupIndex, rowIndex)
)
if groupSize == 1 {
row, err := rawdb.ReadFilterMapSingleRow(m.db, mapRowIndex, dbLayer, m.params.logMapWidth)
if err != nil {
return fmt.Errorf("failed to retrieve row %d of row %d layer %d: %v", groupIndex, rowIndex, dbLayer, err)
}
*appendTo[0] = append(*appendTo[0], FilterRow(row)...)
return nil
}
rows, err := rawdb.ReadFilterMapRowGroup(m.db, mapRowIndex, dbLayer, groupSize, m.params.logMapWidth)
if err != nil {
return fmt.Errorf("failed to retrieve row group %d of row %d layer %d: %v", groupIndex, rowIndex, dbLayer, err)
}
for i, mapIndex := range mapIndices {
if m.params.mapGroupIndex(mapIndex, dbLayer) != groupIndex {
return fmt.Errorf("maps are not in the same base row group, index: %d, group: %d", mapIndex, groupIndex)
}
*appendTo[i] = append(*appendTo[i], FilterRow(rows[m.params.mapGroupOffset(mapIndex, dbLayer)])...)
}
return nil
}
// getFilterMap returns the filter map at the specified index.
func (m *mapDatabase) getFilterMap(mapIndex uint32) (*finishedMap, error) {
fm := new(finishedMap)
lastBlock, lbHash, err := m.getLastBlockOfMap(mapIndex)
if err != nil {
return nil, fmt.Errorf("failed to retrieve last block of map %d: %v", mapIndex, err)
}
fm.lastBlock = lastBlockOfMap{number: lastBlock, hash: lbHash}
var firstBlock uint64
if mapIndex > 0 {
plb, _, err := m.getLastBlockOfMap(mapIndex - 1)
if err != nil {
return nil, fmt.Errorf("failed to retrieve last block of map %d: %v", mapIndex-1, err)
}
firstBlock = plb + 1
}
fm.blockPtrs = make([]uint64, lastBlock+1-firstBlock)
for i := range fm.blockPtrs {
blockPtr, err := m.getBlockLvPointer(firstBlock + uint64(i))
if err != nil {
return nil, fmt.Errorf("failed to retrieve block pointer %d: %v", firstBlock+uint64(i), err)
}
fm.blockPtrs[i] = blockPtr
}
for rowIndex := range m.params.mapHeight {
var mi [1]uint32
mi[0] = mapIndex
row, err := m.getFilterMapRows(mi[:], rowIndex, uint32(len(m.params.rowGroupSize)-1))
if err != nil {
return nil, fmt.Errorf("failed to retrieve row %d of map %d: %v", rowIndex, mapIndex, err)
}
fm.rowData = append(fm.rowData, row[0]...)
fm.rowPtrs = append(fm.rowPtrs, uint16(len(fm.rowData)))
}
return fm, nil
}
// writeMapRows updates a section of the filter map range, writing a continuous
// range of new map data and/or removing dirty data. writeRange and deleteRange
// can and should overlap if dirty data is being overwritten with new map data.
// writeRange alone should only be specified where the database is known to be
// empty.
// keepEmptyRange is optional, it signals that a certain range is known to be
// empty in the database and should also stay so. It should not overlap with the
// other two ranges. If these three ranges together cover an entire row group
// then the entire group can be updated without reading its previous value,
// thereby speeding up the write process significantly.
// Note that writeMapRows only updates the filter map rows and does not touch
// the belonging pointers.
func (m *mapDatabase) writeMapRows(writeRange, deleteRange, keepEmptyRange common.Range[uint32], maps []*finishedMap, stopCallback func() bool) (bool, error) {
if !writeRange.Intersection(keepEmptyRange).IsEmpty() || !deleteRange.Intersection(keepEmptyRange).IsEmpty() {
panic("invalid writeMapRows map ranges")
}
writePattern := m.makeWritePattern(writeRange, deleteRange, keepEmptyRange)
batch := m.db.NewBatch()
rowsPerBatch := uint32(max(maxWritesPerBatch/len(writePattern), 1))
var (
batchCnt uint32
writeErr error
)
checkStopOrCommit := func() bool {
batchCnt++
if batchCnt >= rowsPerBatch {
if writeErr = batch.Write(); writeErr != nil {
return true
}
if stopCallback() {
return true
}
batch = m.db.NewBatch()
batchCnt = 0
}
return false
}
for rowIndex := range m.params.mapHeight {
if err := m.writeRowUpdates(batch, writePattern, writeRange, maps, rowIndex); err != nil {
return false, err
}
if checkStopOrCommit() {
return false, writeErr
}
}
if err := batch.Write(); err != nil {
return false, err
}
return true, nil
}
type writePatterItem struct {
mapIndex, dbLayer uint32
writeRange, deleteRange, keepEmptyRange common.Range[uint32]
}
// makeWritePattern pre-generates the order of operations that need to be repeated
// for every row.
func (m *mapDatabase) makeWritePattern(writeRange, deleteRange, keepEmptyRange common.Range[uint32]) (writePattern []writePatterItem) {
for dbLayer, groupSize := range m.params.rowGroupSize {
updateRange := deleteRange.Union(writeRange)
firstGroup, lastGroup := updateRange.First()/groupSize, updateRange.Last()/groupSize
for i := firstGroup; i <= lastGroup; i++ {
groupRange := common.NewRange[uint32](i*groupSize, groupSize)
writePattern = append(writePattern, writePatterItem{
mapIndex: i * groupSize,
dbLayer: uint32(dbLayer),
writeRange: writeRange.Intersection(groupRange),
deleteRange: deleteRange.Intersection(groupRange),
keepEmptyRange: keepEmptyRange.Intersection(groupRange),
})
}
}
sort.Slice(writePattern, func(i, j int) bool {
return writePattern[i].mapIndex < writePattern[j].mapIndex ||
(writePattern[i].mapIndex == writePattern[j].mapIndex && writePattern[i].dbLayer < writePattern[j].dbLayer)
})
return
}
// writeRowUpdates performs the filter row update operation on a single row.
func (m *mapDatabase) writeRowUpdates(batch ethdb.Batch, writePattern []writePatterItem, writeRange common.Range[uint32], maps []*finishedMap, rowIndex uint32) error {
for _, w := range writePattern {
if groupSize := m.params.rowGroupSize[w.dbLayer]; groupSize == 1 {
var row FilterRow
if writeRange.Includes(w.mapIndex) {
row = maps[w.mapIndex-writeRange.First()].getRow(rowIndex, m.params.maxRowLength[w.dbLayer])
}
var from uint32
if w.dbLayer > 0 {
from = m.params.maxRowLength[w.dbLayer-1]
}
if uint32(len(row)) > from {
row = row[from:]
} else {
row = nil
}
if len(row) > 0 || !w.deleteRange.IsEmpty() {
rawdb.WriteFilterMapSingleRow(batch, m.params.mapRowIndex(w.mapIndex, rowIndex), w.dbLayer, row, m.params.logMapWidth)
}
} else {
rows := make([][]uint32, groupSize)
writeGroup := !w.deleteRange.IsEmpty()
if w.writeRange.Count()+w.deleteRange.Count()-w.writeRange.Intersection(w.deleteRange).Count()+w.keepEmptyRange.Count() < groupSize {
oldRows, err := rawdb.ReadFilterMapRowGroup(m.db, m.params.mapRowIndex(w.mapIndex, rowIndex), w.dbLayer, groupSize, m.params.logMapWidth)
m.testReadRows = true
if err != nil {
return err
}
for i := range groupSize {
if mapIndex := w.mapIndex + i; !w.writeRange.Includes(mapIndex) && !w.deleteRange.Includes(mapIndex) && !w.keepEmptyRange.Includes(mapIndex) {
rows[i] = oldRows[i]
}
}
}
var from uint32
if w.dbLayer > 0 {
from = m.params.maxRowLength[w.dbLayer-1]
}
to := m.params.maxRowLength[w.dbLayer]
for i := range groupSize {
if writeRange.Includes(w.mapIndex + i) {
if row := maps[w.mapIndex+i-writeRange.First()].getRow(rowIndex, to); uint32(len(row)) > from {
rows[i] = row[from:]
writeGroup = true
}
}
}
if writeGroup {
rawdb.WriteFilterMapRowGroup(batch, m.params.mapRowIndex(w.mapIndex, rowIndex), w.dbLayer, rows, m.params.logMapWidth)
}
}
}
return nil
}
// safeDeleteWithLogs is a wrapper for a function that performs a safe range
// delete operation using rawdb.SafeDeleteRange. It emits log messages if the
// process takes long enough to call the stop callback.
func (m *mapDatabase) safeDeleteWithLogs(deleteFn func(db ethdb.KeyValueStore, hashScheme bool, stopCb func(bool) bool) error, action string, stopCb func() bool) error {
var (
start = time.Now()
logPrinted bool
lastLogPrinted = start
)
switch err := deleteFn(m.db, m.hashScheme, func(deleted bool) bool {
if deleted && !logPrinted || time.Since(lastLogPrinted) > time.Second*10 {
log.Info(action+" in progress...", "elapsed", common.PrettyDuration(time.Since(start)))
logPrinted, lastLogPrinted = true, time.Now()
}
return stopCb()
}); {
case err == nil:
if logPrinted {
log.Info(action+" finished", "elapsed", common.PrettyDuration(time.Since(start)))
}
return nil
case errors.Is(err, rawdb.ErrDeleteRangeInterrupted):
log.Warn(action+" interrupted", "elapsed", common.PrettyDuration(time.Since(start)))
return err
default:
log.Error(action+" failed", "error", err)
return err
}
}
// removeBloomBits removes old bloom bits data from the database.
func (m *mapDatabase) removeBloomBits(stopCb func() bool) {
m.safeDeleteWithLogs(rawdb.DeleteBloomBitsDb, "Removing old bloom bits database", stopCb)
}
// storeEpochCheckpoint writes a single checkpoint (known epoch boundary) to the
// database.
func (m *mapDatabase) storeEpochCheckpoint(epoch uint32, cp epochCheckpoint) {
m.storeLastBlockOfMap(m.db, m.params.lastEpochMap(epoch), cp.BlockNumber, cp.BlockHash)
m.storeBlockLvPointer(m.db, cp.BlockNumber, cp.FirstIndex)
}
// storeCheckpointList writes a list of checkpoints to the database.
func (m *mapDatabase) storeCheckpointList(firstEpoch uint32, cpList checkpointList) {
for i, cp := range cpList {
m.storeEpochCheckpoint(firstEpoch+uint32(i), cp)
}
}

79
core/filtermaps/map_database_test.go Normal file
View file

@ -0,0 +1,79 @@
// Copyright 2025 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package filtermaps
import (
"math"
"testing"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/ethdb/memorydb"
)
var testParams = Params{
logMapHeight: 8,
logMapWidth: 24,
logMapsPerEpoch: 6,
logValuesPerMap: 8,
logMappingFrequency: []uint{6, 4, 2, 0},
maxRowLength: []uint32{4, 16, 64, 256},
rowGroupSize: []uint32{16, 4, 1, 1},
}
func testStop() bool { return false }
func TestMapDatabase(t *testing.T) {
params := testParams
params.sanitize()
db := memorydb.New()
mapDb := newMapDatabase(&params, db, false)
reader := mapReader{
getFilterMapRows: mapDb.getFilterMapRows,
getFilterMap: mapDb.getFilterMap,
getBlockLvPointer: mapDb.getBlockLvPointer,
getLastBlockOfMap: mapDb.getLastBlockOfMap,
}
// initialize database with checkpoints
maps := generateTestMaps(&params, nil, 0x200)
cpList := generateTestCheckpoints(&params, maps)
mapDb.storeCheckpointList(0, cpList)
// add new maps to the head
maps = generateTestMaps(&params, maps, 0x50)
mapDb.writeMapRows(common.NewRange[uint32](0x200, 0x50), common.Range[uint32]{}, common.NewRange[uint32](0x250, math.MaxUint32-0x250), maps[0x200:], testStop)
mapDb.writePointers(common.NewRange[uint32](0x200, 0x50), maps[0x200:], testStop)
testMapReader(t, "mapDatabase test #1", &params, reader, cpList, maps[0x200:])
// backfill previous epoch with a single write
mapDb.writeMapRows(common.NewRange[uint32](0x1c0, 0x40), common.Range[uint32]{}, common.Range[uint32]{}, maps[0x1c0:0x200], testStop)
mapDb.writePointers(common.NewRange[uint32](0x1c0, 0x40), maps[0x1c0:0x200], testStop)
testMapReader(t, "mapDatabase test #2", &params, reader, cpList[:7], maps[0x1c0:])
// backfill previous epoch in two steps
mapDb.writeMapRows(common.NewRange[uint32](0x180, 0x10), common.Range[uint32]{}, common.NewRange[uint32](0x190, 0x30), maps[0x180:0x190], testStop)
mapDb.writePointers(common.NewRange[uint32](0x180, 0x10), maps[0x180:0x190], testStop)
mapDb.writeMapRows(common.NewRange[uint32](0x190, 0x30), common.Range[uint32]{}, common.Range[uint32]{}, maps[0x190:0x1c0], testStop)
mapDb.writePointers(common.NewRange[uint32](0x190, 0x30), maps[0x190:0x1c0], testStop)
testMapReader(t, "mapDatabase test #3", &params, reader, cpList[:6], maps[0x180:])
// add new maps while reorging some existing ones
maps = generateTestMaps(&params, maps[:0x230], 0x30)
mapDb.writeMapRows(common.NewRange[uint32](0x230, 0x30), common.NewRange[uint32](0x230, 0x30), common.NewRange[uint32](0x260, math.MaxUint32-0x260), maps[0x230:], testStop)
mapDb.deletePointers(common.NewRange[uint32](0x230, math.MaxUint32-0x230), testStop)
mapDb.writePointers(common.NewRange[uint32](0x230, 0x30), maps[0x230:], testStop)
testMapReader(t, "mapDatabase test #4", &params, reader, cpList[:6], maps[0x180:])
// unindex tail epoch
mapDb.writeMapRows(common.Range[uint32]{}, common.NewRange[uint32](0x180, 0x40), common.Range[uint32]{}, nil, testStop)
mapDb.deletePointers(common.NewRange[uint32](0x180, 0x40), testStop)
testMapReader(t, "mapDatabase test #5", &params, reader, cpList[:7], maps[0x1c0:])
}

815
core/filtermaps/map_renderer.go
View file

@ -1,815 +0,0 @@
// Copyright 2024 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package filtermaps
import (
"errors"
"fmt"
"math"
"slices"
"sort"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/lru"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/log"
)
const (
maxMapsPerBatch = 32 // maximum number of maps rendered in memory
valuesPerCallback = 1024 // log values processed per event process callback
cachedRowMappings = 10000 // log value to row mappings cached during rendering
// Number of rows written to db in a single batch.
// The map renderer splits up writes like this to ensure that regular
// block processing latency is not affected by large batch writes.
rowsPerBatch = 1024
)
var (
errChainUpdate = errors.New("rendered section of chain updated")
)
// mapRenderer represents a process that renders filter maps in a specified
// range according to the actual targetView.
type mapRenderer struct {
f *FilterMaps
renderBefore uint32
currentMap *renderedMap
finishedMaps map[uint32]*renderedMap
finished common.Range[uint32]
iterator *logIterator
}
// renderedMap represents a single filter map that is being rendered in memory.
type renderedMap struct {
filterMap filterMap
mapIndex uint32
lastBlock uint64
lastBlockId common.Hash
blockLvPtrs []uint64 // start pointers of blocks starting in this map; last one is lastBlock
finished bool // iterator finished; all values rendered
headDelimiter uint64 // if finished then points to the future block delimiter of the head block
}
// firstBlock returns the first block number that starts in the given map.
func (r *renderedMap) firstBlock() uint64 {
return r.lastBlock + 1 - uint64(len(r.blockLvPtrs))
}
// renderMapsBefore creates a mapRenderer that renders the log index until the
// specified map index boundary, starting from the latest available starting
// point that is consistent with the current targetView.
// The renderer ensures that filterMapsRange, indexedView and the actual map
// data are always consistent with each other. If renderBefore is greater than
// the latest existing rendered map then indexedView is updated to targetView,
// otherwise it is checked that the rendered range is consistent with both
// views.
func (f *FilterMaps) renderMapsBefore(renderBefore uint32) (*mapRenderer, error) {
nextMap, startBlock, startLvPtr, err := f.lastCanonicalMapBoundaryBefore(renderBefore)
if err != nil {
return nil, err
}
if snapshot := f.lastCanonicalSnapshotOfMap(nextMap); snapshot != nil {
return f.renderMapsFromSnapshot(snapshot)
}
if nextMap >= renderBefore {
return nil, nil
}
return f.renderMapsFromMapBoundary(nextMap, renderBefore, startBlock, startLvPtr)
}
// renderMapsFromSnapshot creates a mapRenderer that starts rendering from a
// snapshot made at a block boundary.
func (f *FilterMaps) renderMapsFromSnapshot(cp *renderedMap) (*mapRenderer, error) {
f.testSnapshotUsed = true
iter, err := f.newLogIteratorFromBlockDelimiter(cp.lastBlock, cp.headDelimiter)
if err != nil {
return nil, fmt.Errorf("failed to create log iterator from block delimiter %d: %v", cp.lastBlock, err)
}
return &mapRenderer{
f: f,
currentMap: &renderedMap{
filterMap: cp.filterMap.fullCopy(),
mapIndex: cp.mapIndex,
lastBlock: cp.lastBlock,
blockLvPtrs: slices.Clone(cp.blockLvPtrs),
},
finishedMaps: make(map[uint32]*renderedMap),
finished: common.NewRange(cp.mapIndex, 0),
renderBefore: math.MaxUint32,
iterator: iter,
}, nil
}
// renderMapsFromMapBoundary creates a mapRenderer that starts rendering at a
// map boundary.
func (f *FilterMaps) renderMapsFromMapBoundary(firstMap, renderBefore uint32, startBlock, startLvPtr uint64) (*mapRenderer, error) {
iter, err := f.newLogIteratorFromMapBoundary(firstMap, startBlock, startLvPtr)
if err != nil {
return nil, fmt.Errorf("failed to create log iterator from map boundary %d: %v", firstMap, err)
}
return &mapRenderer{
f: f,
currentMap: &renderedMap{
filterMap: f.emptyFilterMap(),
mapIndex: firstMap,
lastBlock: iter.blockNumber,
},
finishedMaps: make(map[uint32]*renderedMap),
finished: common.NewRange(firstMap, 0),
renderBefore: renderBefore,
iterator: iter,
}, nil
}
// lastCanonicalSnapshotOfMap returns the latest cached snapshot of the given map
// that is also consistent with the current targetView.
func (f *FilterMaps) lastCanonicalSnapshotOfMap(mapIndex uint32) *renderedMap {
var best *renderedMap
for _, blockNumber := range f.renderSnapshots.Keys() {
if cp, _ := f.renderSnapshots.Get(blockNumber); cp != nil && blockNumber < f.indexedRange.blocks.AfterLast() &&
blockNumber <= f.indexedView.HeadNumber() && f.indexedView.BlockId(blockNumber) == cp.lastBlockId &&
blockNumber <= f.targetView.HeadNumber() && f.targetView.BlockId(blockNumber) == cp.lastBlockId &&
cp.mapIndex == mapIndex && (best == nil || blockNumber > best.lastBlock) {
best = cp
}
}
return best
}
// lastCanonicalMapBoundaryBefore returns the latest map boundary before the
// specified map index that matches the current targetView. This can either
// be a checkpoint (hardcoded or left from a previously unindexed tail epoch)
// or the boundary of a currently rendered map.
// Along with the next map index where the rendering can be started, the number
// and starting log value pointer of the last block is also returned.
func (f *FilterMaps) lastCanonicalMapBoundaryBefore(renderBefore uint32) (nextMap uint32, startBlock, startLvPtr uint64, err error) {
if !f.indexedRange.initialized {
return 0, 0, 0, nil
}
mapIndex := renderBefore
for {
var ok bool
if mapIndex, ok = f.lastMapBoundaryBefore(mapIndex); !ok {
return 0, 0, 0, nil
}
lastBlock, lastBlockId, err := f.getLastBlockOfMap(mapIndex)
if err != nil {
return 0, 0, 0, fmt.Errorf("failed to retrieve last block of reverse iterated map %d: %v", mapIndex, err)
}
if (f.indexedRange.headIndexed && mapIndex >= f.indexedRange.maps.Last()) ||
lastBlock >= f.targetView.HeadNumber() || lastBlockId != f.targetView.BlockId(lastBlock) {
continue // map is not full or inconsistent with targetView; roll back
}
lvPtr, err := f.getBlockLvPointer(lastBlock)
if err != nil {
return 0, 0, 0, fmt.Errorf("failed to retrieve log value pointer of last canonical boundary block %d: %v", lastBlock, err)
}
return mapIndex + 1, lastBlock, lvPtr, nil
}
}
// lastMapBoundaryBefore returns the latest map boundary before the specified
// map index.
func (f *FilterMaps) lastMapBoundaryBefore(renderBefore uint32) (uint32, bool) {
if !f.indexedRange.initialized || f.indexedRange.maps.AfterLast() == 0 || renderBefore == 0 {
return 0, false
}
afterLastFullMap := f.indexedRange.maps.AfterLast()
if afterLastFullMap > 0 && f.indexedRange.headIndexed {
afterLastFullMap-- // last map is not full
}
firstRendered := min(renderBefore-1, afterLastFullMap)
if firstRendered == 0 {
return 0, false
}
if firstRendered >= f.indexedRange.maps.First() {
return firstRendered - 1, true
}
if firstRendered+f.mapsPerEpoch > f.indexedRange.maps.First() {
firstRendered = min(firstRendered, f.indexedRange.maps.First()-f.mapsPerEpoch+f.indexedRange.tailPartialEpoch)
} else {
firstRendered = (firstRendered >> f.logMapsPerEpoch) << f.logMapsPerEpoch
}
if firstRendered == 0 {
return 0, false
}
return firstRendered - 1, true
}
// emptyFilterMap returns an empty filter map.
func (f *FilterMaps) emptyFilterMap() filterMap {
return make(filterMap, f.mapHeight)
}
// loadHeadSnapshot loads the last rendered map from the database and creates
// a snapshot.
func (f *FilterMaps) loadHeadSnapshot() error {
fm, err := f.getFilterMap(f.indexedRange.maps.Last())
if err != nil {
return fmt.Errorf("failed to load head snapshot map %d: %v", f.indexedRange.maps.Last(), err)
}
lastBlock, _, err := f.getLastBlockOfMap(f.indexedRange.maps.Last())
if err != nil {
return fmt.Errorf("failed to retrieve last block of head snapshot map %d: %v", f.indexedRange.maps.Last(), err)
}
var firstBlock uint64
if f.indexedRange.maps.AfterLast() > 1 {
prevLastBlock, _, err := f.getLastBlockOfMap(f.indexedRange.maps.Last() - 1)
if err != nil {
return fmt.Errorf("failed to retrieve last block of map %d before head snapshot: %v", f.indexedRange.maps.Last()-1, err)
}
firstBlock = prevLastBlock + 1
}
lvPtrs := make([]uint64, lastBlock+1-firstBlock)
for i := range lvPtrs {
lvPtrs[i], err = f.getBlockLvPointer(firstBlock + uint64(i))
if err != nil {
return fmt.Errorf("failed to retrieve log value pointer of head snapshot block %d: %v", firstBlock+uint64(i), err)
}
}
f.renderSnapshots.Add(f.indexedRange.blocks.Last(), &renderedMap{
filterMap: fm.fullCopy(),
mapIndex: f.indexedRange.maps.Last(),
lastBlock: f.indexedRange.blocks.Last(),
lastBlockId: f.indexedView.BlockId(f.indexedRange.blocks.Last()),
blockLvPtrs: lvPtrs,
finished: true,
headDelimiter: f.indexedRange.headDelimiter,
})
return nil
}
// makeSnapshot creates a snapshot of the current state of the rendered map.
func (r *mapRenderer) makeSnapshot() {
if r.iterator.blockNumber != r.currentMap.lastBlock || r.iterator.chainView != r.f.targetView {
panic("iterator state inconsistent with current rendered map")
}
r.f.renderSnapshots.Add(r.currentMap.lastBlock, &renderedMap{
filterMap: r.currentMap.filterMap.fastCopy(),
mapIndex: r.currentMap.mapIndex,
lastBlock: r.currentMap.lastBlock,
lastBlockId: r.iterator.chainView.BlockId(r.currentMap.lastBlock),
blockLvPtrs: r.currentMap.blockLvPtrs,
finished: true,
headDelimiter: r.iterator.lvIndex,
})
}
// run does the actual map rendering. It periodically calls the stopCb callback
// and if it returns true the process is interrupted an can be resumed later
// by calling run again. The writeCb callback is called after new maps have
// been written to disk and the index range has been updated accordingly.
func (r *mapRenderer) run(stopCb func() bool, writeCb func()) (bool, error) {
for {
if done, err := r.renderCurrentMap(stopCb); !done {
return done, err // stopped or failed
}
// map finished
r.finishedMaps[r.currentMap.mapIndex] = r.currentMap
r.finished.SetLast(r.finished.AfterLast())
if len(r.finishedMaps) >= maxMapsPerBatch || r.f.mapGroupOffset(r.finished.AfterLast()) == 0 {
if err := r.writeFinishedMaps(stopCb); err != nil {
return false, err
}
writeCb()
}
if r.finished.AfterLast() == r.renderBefore || r.iterator.finished {
if err := r.writeFinishedMaps(stopCb); err != nil {
return false, err
}
writeCb()
return true, nil
}
r.currentMap = &renderedMap{
filterMap: r.f.emptyFilterMap(),
mapIndex: r.finished.AfterLast(),
}
}
}
// renderCurrentMap renders a single map.
func (r *mapRenderer) renderCurrentMap(stopCb func() bool) (bool, error) {
var (
totalTime time.Duration
logValuesProcessed, blocksProcessed int64
)
start := time.Now()
if !r.iterator.updateChainView(r.f.targetView) {
return false, errChainUpdate
}
var waitCnt int
if r.iterator.lvIndex == 0 {
r.currentMap.blockLvPtrs = []uint64{0}
}
type lvPos struct{ rowIndex, layerIndex uint32 }
rowMappingCache := lru.NewCache[common.Hash, lvPos](cachedRowMappings)
defer rowMappingCache.Purge()
for r.iterator.lvIndex < uint64(r.currentMap.mapIndex+1)<<r.f.logValuesPerMap && !r.iterator.finished {
waitCnt++
if waitCnt >= valuesPerCallback {
totalTime += time.Since(start)
if stopCb() {
return false, nil
}
start = time.Now()
if !r.iterator.updateChainView(r.f.targetView) {
return false, errChainUpdate
}
waitCnt = 0
}
if logValue := r.iterator.getValueHash(); logValue != (common.Hash{}) {
lvp, cached := rowMappingCache.Get(logValue)
if !cached {
lvp = lvPos{rowIndex: r.f.rowIndex(r.currentMap.mapIndex, 0, logValue)}
}
for uint32(len(r.currentMap.filterMap[lvp.rowIndex])) >= r.f.maxRowLength(lvp.layerIndex) {
lvp.layerIndex++
lvp.rowIndex = r.f.rowIndex(r.currentMap.mapIndex, lvp.layerIndex, logValue)
cached = false
}
r.currentMap.filterMap[lvp.rowIndex] = append(r.currentMap.filterMap[lvp.rowIndex], r.f.columnIndex(r.iterator.lvIndex, &logValue))
if !cached {
rowMappingCache.Add(logValue, lvp)
}
}
if err := r.iterator.next(); err != nil {
return false, fmt.Errorf("failed to advance log iterator at %d while rendering map %d: %v", r.iterator.lvIndex, r.currentMap.mapIndex, err)
}
if !r.iterator.skipToBoundary {
logValuesProcessed++
r.currentMap.lastBlock = r.iterator.blockNumber
if r.iterator.blockStart {
blocksProcessed++
r.currentMap.blockLvPtrs = append(r.currentMap.blockLvPtrs, r.iterator.lvIndex)
}
if !r.f.testDisableSnapshots && r.renderBefore >= r.f.indexedRange.maps.AfterLast() &&
(r.iterator.delimiter || r.iterator.finished) {
r.makeSnapshot()
}
}
}
if r.iterator.finished {
r.currentMap.finished = true
r.currentMap.headDelimiter = r.iterator.lvIndex
}
r.currentMap.lastBlockId = r.f.targetView.BlockId(r.currentMap.lastBlock)
totalTime += time.Since(start)
mapRenderTimer.Update(totalTime)
mapLogValueMeter.Mark(logValuesProcessed)
mapBlockMeter.Mark(blocksProcessed)
return true, nil
}
// writeFinishedMaps writes rendered maps to the database and updates
// filterMapsRange and indexedView accordingly.
func (r *mapRenderer) writeFinishedMaps(pauseCb func() bool) error {
var totalTime time.Duration
start := time.Now()
if len(r.finishedMaps) == 0 {
return nil
}
r.f.indexLock.Lock()
defer r.f.indexLock.Unlock()
oldRange := r.f.indexedRange
tempRange, err := r.getTempRange()
if err != nil {
return fmt.Errorf("failed to get temporary rendered range: %v", err)
}
newRange, err := r.getUpdatedRange()
if err != nil {
return fmt.Errorf("failed to get updated rendered range: %v", err)
}
renderedView := r.f.targetView // pauseCb callback might still change targetView while writing finished maps
batch := r.f.db.NewBatch()
var writeCnt int
checkWriteCnt := func() {
writeCnt++
if writeCnt == rowsPerBatch {
writeCnt = 0
if err := batch.Write(); err != nil {
log.Crit("Error writing log index update batch", "error", err)
}
// do not exit while in partially written state but do allow processing
// events and pausing while block processing is in progress
r.f.indexLock.Unlock()
totalTime += time.Since(start)
pauseCb()
start = time.Now()
r.f.indexLock.Lock()
batch = r.f.db.NewBatch()
}
}
if tempRange != r.f.indexedRange {
r.f.setRange(batch, r.f.indexedView, tempRange, true)
}
// add or update filter rows
for rowIndex := uint32(0); rowIndex < r.f.mapHeight; rowIndex++ {
var (
mapIndices []uint32
rows []FilterRow
)
for mapIndex := range r.finished.Iter() {
row := r.finishedMaps[mapIndex].filterMap[rowIndex]
if fm, _ := r.f.filterMapCache.Get(mapIndex); fm != nil && row.Equal(fm[rowIndex]) {
continue
}
mapIndices = append(mapIndices, mapIndex)
rows = append(rows, row)
}
if newRange.maps.AfterLast() == r.finished.AfterLast() { // head updated; remove future entries
for mapIndex := r.finished.AfterLast(); mapIndex < oldRange.maps.AfterLast(); mapIndex++ {
if fm, _ := r.f.filterMapCache.Get(mapIndex); fm != nil && len(fm[rowIndex]) == 0 {
continue
}
mapIndices = append(mapIndices, mapIndex)
rows = append(rows, nil)
}
}
if err := r.f.storeFilterMapRows(batch, mapIndices, rowIndex, rows); err != nil {
return fmt.Errorf("failed to store filter maps %v row %d: %v", mapIndices, rowIndex, err)
}
checkWriteCnt()
}
// update filter map cache
if newRange.maps.AfterLast() == r.finished.AfterLast() {
// head updated; cache new head maps and remove future entries
for mapIndex := range r.finished.Iter() {
r.f.filterMapCache.Add(mapIndex, r.finishedMaps[mapIndex].filterMap)
}
for mapIndex := r.finished.AfterLast(); mapIndex < oldRange.maps.AfterLast(); mapIndex++ {
r.f.filterMapCache.Remove(mapIndex)
}
} else {
// head not updated; do not cache maps during tail rendering because we
// need head maps to be available in the cache
for mapIndex := range r.finished.Iter() {
r.f.filterMapCache.Remove(mapIndex)
}
}
var blockNumber uint64
if r.finished.First() > 0 {
// in order to always ensure continuous block pointers, initialize
// blockNumber based on the last block of the previous map, then verify
// against the first block associated with each rendered map
lastBlock, _, err := r.f.getLastBlockOfMap(r.finished.First() - 1)
if err != nil {
return fmt.Errorf("failed to get last block of previous map %d: %v", r.finished.First()-1, err)
}
blockNumber = lastBlock + 1
}
// add or update block pointers
for mapIndex := range r.finished.Iter() {
renderedMap := r.finishedMaps[mapIndex]
if blockNumber != renderedMap.firstBlock() {
return fmt.Errorf("non-continuous block numbers in rendered map %d (next expected: %d first rendered: %d)", mapIndex, blockNumber, renderedMap.firstBlock())
}
r.f.storeLastBlockOfMap(batch, mapIndex, renderedMap.lastBlock, renderedMap.lastBlockId)
checkWriteCnt()
for _, lvPtr := range renderedMap.blockLvPtrs {
r.f.storeBlockLvPointer(batch, blockNumber, lvPtr)
checkWriteCnt()
blockNumber++
}
}
if newRange.maps.AfterLast() == r.finished.AfterLast() { // head updated; remove future entries
for mapIndex := r.finished.AfterLast(); mapIndex < oldRange.maps.AfterLast(); mapIndex++ {
r.f.deleteLastBlockOfMap(batch, mapIndex)
checkWriteCnt()
}
for ; blockNumber < oldRange.blocks.AfterLast(); blockNumber++ {
r.f.deleteBlockLvPointer(batch, blockNumber)
checkWriteCnt()
}
}
r.finishedMaps = make(map[uint32]*renderedMap)
r.finished.SetFirst(r.finished.AfterLast())
r.f.setRange(batch, renderedView, newRange, false)
if err := batch.Write(); err != nil {
log.Crit("Error writing log index update batch", "error", err)
}
totalTime += time.Since(start)
mapWriteTimer.Update(totalTime)
return nil
}
// getTempRange returns a temporary filterMapsRange that is committed to the
// database while the newly rendered maps are partially written. Writing all
// processed maps in a single database batch would be a serious hit on db
// performance so instead safety is ensured by first reverting the valid map
// range to the unchanged region until all new map data is committed.
func (r *mapRenderer) getTempRange() (filterMapsRange, error) {
tempRange := r.f.indexedRange
if err := tempRange.addRenderedRange(r.finished.First(), r.finished.First(), r.renderBefore, r.f.mapsPerEpoch); err != nil {
return filterMapsRange{}, fmt.Errorf("failed to update temporary rendered range: %v", err)
}
if tempRange.maps.First() != r.f.indexedRange.maps.First() {
// first rendered map changed; update first indexed block
if tempRange.maps.First() > 0 {
firstBlock, _, err := r.f.getLastBlockOfMap(tempRange.maps.First() - 1)
if err != nil {
return filterMapsRange{}, fmt.Errorf("failed to retrieve last block of map %d before temporary range: %v", tempRange.maps.First()-1, err)
}
tempRange.blocks.SetFirst(firstBlock + 1) // firstBlock is probably partially rendered
} else {
tempRange.blocks.SetFirst(0)
}
}
if tempRange.maps.AfterLast() != r.f.indexedRange.maps.AfterLast() {
// last rendered map changed; update last indexed block
if !tempRange.maps.IsEmpty() {
lastBlock, _, err := r.f.getLastBlockOfMap(tempRange.maps.Last())
if err != nil {
return filterMapsRange{}, fmt.Errorf("failed to retrieve last block of map %d at the end of temporary range: %v", tempRange.maps.Last(), err)
}
tempRange.blocks.SetAfterLast(lastBlock) // lastBlock is probably partially rendered
} else {
tempRange.blocks.SetAfterLast(0)
}
tempRange.headIndexed = false
tempRange.headDelimiter = 0
}
return tempRange, nil
}
// getUpdatedRange returns the updated filterMapsRange after writing the newly
// rendered maps.
func (r *mapRenderer) getUpdatedRange() (filterMapsRange, error) {
// update filterMapsRange
newRange := r.f.indexedRange
if err := newRange.addRenderedRange(r.finished.First(), r.finished.AfterLast(), r.renderBefore, r.f.mapsPerEpoch); err != nil {
return filterMapsRange{}, fmt.Errorf("failed to update rendered range: %v", err)
}
if newRange.maps.First() != r.f.indexedRange.maps.First() {
// first rendered map changed; update first indexed block
if newRange.maps.First() > 0 {
firstBlock, _, err := r.f.getLastBlockOfMap(newRange.maps.First() - 1)
if err != nil {
return filterMapsRange{}, fmt.Errorf("failed to retrieve last block of map %d before rendered range: %v", newRange.maps.First()-1, err)
}
newRange.blocks.SetFirst(firstBlock + 1) // firstBlock is probably partially rendered
} else {
newRange.blocks.SetFirst(0)
}
}
if newRange.maps.AfterLast() == r.finished.AfterLast() {
// last rendered map changed; update last indexed block and head pointers
lm := r.finishedMaps[r.finished.Last()]
newRange.headIndexed = lm.finished
if lm.finished {
newRange.blocks.SetLast(r.f.targetView.HeadNumber())
if lm.lastBlock != r.f.targetView.HeadNumber() {
panic("map rendering finished but last block != head block")
}
newRange.headDelimiter = lm.headDelimiter
} else {
newRange.blocks.SetAfterLast(lm.lastBlock) // lastBlock is probably partially rendered
newRange.headDelimiter = 0
}
} else {
// last rendered map not replaced; ensure that target chain view matches
// indexed chain view on the rendered section
if lastBlock := r.finishedMaps[r.finished.Last()].lastBlock; !matchViews(r.f.indexedView, r.f.targetView, lastBlock) {
return filterMapsRange{}, errChainUpdate
}
}
return newRange, nil
}
// addRenderedRange adds the range [firstRendered, afterLastRendered) and
// removes [afterLastRendered, afterLastRemoved) from the set of rendered maps.
func (fmr *filterMapsRange) addRenderedRange(firstRendered, afterLastRendered, afterLastRemoved, mapsPerEpoch uint32) error {
if !fmr.initialized {
return errors.New("log index not initialized")
}
// Here we create a slice of endpoints for the rendered sections. There are two endpoints
// for each section: the index of the first map, and the index after the last map in the
// section. We then iterate the endpoints -- adding d values -- to determine whether the
// sections are contiguous or whether they have a gap.
type endpoint struct {
m uint32
d int
}
endpoints := []endpoint{{fmr.maps.First(), 1}, {fmr.maps.AfterLast(), -1}, {firstRendered, 1}, {afterLastRendered, -101}, {afterLastRemoved, 100}}
if fmr.tailPartialEpoch > 0 {
endpoints = append(endpoints, []endpoint{{fmr.maps.First() - mapsPerEpoch, 1}, {fmr.maps.First() - mapsPerEpoch + fmr.tailPartialEpoch, -1}}...)
}
sort.Slice(endpoints, func(i, j int) bool { return endpoints[i].m < endpoints[j].m })
var (
sum int
merged []uint32
last bool
)
for i, e := range endpoints {
sum += e.d
if i < len(endpoints)-1 && endpoints[i+1].m == e.m {
continue
}
if (sum > 0) != last {
merged = append(merged, e.m)
last = !last
}
}
switch len(merged) {
case 0:
// Initialized database, but no finished maps yet.
fmr.tailPartialEpoch = 0
fmr.maps = common.NewRange(firstRendered, 0)
case 2:
// One rendered section (no partial tail epoch).
fmr.tailPartialEpoch = 0
fmr.maps = common.NewRange(merged[0], merged[1]-merged[0])
case 4:
// Two rendered sections (with a gap).
// First section (merged[0]-merged[1]) is for the partial tail epoch,
// and it has to start exactly one epoch before the main section.
if merged[2] != merged[0]+mapsPerEpoch {
return fmt.Errorf("invalid tail partial epoch: %v", merged)
}
fmr.tailPartialEpoch = merged[1] - merged[0]
fmr.maps = common.NewRange(merged[2], merged[3]-merged[2])
default:
return fmt.Errorf("invalid number of rendered sections: %v", merged)
}
return nil
}
// logIterator iterates on the linear log value index range.
type logIterator struct {
params *Params
chainView *ChainView
blockNumber uint64
receipts types.Receipts
blockStart, delimiter, skipToBoundary, finished bool
txIndex, logIndex, topicIndex int
lvIndex uint64
}
var errUnindexedRange = errors.New("unindexed range")
// newLogIteratorFromBlockDelimiter creates a logIterator starting at the
// given block's first log value entry (the block delimiter), according to the
// current targetView.
func (f *FilterMaps) newLogIteratorFromBlockDelimiter(blockNumber, lvIndex uint64) (*logIterator, error) {
if blockNumber > f.targetView.HeadNumber() {
return nil, fmt.Errorf("iterator entry point %d after target chain head block %d", blockNumber, f.targetView.HeadNumber())
}
if !f.indexedRange.blocks.Includes(blockNumber) {
return nil, errUnindexedRange
}
finished := blockNumber == f.targetView.HeadNumber()
l := &logIterator{
chainView: f.targetView,
params: &f.Params,
blockNumber: blockNumber,
finished: finished,
delimiter: !finished,
lvIndex: lvIndex,
}
l.enforceValidState()
return l, nil
}
// newLogIteratorFromMapBoundary creates a logIterator starting at the given
// map boundary, according to the current targetView.
func (f *FilterMaps) newLogIteratorFromMapBoundary(mapIndex uint32, startBlock, startLvPtr uint64) (*logIterator, error) {
if startBlock > f.targetView.HeadNumber() {
return nil, fmt.Errorf("iterator entry point %d after target chain head block %d", startBlock, f.targetView.HeadNumber())
}
// get block receipts
receipts := f.targetView.RawReceipts(startBlock)
if receipts == nil {
return nil, fmt.Errorf("receipts not found for start block %d", startBlock)
}
// initialize iterator at block start
l := &logIterator{
chainView: f.targetView,
params: &f.Params,
blockNumber: startBlock,
receipts: receipts,
blockStart: true,
lvIndex: startLvPtr,
}
l.enforceValidState()
targetIndex := uint64(mapIndex) << f.logValuesPerMap
if l.lvIndex > targetIndex {
return nil, fmt.Errorf("log value pointer %d of last block of map is after map boundary %d", l.lvIndex, targetIndex)
}
// iterate to map boundary
for l.lvIndex < targetIndex {
if l.finished {
return nil, fmt.Errorf("iterator already finished at %d before map boundary target %d", l.lvIndex, targetIndex)
}
if err := l.next(); err != nil {
return nil, fmt.Errorf("failed to advance log iterator at %d before map boundary target %d: %v", l.lvIndex, targetIndex, err)
}
}
return l, nil
}
// updateChainView updates the iterator's chain view if it still matches the
// previous view at the current position. Returns true if successful.
func (l *logIterator) updateChainView(cv *ChainView) bool {
if !matchViews(cv, l.chainView, l.blockNumber) {
return false
}
l.chainView = cv
return true
}
// getValueHash returns the log value hash at the current position.
func (l *logIterator) getValueHash() common.Hash {
if l.delimiter || l.finished || l.skipToBoundary {
return common.Hash{}
}
log := l.receipts[l.txIndex].Logs[l.logIndex]
if l.topicIndex == 0 {
return addressValue(log.Address)
}
return topicValue(log.Topics[l.topicIndex-1])
}
// next moves the iterator to the next log value index.
func (l *logIterator) next() error {
if l.skipToBoundary {
l.lvIndex++
if l.lvIndex%l.params.valuesPerMap == 0 {
l.skipToBoundary = false
}
return nil
}
if l.finished {
return nil
}
if l.delimiter {
l.delimiter = false
l.blockNumber++
l.receipts = l.chainView.RawReceipts(l.blockNumber)
if l.receipts == nil {
return fmt.Errorf("receipts not found for block %d", l.blockNumber)
}
l.txIndex, l.logIndex, l.topicIndex, l.blockStart = 0, 0, 0, true
} else {
l.topicIndex++
l.blockStart = false
}
l.lvIndex++
l.enforceValidState()
return nil
}
// enforceValidState updates the internal transaction, log and topic index pointers
// to the next existing log value of the given block if necessary.
// Note that enforceValidState does not advance the log value index pointer.
func (l *logIterator) enforceValidState() {
if l.delimiter || l.finished || l.skipToBoundary {
return
}
for ; l.txIndex < len(l.receipts); l.txIndex++ {
receipt := l.receipts[l.txIndex]
for ; l.logIndex < len(receipt.Logs); l.logIndex++ {
log := receipt.Logs[l.logIndex]
if l.topicIndex == 0 && uint64(len(log.Topics)+1) > l.params.valuesPerMap-l.lvIndex%l.params.valuesPerMap {
// next log would be split by map boundary; skip to boundary
l.skipToBoundary = true
return
}
if l.topicIndex <= len(log.Topics) {
return
}
l.topicIndex = 0
}
l.logIndex = 0
}
if l.blockNumber == l.chainView.HeadNumber() {
l.finished = true
} else {
l.delimiter = true
}
}

711
core/filtermaps/map_storage.go Normal file
View file

@ -0,0 +1,711 @@
// Copyright 2025 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package filtermaps
import (
"errors"
"fmt"
"math"
"sync"
"sync/atomic"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/log"
)
var ErrOutOfRange = errors.New("pointer out of indexed range")
// mapStorage implements a filter map storage layer over mapDatabase that ensures
// efficient database usage while also providing a low latency interface to the
// indexer. It uses a memory layer over mapDatabase allowing consistently quick
// addMap and deleteMaps calls while doing the actual database updates in the
// background asynchronously.
type mapStorage struct {
params *Params
mapDb *mapDatabase
triggerCh, closeCh chan struct{}
closeWg sync.WaitGroup
mtForceWrite, mtBusy uint32
lock sync.RWMutex
initialized bool
knownEpochs uint32 // epochs initialized with last map block pointer and corresponding reverse block lv pointer
knownEpochBlocks uint64
valid, dirty rangeSet[uint32] // valid and dirty maps in database
writeInProgress, deleteInProgress rangeSet[uint32] // write cycle in progress
overlay rangeSet[uint32] // memory maps
overlayCount uint32
overlayMaps map[uint32]*finishedMap
validBlocks, overlayBlocks rangeSet[uint64]
epochTrigger rangeSet[uint32]
suspended uint32
testHookCh chan bool
}
// newMapStorage creates a new mapStorage layer over the given mapDatabase.
func newMapStorage(params *Params, mapDb *mapDatabase, testHookCh chan bool) *mapStorage {
m := &mapStorage{
params: params,
mapDb: mapDb,
triggerCh: make(chan struct{}, 1),
closeCh: make(chan struct{}),
overlayMaps: make(map[uint32]*finishedMap),
testHookCh: testHookCh,
mtForceWrite: params.rowGroupSize[0] * 9 / 8,
mtBusy: params.rowGroupSize[0] * 17 / 8,
}
if valid, dirty, knownEpochs, ok := m.mapDb.loadMapRange(); ok {
m.valid, m.dirty, m.knownEpochs, m.initialized = valid, dirty, knownEpochs, true
if knownEpochs > 0 {
if lastBlock, _, err := m.mapDb.getLastBlockOfMap(m.params.lastEpochMap(knownEpochs - 1)); err == nil {
m.knownEpochBlocks = lastBlock + 1
} else {
m.resetWithError(fmt.Sprintf("could not initialize known epoch range: %v", err))
}
}
if err := m.updateValidBlocks(); err != nil {
m.resetWithError(fmt.Sprintf("could not initialize valid block range: %v", err))
}
}
m.closeWg.Add(1)
go m.eventLoop()
return m
}
// stop stops mapStorage.
func (m *mapStorage) stop() {
close(m.closeCh)
m.closeWg.Wait()
}
// isReady returns false if there are too many memory overlay maps at the moment.
// In this case the caller should suspend the indexing process until some maps
// are written to the database.
func (m *mapStorage) isReady() bool {
m.lock.Lock()
defer m.lock.Unlock()
if m.overlayCount < m.mtBusy {
m.trigger()
return true
}
return false
}
// tailEpoch returns the first epoch of the continuous rendered range. If it is
// larger than zero then the tail renderer can start indexing the previous epoch
// if necessary. After checkpoint initialization it points right after the last
// known epoch boundary (in this case the tail epoch can be empty).
func (m *mapStorage) tailEpoch() uint32 {
m.lock.Lock()
defer m.lock.Unlock()
mapRange := m.valid.union(m.overlay)
if len(mapRange) > 0 && m.params.mapEpoch(mapRange[len(mapRange)-1].AfterLast()) >= m.knownEpochs {
return min(m.knownEpochs, m.params.mapEpoch(mapRange[len(mapRange)-1].First()+m.params.mapsPerEpoch-1))
}
return m.knownEpochs
}
// tailNumberOfEpoch returns the number of the first block that starts in the
// given epoch.
func (m *mapStorage) tailNumberOfEpoch(epoch uint32) (uint64, error) {
if epoch == 0 {
return 0, nil
}
number, _, err := m.getLastBlockOfMap(m.params.lastEpochMap(epoch - 1))
if err != nil {
return 0, err
}
return number + 1, nil
}
// lastBoundaryBefore returns the most recent map index that is less than
// or equal to the given mapIndex parameter and is either located after a stored
// map or after a known epoch boundary.
// The returned map index position may or may not contain a stored map but if it
// is empty then it is always suitable to start a rendering process.
func (m *mapStorage) lastBoundaryBefore(mapIndex uint32) uint32 {
m.lock.Lock()
defer m.lock.Unlock()
if mapIndex == 0 {
return 0
}
lastBoundary := m.params.firstEpochMap(min(m.params.mapEpoch(mapIndex), m.knownEpochs))
if m, ok := m.valid.closestLte(mapIndex - 1); ok {
lastBoundary = max(lastBoundary, m+1)
}
if m, ok := m.overlay.closestLte(mapIndex - 1); ok {
lastBoundary = max(lastBoundary, m+1)
}
return lastBoundary
}
// matchKnownEpochs returns true if the given list of checkpoints matches the
// checkpoints already stored in the database (always true with an empty database).
func (m *mapStorage) matchKnownEpochs(cpList checkpointList) bool {
m.lock.Lock()
defer m.lock.Unlock()
if m.knownEpochs == 0 || len(cpList) == 0 {
return true
}
epoch := min(m.knownEpochs, uint32(len(cpList))) - 1
number, hash, err := m.getLastBlockOfMap(m.params.lastEpochMap(epoch))
if err != nil {
m.resetWithError(fmt.Sprintf("could not read last known epoch boundary: %v", err))
return true
}
return number == cpList[epoch].BlockNumber && hash == cpList[epoch].BlockHash
}
// addKnownEpochs adds the known epoch boundaries based on the given checkpoints.
func (m *mapStorage) addKnownEpochs(cpList checkpointList) error {
m.lock.Lock()
defer m.lock.Unlock()
if uint32(len(cpList)) <= m.knownEpochs {
return errors.New("checkpoint init list has no new epochs")
}
if m.knownEpochs > 0 {
lastNumber, lastHash, err := m.mapDb.getLastBlockOfMap(m.params.lastEpochMap(m.knownEpochs - 1))
if err != nil {
return err
}
lvPointer, err := m.mapDb.getBlockLvPointer(lastNumber)
if err != nil {
return err
}
if cp := cpList[m.knownEpochs-1]; cp.BlockNumber != lastNumber || cp.BlockHash != lastHash || cp.FirstIndex != lvPointer {
return errors.New("checkpoint init list does not match known epochs")
}
}
m.mapDb.storeCheckpointList(m.knownEpochs, cpList[m.knownEpochs:])
m.knownEpochs = uint32(len(cpList))
m.knownEpochBlocks = cpList[len(cpList)-1].BlockNumber + 1
m.mapDb.storeMapRange(m.valid, m.dirty, m.knownEpochs)
return nil
}
// addMap adds a new map to the storage. If forceCommit is true then a write is
// always triggered. If it is false then a write is only triggered when a row
// group boundary is reached or if the total number of memory maps reaches a limit.
// addMap always returns right after adding the new map to the memory layer.
func (m *mapStorage) addMap(mapIndex uint32, fm *finishedMap, forceCommit bool) {
m.lock.Lock()
defer m.lock.Unlock()
if fm == nil {
panic("trying to add nil map")
}
if m.valid.includes(mapIndex) || m.overlay.includes(mapIndex) {
panic("addMap to non-empty map index")
}
epoch := m.params.mapEpoch(mapIndex)
if (epoch > m.knownEpochs || mapIndex != m.params.firstEpochMap(epoch)) &&
!m.valid.includes(mapIndex-1) && !m.overlay.includes(mapIndex-1) {
panic("addMap to map index with no known boundary")
}
mapRs := singleRangeSet[uint32](common.NewRange[uint32](mapIndex, 1))
m.overlay = m.overlay.union(mapRs)
m.writeInProgress = m.writeInProgress.exclude(mapRs)
m.overlayMaps[mapIndex] = fm
m.updateOverlayBlocks()
if forceCommit || (mapIndex+1)%m.params.rowGroupSize[0] == 0 {
m.epochTrigger = m.epochTrigger.union(singleRangeSet[uint32](common.NewRange[uint32](epoch, 1)))
m.trigger()
}
}
// deleteMaps deletes the given map range. Note that similarly to addMap, it only
// performs memory operations before returning. The deleted map range is marked
// dirty, then later the database update goroutine will actually delete or
// overwrite the dirty maps.
func (m *mapStorage) deleteMaps(maps common.Range[uint32]) {
m.lock.Lock()
defer m.lock.Unlock()
dr := singleRangeSet[uint32](maps)
for i := range dr.intersection(m.overlay).iter() {
delete(m.overlayMaps, i)
}
m.writeInProgress = m.writeInProgress.exclude(dr)
knownEpochs := m.knownEpochs
if maps.Includes(m.params.firstEpochMap(knownEpochs)) {
knownEpochs = m.params.mapEpoch(maps.First())
}
if err := m.updateRange(m.valid.exclude(dr), m.dirty.union(dr.intersection(m.valid)), m.overlay.exclude(dr), knownEpochs); err != nil {
m.resetWithError(fmt.Sprintf("could not revert valid block range: %v", err))
}
m.trigger()
}
// suspendOrResume suspends or resumes the background database update operations.
func (m *mapStorage) suspendOrResume(suspend bool) {
var suspendU32 uint32
if suspend {
suspendU32 = 1
}
old := atomic.SwapUint32(&m.suspended, suspendU32)
if suspendU32 < old {
m.trigger()
}
}
// eventLoop is the main event loop of the database update operations.
func (m *mapStorage) eventLoop() {
defer m.closeWg.Done()
var stopped bool
selectEvent := func(blocking bool) {
if m.testHookCh != nil {
select {
case <-m.closeCh:
stopped = true
return
case m.testHookCh <- blocking:
}
}
if blocking {
select {
case <-m.closeCh:
stopped = true
return
case <-m.triggerCh:
}
} else {
select {
case <-m.closeCh:
stopped = true
return
case <-m.triggerCh:
default:
}
}
if m.testHookCh != nil {
select {
case <-m.closeCh:
stopped = true
return
case <-m.testHookCh:
}
}
}
stopCallback := func() bool {
selectEvent(atomic.LoadUint32(&m.suspended) == 1)
return stopped
}
for !stopped {
if !m.initialized {
if done, _ := m.mapDb.reset(stopCallback); !done {
return // node stopped before cleaning old database
}
m.lock.Lock()
m.initialized = true
m.lock.Unlock()
}
more, err := m.doWriteCycle(stopCallback)
if err != nil {
m.resetWithError(fmt.Sprintf("write cycle failed: %v", err))
continue
}
selectEvent(!more && !stopped) // wait for next event if no changes done
}
}
// trigger ensures that the database update cycle will restart if it was in a
// waiting state.
func (m *mapStorage) trigger() {
select {
case m.triggerCh <- struct{}{}:
default:
}
}
// getBlockLvPointer returns the starting log value index where the log values
// generated by the given block are located.
func (m *mapStorage) getBlockLvPointer(blockNumber uint64) (uint64, error) {
m.lock.RLock()
defer m.lock.RUnlock()
if m.overlayBlocks.includes(blockNumber) {
for _, fm := range m.overlayMaps {
if fm.blocks().Includes(blockNumber) {
return fm.blockPtrs[blockNumber-fm.firstBlock()], nil
}
}
return 0, errors.New("memory overlay block pointer not found")
}
if blockNumber < m.knownEpochBlocks || m.validBlocks.includes(blockNumber) {
return m.mapDb.getBlockLvPointer(blockNumber)
}
return 0, ErrOutOfRange
}
// getLastBlockOfMap returns the number and hash of the block that generated the
// last log value entry of the given map.
func (m *mapStorage) getLastBlockOfMap(mapIndex uint32) (uint64, common.Hash, error) {
m.lock.RLock()
defer m.lock.RUnlock()
if m.overlay.includes(mapIndex) {
fm := m.overlayMaps[mapIndex]
if fm == nil {
return 0, common.Hash{}, errors.New("memory overlay map not found")
}
return fm.lastBlock.number, fm.lastBlock.hash, nil
}
if mapIndex < m.params.firstEpochMap(m.knownEpochs) || m.valid.includes(mapIndex) || m.valid.includes(mapIndex+1) {
return m.mapDb.getLastBlockOfMap(mapIndex)
}
return 0, common.Hash{}, ErrOutOfRange
}
// getFilterMapRows returns a batch of filter maps rows from the same row index,
// each truncated to the length limit of the specified mapping layer.
// The function assumes that the map indices are in strictly ascending order.
func (m *mapStorage) getFilterMapRows(mapIndices []uint32, rowIndex, layerIndex uint32) ([]FilterRow, error) {
m.lock.RLock()
defer m.lock.RUnlock()
rows := make([]FilterRow, len(mapIndices))
dbMaps := make([]uint32, 0, len(mapIndices))
for i, mapIndex := range mapIndices {
if m.overlay.includes(mapIndex) {
fm := m.overlayMaps[mapIndex]
if fm == nil {
return nil, errors.New("memory overlay map not found")
}
rows[i] = fm.getRow(rowIndex, m.params.getMaxRowLength(layerIndex))
} else {
dbMaps = append(dbMaps, mapIndex)
}
}
dbRows, err := m.mapDb.getFilterMapRows(dbMaps, rowIndex, layerIndex)
if err != nil {
return nil, err
}
var j int
for i, row := range rows {
if row == nil { // zero length row is represented as zero length slice
rows[i] = dbRows[j]
j++
}
}
if j != len(dbMaps) {
panic("rows length mismatch")
}
return rows, nil
}
// getFilterMap returns the filter map at the specified index.
func (m *mapStorage) getFilterMap(mapIndex uint32) (*finishedMap, error) {
m.lock.RLock()
defer m.lock.RUnlock()
if m.overlay.includes(mapIndex) {
fm := m.overlayMaps[mapIndex]
if fm == nil {
return nil, errors.New("memory overlay map not found")
}
return fm, nil
}
if m.valid.includes(mapIndex) {
return m.mapDb.getFilterMap(mapIndex)
}
return nil, nil
}
// extendDeletedPointerRange takes a map range where pointers need to be deleted
// and extends it so that on both ends there is a neighboring stored map, a
// known epoch boundary or one end of the map index range. This is required
// in order to determine the block range covered by the deleted map range.
func (m *mapStorage) extendDeletedPointerRange(deleteRange common.Range[uint32]) common.Range[uint32] {
epoch := m.params.mapEpoch(deleteRange.First())
if m.params.mapEpoch(deleteRange.Last()) != epoch {
panic("deleted map range crosses epoch boundary")
}
first := m.params.firstEpochMap(min(m.knownEpochs, epoch))
if deleteRange.First() > 0 {
if c, ok := m.valid.closestLte(deleteRange.First() - 1); ok {
first = max(first, c+1)
}
}
afterLast := uint32(math.MaxUint32)
if epoch < m.knownEpochs {
afterLast = m.params.firstEpochMap(epoch + 1)
}
if fa, ok := m.valid.closestGte(deleteRange.AfterLast()); ok {
afterLast = min(afterLast, fa)
}
return common.NewRange[uint32](first, afterLast-first)
}
// selectEpochTriggeredWrite selects the next epoch to update if one of the epochs
// in the canSelect list has been triggered by addMap.
func (m *mapStorage) selectEpochTriggeredWrite(canSelect []uint32) (uint32, bool) {
for _, epoch := range canSelect {
if !m.epochTrigger.includes(epoch) {
continue
}
m.epochTrigger = m.epochTrigger.exclude(singleRangeSet[uint32](common.NewRange[uint32](epoch, 1)))
epochRange := common.NewRange[uint32](m.params.firstEpochMap(epoch), m.params.mapsPerEpoch)
if len(m.overlay.intersection(singleRangeSet[uint32](epochRange))) > 0 {
return epoch, true
}
}
return 0, false
}
// selectEpochForcedWrite selects the next epoch to update if the total number
// of memory maps has reached a threshold.
func (m *mapStorage) selectEpochForcedWrite(canSelect []uint32) (uint32, bool) {
if m.overlayCount < m.mtForceWrite {
return 0, false
}
var best, count uint32
for _, epoch := range canSelect {
epochRange := common.NewRange[uint32](m.params.firstEpochMap(epoch), m.params.mapsPerEpoch)
if c := m.overlay.intersection(singleRangeSet[uint32](epochRange)).count(); c > count {
best, count = epoch, c
}
}
if count == 0 {
return 0, false
}
return best, true
}
// mapToEpochRange returns the set of epochs that are either fully or partially
// covered by the specified mapRange.
func (m *mapStorage) mapToEpochRange(mapRange rangeSet[uint32]) rangeSet[uint32] {
vb := make(rangeBoundaries[uint32], 0, len(mapRange)*2)
for _, r := range mapRange {
first := m.params.mapEpoch(r.First())
last := m.params.mapEpoch(r.Last())
vb.add(common.NewRange[uint32](first, last+1-first), 1)
}
return vb.makeSet(1)
}
// selectEpochDeleteOnly selects an epoch to update where there are no memory
// maps to write, only dirty maps to clean up.
func (m *mapStorage) selectEpochDeleteOnly() (uint32, bool) {
epochs := m.mapToEpochRange(m.dirty).exclude(m.mapToEpochRange(m.overlay))
if len(epochs) == 0 {
return 0, false
}
return epochs[0].First(), true
}
// doWriteCycle selects an epoch where there are memory overlay maps and/or
// dirty maps in the underlying database, cleans up dirty data and writes the
// new maps to the database.
// If an epoch has been successfully updated then the function returns (true, nil).
// In this case a new write cycle can be attempted immediately until there is
// nothing more left to do.
// If there was nothing to do or if stopCallback aborted the operation then the
// function returns (false, nil). In case of an error it returns (false, err).
// Note that if the operation is aborted or failed then the partially updated
// maps are left as marked dirty and the database remains consistent.
func (m *mapStorage) doWriteCycle(stopCallback func() bool) (bool, error) {
m.lock.Lock()
defer m.lock.Unlock()
// always operate on a single epoch
var canSelect []uint32
if len(m.valid) == 0 {
canSelect = []uint32{m.knownEpochs}
} else {
lastValid := m.valid[len(m.valid)-1]
tailEpoch := m.params.mapEpoch(lastValid.First())
headEpoch := m.params.mapEpoch(lastValid.AfterLast())
if tailEpoch > 0 {
canSelect = []uint32{headEpoch, tailEpoch - 1}
} else {
canSelect = []uint32{headEpoch}
}
}
epoch, ok := m.selectEpochTriggeredWrite(canSelect)
if !ok {
epoch, ok = m.selectEpochForcedWrite(canSelect)
if !ok {
epoch, ok = m.selectEpochDeleteOnly()
if !ok {
return false, nil
}
}
}
epochRange := singleRangeSet[uint32](common.NewRange[uint32](m.params.firstEpochMap(epoch), m.params.mapsPerEpoch))
writeMaps := epochRange.intersection(m.overlay).singleRange()
validInEpoch := epochRange.intersection(m.valid).singleRange()
dirtyInEpoch := epochRange.intersection(m.dirty).singleRange()
keepEmptyFrom := max(m.params.firstEpochMap(epoch), writeMaps.AfterLast(), validInEpoch.AfterLast(), dirtyInEpoch.AfterLast())
keepEmptyInEpoch := common.NewRange[uint32](keepEmptyFrom, m.params.firstEpochMap(epoch+1)-keepEmptyFrom)
// delete old pointers
if !dirtyInEpoch.IsEmpty() {
m.mapDb.deletePointers(m.extendDeletedPointerRange(dirtyInEpoch), stopCallback)
}
if writeMaps.IsEmpty() && validInEpoch.IsEmpty() {
// delete map rows of entire epoch if nothing to write or keep
m.lock.Unlock()
done, err := m.mapDb.deleteEpochRows(epoch, stopCallback)
m.lock.Lock()
if done {
if err := m.updateRange(m.valid, m.dirty.exclude(epochRange), m.overlay, m.knownEpochs); err != nil {
return false, err
}
}
return done, err
}
maps := make([]*finishedMap, writeMaps.Count())
for i := range writeMaps.Iter() {
maps[i-writeMaps.First()] = m.overlayMaps[i]
}
// temporarily mark newly written maps as dirty (replaced/deleted maps are already dirty)
writeInProgress := singleRangeSet[uint32](writeMaps)
deleteInProgress := singleRangeSet[uint32](dirtyInEpoch).exclude(writeInProgress)
if err := m.updateRange(m.valid, m.dirty.union(writeInProgress), m.overlay, m.knownEpochs); err != nil {
return false, err
}
m.writeInProgress = writeInProgress
m.deleteInProgress = deleteInProgress
m.lock.Unlock()
// write/overwrite map rows and delete dirty map data, write new pointers
done, err := m.mapDb.writeMapRows(writeMaps, dirtyInEpoch, keepEmptyInEpoch, maps, stopCallback)
if done {
done, err = m.mapDb.writePointers(writeMaps, maps, stopCallback)
}
m.lock.Lock()
writeInProgress = m.writeInProgress
m.writeInProgress, m.deleteInProgress = nil, nil
if !done {
return false, err
}
for mapIndex := range writeInProgress.iter() {
delete(m.overlayMaps, mapIndex)
}
knownEpochs := m.knownEpochs
if len(writeInProgress) > 0 {
knownEpochs = max(knownEpochs, m.params.mapEpoch(writeInProgress[len(writeInProgress)-1].Last()))
}
if err := m.updateRange(m.valid.union(writeInProgress), m.dirty.exclude(writeInProgress.union(deleteInProgress)), m.overlay.exclude(writeInProgress), knownEpochs); err != nil {
return false, err
}
return true, nil
}
// updateRange updates the stored valid, dirty and memory overlay map ranges and
// the number of known epoch boundaries. It also stores these ranges in the
// database and determines the corresponding block ranges.
func (m *mapStorage) updateRange(valid, dirty, overlay rangeSet[uint32], knownEpochs uint32) error {
if !valid.equal(m.valid) {
m.valid = valid
if err := m.updateValidBlocks(); err != nil {
return err
}
}
if knownEpochs != m.knownEpochs {
m.knownEpochs = knownEpochs
if err := m.updateKnownEpochBlocks(); err != nil {
return err
}
}
m.dirty = dirty
if !overlay.equal(m.overlay) {
m.overlay = overlay
m.updateOverlayBlocks()
}
m.mapDb.storeMapRange(valid, dirty, m.knownEpochs)
return nil
}
// resetWithError resets an invalid log index database after an unrecoverable error.
func (m *mapStorage) resetWithError(errStr string) {
log.Error("Resetting invalid log index database", "error", errStr)
m.uninitialize()
}
// uninitialize resets the map storage and returns to its non-initialized state.
func (m *mapStorage) uninitialize() {
m.valid, m.dirty, m.overlay, m.knownEpochs, m.initialized = nil, nil, nil, 0, false
m.mapDb.deleteMapRange()
m.trigger()
}
// updateKnownEpochBlocks determines the block number belonging to the last known
// epoch boundary.
func (m *mapStorage) updateKnownEpochBlocks() error {
if m.knownEpochs > 0 {
if lastBlock, _, err := m.mapDb.getLastBlockOfMap(m.params.lastEpochMap(m.knownEpochs - 1)); err == nil {
m.knownEpochBlocks = lastBlock + 1
} else {
return err
}
} else {
m.knownEpochBlocks = 0
}
return nil
}
// updateValidBlocks determines the the set of blocks where the starting log value
// pointer points into the valid map set.
func (m *mapStorage) updateValidBlocks() error {
if len(m.valid) > 2 || (len(m.valid) == 2 && m.valid[0].Count() >= m.params.mapsPerEpoch) {
panic("invalid mapStorage.valid")
return errors.New("invalid mapStorage.valid range set")
}
vb := make(rangeBoundaries[uint64], 0, len(m.valid)*2)
for _, vr := range m.valid {
var first uint64
if vr.First() > 0 {
lb, _, err := m.mapDb.getLastBlockOfMap(vr.First() - 1)
if err != nil {
return err
}
first = lb + 1
}
last, _, err := m.mapDb.getLastBlockOfMap(vr.Last())
if err != nil {
return err
}
vb.add(common.NewRange[uint64](first, last+1-first), 1)
}
m.validBlocks = vb.makeSet(1)
return nil
}
// updateOverlayBlocks determines the the set of blocks where the starting log
// value pointer points into the memory overlay map set.
func (m *mapStorage) updateOverlayBlocks() {
ob := make(rangeBoundaries[uint64], 0, len(m.overlay)*2)
for _, or := range m.overlay {
first := m.overlayMaps[or.First()].firstBlock()
last := m.overlayMaps[or.Last()].lastBlock.number
ob.add(common.NewRange[uint64](first, last+1-first), 1)
}
m.overlayBlocks = ob.makeSet(1)
m.overlayCount = m.overlay.count()
}

150
core/filtermaps/map_storage_test.go Normal file
View file

@ -0,0 +1,150 @@
// Copyright 2025 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package filtermaps
import (
"math"
"testing"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/ethdb/memorydb"
)
func TestMapStorage(t *testing.T) {
params := testParams
params.sanitize()
db := memorydb.New()
mapDb := newMapDatabase(&params, db, false)
ms := newMapStorage(&params, mapDb, make(chan bool))
<-ms.testHookCh
defer ms.stop()
reader := mapReader{
getFilterMapRows: ms.getFilterMapRows,
getFilterMap: ms.getFilterMap,
getBlockLvPointer: ms.getBlockLvPointer,
getLastBlockOfMap: ms.getLastBlockOfMap,
}
waitCycle := func() bool {
ms.testHookCh <- true
return !<-ms.testHookCh
}
expKnownEpochs := func(testCase int, exp uint32) {
if ms.knownEpochs != exp {
t.Fatalf("Invalid known epochs number in test case #%d (expected %d, got %d)", testCase, exp, ms.knownEpochs)
}
}
expTailEpoch := func(testCase int, exp uint32) {
if ms.tailEpoch() != exp {
t.Fatalf("Invalid tail epoch in test case #%d (expected %d, got %d)", testCase, exp, ms.tailEpoch())
}
}
expReadRows := func(testCase int, exp bool) {
if mapDb.testReadRows != exp {
t.Fatalf("Invalid read rows flag in test case #%d (expected %v, got %v)", testCase, exp, mapDb.testReadRows)
}
mapDb.testReadRows = false
}
// initialize database with checkpoints
maps := generateTestMaps(&params, nil, 0x200)
cpList := generateTestCheckpoints(&params, maps)
ms.addKnownEpochs(cpList)
expTailEpoch(0, 8)
expKnownEpochs(0, 8)
// add new maps to the head
maps = generateTestMaps(&params, maps, 0x50)
for m := uint32(0x200); m < 0x250; m++ {
ms.addMap(m, maps[m], true)
}
for waitCycle() {
testMapReader(t, "mapStorage test #1", &params, reader, cpList, maps[0x200:])
}
expReadRows(1, false)
expTailEpoch(1, 8)
expKnownEpochs(1, 9)
// backfill previous epoch with a single write
for m := uint32(0x1c0); m < 0x200; m++ {
ms.addMap(m, maps[m], false)
}
for waitCycle() {
testMapReader(t, "mapDatabase test #2", &params, reader, cpList[:7], maps[0x1c0:])
}
expReadRows(2, false)
expTailEpoch(2, 7)
expKnownEpochs(2, 9)
// backfill previous epoch in two steps
for m := uint32(0x180); m < 0x192; m++ {
ms.addMap(m, maps[m], true)
}
for waitCycle() {
}
expReadRows(3, false)
expTailEpoch(3, 7)
for m := uint32(0x192); m < 0x1c0; m++ {
ms.addMap(m, maps[m], false)
}
for waitCycle() {
testMapReader(t, "mapDatabase test #3", &params, reader, cpList[:6], maps[0x180:])
}
expReadRows(3, true)
expTailEpoch(3, 6)
expKnownEpochs(3, 9)
// add new maps while reorging some existing ones
maps = generateTestMaps(&params, maps[:0x234], 0x30)
ms.deleteMaps(common.NewRange[uint32](0x234, math.MaxUint32-0x234))
expKnownEpochs(4, 8)
for m := uint32(0x234); m < 0x264; m++ {
ms.addMap(m, maps[m], true)
}
for waitCycle() {
testMapReader(t, "mapDatabase test #4", &params, reader, cpList[:6], maps[0x180:])
}
expReadRows(4, true)
expTailEpoch(4, 6)
expKnownEpochs(4, 9)
// unindex tail epoch
ms.deleteMaps(common.NewRange[uint32](0x180, 0x40))
expTailEpoch(5, 7)
for waitCycle() {
testMapReader(t, "mapDatabase test #5", &params, reader, cpList[:7], maps[0x1c0:])
}
expReadRows(5, false)
expTailEpoch(5, 7)
expKnownEpochs(5, 9)
// remove head maps
maps = maps[:0x253]
ms.deleteMaps(common.NewRange[uint32](0x253, math.MaxUint32-0x253))
for waitCycle() {
testMapReader(t, "mapDatabase test #6", &params, reader, cpList[:7], maps[0x1c0:])
}
expReadRows(6, true)
expTailEpoch(6, 7)
expKnownEpochs(6, 9)
// add maps until epoch boundary and check known epochs increase
maps = generateTestMaps(&params, maps, 0x30)
for m := uint32(0x253); m < 0x283; m++ {
ms.addMap(m, maps[m], true)
}
expTailEpoch(7, 7)
expKnownEpochs(7, 9)
for waitCycle() {
testMapReader(t, "mapDatabase test #7", &params, reader, cpList[:7], maps[0x1c0:])
}
expReadRows(7, true)
expTailEpoch(7, 7)
expKnownEpochs(7, 10)
}

171
core/filtermaps/matcher.go
View file

@ -28,6 +28,18 @@ import (
"github.com/ethereum/go-ethereum/common/mclock"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
)
var (
matchRequestTimer = metrics.NewRegisteredTimer("filtermaps/match/requesttime", nil) // processing time a matching request in a single epoch
matchEpochTimer = metrics.NewRegisteredTimer("filtermaps/match/epochtime", nil) // total processing time a matching request
matchBaseRowAccessMeter = metrics.NewRegisteredMeter("filtermaps/match/baserowaccess", nil) // number of accessed rows on layer 0
matchBaseRowSizeMeter = metrics.NewRegisteredMeter("filtermaps/match/baserowsize", nil) // size of accessed rows on layer 0
matchExtRowAccessMeter = metrics.NewRegisteredMeter("filtermaps/match/extrowaccess", nil) // number of accessed rows on higher layers
matchExtRowSizeMeter = metrics.NewRegisteredMeter("filtermaps/match/extrowsize", nil) // size of accessed rows on higher layers
matchLogLookup = metrics.NewRegisteredMeter("filtermaps/match/loglookup", nil) // number of log lookups based on potential matches
matchAllMeter = metrics.NewRegisteredMeter("filtermaps/match/matchall", nil) // number of requests returned with ErrMatchAll
)
const doRuntimeStats = false
@ -37,32 +49,32 @@ const doRuntimeStats = false
// would actually be slower than reverting to legacy filter.
var ErrMatchAll = errors.New("match all patterns not supported")
// MatcherBackend defines the functions required for searching in the log index
// data structure. It is currently implemented by FilterMapsMatcherBackend but
// once EIP-7745 is implemented and active, these functions can also be trustlessly
// served by a remote prover.
type MatcherBackend interface {
GetParams() *Params
GetBlockLvPointer(ctx context.Context, blockNumber uint64) (uint64, error)
GetFilterMapRows(ctx context.Context, mapIndices []uint32, rowIndex uint32, baseLayerOnly bool) ([]FilterRow, error)
GetLogByLvIndex(ctx context.Context, lvIndex uint64) (*types.Log, error)
SyncLogIndex(ctx context.Context) (SyncRange, error)
Close()
type LogPosition struct {
BlockNumber, startPtr, logPtr, valuesPerMap uint64
}
// SyncRange is returned by MatcherBackend.SyncLogIndex. It contains the latest
// chain head, the indexed range that is currently consistent with the chain
// and the valid range that has not been changed and has been consistent with
// all states of the chain since the previous SyncLogIndex or the creation of
// the matcher backend.
type SyncRange struct {
IndexedView *ChainView
// block range where the index has not changed since the last matcher sync
// and therefore the set of matches found in this region is guaranteed to
// be valid and complete.
ValidBlocks common.Range[uint64]
// block range indexed according to the given chain head.
IndexedBlocks common.Range[uint64]
func (l *LogPosition) GetLog(receipts types.Receipts) (*types.Log, error) {
ptr := l.startPtr
for _, receipt := range receipts {
for _, log := range receipt.Logs {
step := uint64(len(log.Topics) + 1)
subPtr := ptr % l.valuesPerMap
if subPtr+step > l.valuesPerMap {
ptr += l.valuesPerMap - subPtr
}
if ptr == l.logPtr {
return log, nil
}
ptr += step
if ptr > l.logPtr {
return nil, nil // log position is in block range but there is no log there (false positive)
}
}
}
if ptr == l.logPtr {
return nil, nil // points to the block delimiter (false positive)
}
return nil, fmt.Errorf("invalid log position: block #%d start pointer %d log pointer %d", l.BlockNumber, l.startPtr, l.logPtr)
}
// GetPotentialMatches returns a list of logs that are potential matches for the
@ -70,14 +82,14 @@ type SyncRange struct {
// missing or changed during the search process then the resulting logs belonging
// to that block range might be missing or incorrect.
// Also note that the returned list may contain false positives.
func GetPotentialMatches(ctx context.Context, backend MatcherBackend, firstBlock, lastBlock uint64, addresses []common.Address, topics [][]common.Hash) ([]*types.Log, error) {
func GetPotentialMatches(ctx context.Context, backend *IndexView, firstBlock, lastBlock uint64, addresses []common.Address, topics [][]common.Hash) ([]LogPosition, error) {
params := backend.GetParams()
// find the log value index range to search
firstIndex, err := backend.GetBlockLvPointer(ctx, firstBlock)
firstIndex, err := backend.GetBlockLvPointer(firstBlock)
if err != nil {
return nil, fmt.Errorf("failed to retrieve log value pointer for first block %d: %v", firstBlock, err)
}
lastIndex, err := backend.GetBlockLvPointer(ctx, lastBlock+1)
lastIndex, err := backend.GetBlockLvPointer(lastBlock + 1)
if err != nil {
return nil, fmt.Errorf("failed to retrieve log value pointer after last block %d: %v", lastBlock, err)
}
@ -112,10 +124,11 @@ func GetPotentialMatches(ctx context.Context, backend MatcherBackend, firstBlock
matcher := newMatchSequence(params, matchers)
m := &matcherEnv{
ctx: ctx,
backend: backend,
params: params,
matcher: matcher,
firstBlock: firstBlock,
lastBlock: lastBlock,
firstIndex: firstIndex,
lastIndex: lastIndex,
firstMap: uint32(firstIndex >> params.logValuesPerMap),
@ -123,7 +136,7 @@ func GetPotentialMatches(ctx context.Context, backend MatcherBackend, firstBlock
}
start := time.Now()
res, err := m.process()
res, err := m.process(ctx)
matchRequestTimer.Update(time.Since(start))
if doRuntimeStats {
@ -142,18 +155,18 @@ func GetPotentialMatches(ctx context.Context, backend MatcherBackend, firstBlock
type matcherEnv struct {
getLogStats runtimeStats // 64 bit aligned
ctx context.Context
backend MatcherBackend
backend *IndexView
params *Params
matcher matcher
firstBlock, lastBlock uint64
firstIndex, lastIndex uint64
firstMap, lastMap uint32
}
func (m *matcherEnv) process() ([]*types.Log, error) {
func (m *matcherEnv) process(ctx context.Context) ([]LogPosition, error) {
type task struct {
epochIndex uint32
logs []*types.Log
logs []LogPosition
err error
done chan struct{}
}
@ -182,7 +195,7 @@ func (m *matcherEnv) process() ([]*types.Log, error) {
}
firstEpoch, lastEpoch := m.firstMap>>m.params.logMapsPerEpoch, m.lastMap>>m.params.logMapsPerEpoch
var logs []*types.Log
var logs []LogPosition
// startEpoch is the next task to send whenever a worker can accept it.
// waitEpoch is the next task we are waiting for to finish in order to append
// results in the correct order.
@ -214,15 +227,17 @@ func (m *matcherEnv) process() ([]*types.Log, error) {
tasks[waitEpoch] = &task{epochIndex: waitEpoch, done: make(chan struct{})}
}
}
/*case <-ctx.Done(): //TODO
return nil, ctx.Err()*/
}
}
return logs, nil
}
// processEpoch returns the potentially matching logs from the given epoch.
func (m *matcherEnv) processEpoch(epochIndex uint32) ([]*types.Log, error) {
func (m *matcherEnv) processEpoch(epochIndex uint32) ([]LogPosition, error) {
start := time.Now()
var logs []*types.Log
var logs []LogPosition
// create a list of map indices to process
fm, lm := epochIndex<<m.params.logMapsPerEpoch, (epochIndex+1)<<m.params.logMapsPerEpoch-1
if fm < m.firstMap {
@ -260,22 +275,56 @@ func (m *matcherEnv) processEpoch(epochIndex uint32) ([]*types.Log, error) {
return logs, nil
}
func (m *matcherEnv) getLogPosition(lvIndex uint64) (LogPosition, error) {
// find possible block range based on map to block pointers
mapIndex := uint32(lvIndex >> m.params.logValuesPerMap)
lastBlockNumber, _, err := m.backend.GetLastBlockOfMap(mapIndex)
if err != nil {
return LogPosition{}, fmt.Errorf("failed to retrieve last block of map %d containing searched log value index %d: %v", mapIndex, lvIndex, err)
}
var firstBlockNumber uint64
if mapIndex > 0 {
firstBlockNumber, _, err = m.backend.GetLastBlockOfMap(mapIndex - 1)
if err != nil {
return LogPosition{}, fmt.Errorf("failed to retrieve last block of map %d before searched log value index %d: %v", mapIndex, lvIndex, err)
}
}
firstBlockNumber = max(firstBlockNumber, m.firstBlock)
lastBlockNumber = min(lastBlockNumber, m.lastBlock)
// find block with binary search based on block to log value index pointers
for firstBlockNumber < lastBlockNumber {
midBlockNumber := (firstBlockNumber + lastBlockNumber + 1) / 2
midLvPointer, err := m.backend.GetBlockLvPointer(midBlockNumber)
if err != nil {
return LogPosition{}, fmt.Errorf("failed to retrieve log value pointer of block %d while binary searching log value index %d: %v", midBlockNumber, lvIndex, err)
}
if lvIndex < midLvPointer {
lastBlockNumber = midBlockNumber - 1
} else {
firstBlockNumber = midBlockNumber
}
}
blockLvPointer, err := m.backend.GetBlockLvPointer(firstBlockNumber)
if err != nil {
return LogPosition{}, fmt.Errorf("failed to retrieve log value pointer of block %d containing searched log value index %d: %v", firstBlockNumber, lvIndex, err)
}
return LogPosition{BlockNumber: firstBlockNumber, startPtr: blockLvPointer, logPtr: lvIndex, valuesPerMap: m.params.valuesPerMap}, nil
}
// getLogsFromMatches returns the list of potentially matching logs located at
// the given list of matching log indices. Matches outside the firstIndex to
// lastIndex range are not returned.
func (m *matcherEnv) getLogsFromMatches(matches potentialMatches) ([]*types.Log, error) {
var logs []*types.Log
func (m *matcherEnv) getLogsFromMatches(matches potentialMatches) ([]LogPosition, error) {
var logs []LogPosition
for _, match := range matches {
if match < m.firstIndex || match > m.lastIndex {
continue
}
log, err := m.backend.GetLogByLvIndex(m.ctx, match)
log, err := m.getLogPosition(match)
if err != nil {
return logs, fmt.Errorf("failed to retrieve log at index %d: %v", match, err)
}
if log != nil {
logs = append(logs, log)
}
logs = append(logs, log)
matchLogLookup.Mark(1)
}
return logs, nil
@ -288,7 +337,7 @@ func (m *matcherEnv) getAllMatches(mapIndices []uint32) ([]potentialMatches, err
instance := m.matcher.newInstance(mapIndices)
resultsMap := make(map[uint32]potentialMatches)
for layerIndex := uint32(0); len(resultsMap) < len(mapIndices); layerIndex++ {
results, err := instance.getMatchesForLayer(m.ctx, layerIndex)
results, err := instance.getMatchesForLayer(layerIndex)
if err != nil {
return nil, err
}
@ -319,7 +368,7 @@ type matcher interface {
// a result has been returned has no effect. Exactly one matcherResult is
// returned per requested map index unless dropped.
type matcherInstance interface {
getMatchesForLayer(ctx context.Context, layerIndex uint32) ([]matcherResult, error)
getMatchesForLayer(layerIndex uint32) ([]matcherResult, error)
dropIndices(mapIndices []uint32)
}
@ -332,7 +381,7 @@ type matcherResult struct {
// singleMatcher implements matcher by returning matches for a single log value hash.
type singleMatcher struct {
backend MatcherBackend
backend *IndexView
value common.Hash
stats runtimeStats
}
@ -360,7 +409,7 @@ func (m *singleMatcher) newInstance(mapIndices []uint32) matcherInstance {
}
// getMatchesForLayer implements matcherInstance.
func (m *singleMatcherInstance) getMatchesForLayer(ctx context.Context, layerIndex uint32) (results []matcherResult, err error) {
func (m *singleMatcherInstance) getMatchesForLayer(layerIndex uint32) (results []matcherResult, err error) {
var st int
m.stats.setState(&st, stOther)
params := m.backend.GetParams()
@ -378,7 +427,7 @@ func (m *singleMatcherInstance) getMatchesForLayer(ctx context.Context, layerInd
} else {
m.stats.setState(&st, stFetchMore)
}
groupRows, err := m.backend.GetFilterMapRows(ctx, m.mapIndices[ptr:ptr+groupLength], rowIndex, layerIndex == 0)
groupRows, err := m.backend.GetFilterMapRows(m.mapIndices[ptr:ptr+groupLength], rowIndex, layerIndex)
if err != nil {
m.stats.setState(&st, stNone)
return nil, fmt.Errorf("failed to retrieve filter map %d row %d: %v", m.mapIndices[ptr], rowIndex, err)
@ -400,7 +449,7 @@ func (m *singleMatcherInstance) getMatchesForLayer(ctx context.Context, layerInd
}
m.stats.addAmount(st, int64(len(filterRow)))
filterRows = append(filterRows, filterRow)
if uint32(len(filterRow)) < params.maxRowLength(layerIndex) {
if uint32(len(filterRow)) < params.getMaxRowLength(layerIndex) {
m.stats.setState(&st, stProcess)
matches := params.potentialMatches(filterRows, mapIndex, m.value)
m.stats.addAmount(st, int64(len(matches)))
@ -416,6 +465,12 @@ func (m *singleMatcherInstance) getMatchesForLayer(ctx context.Context, layerInd
}
ptr += groupLength
}
if len(m.mapIndices) > 0 {
var r int
for _, res := range results {
r += len(res.matches)
}
}
m.cleanMapIndices()
m.stats.setState(&st, stNone)
return results, nil
@ -491,7 +546,7 @@ func (m matchAny) newInstance(mapIndices []uint32) matcherInstance {
}
// getMatchesForLayer implements matcherInstance.
func (m *matchAnyInstance) getMatchesForLayer(ctx context.Context, layerIndex uint32) (mergedResults []matcherResult, err error) {
func (m *matchAnyInstance) getMatchesForLayer(layerIndex uint32) (mergedResults []matcherResult, err error) {
if len(m.matchAny) == 0 {
// return "wild card" results (potentialMatches(nil) is interpreted as a
// potential match at every log value index of the map).
@ -504,7 +559,7 @@ func (m *matchAnyInstance) getMatchesForLayer(ctx context.Context, layerIndex ui
return mergedResults, nil
}
for i, childInstance := range m.childInstances {
results, err := childInstance.getMatchesForLayer(ctx, layerIndex)
results, err := childInstance.getMatchesForLayer(layerIndex)
if err != nil {
return nil, fmt.Errorf("failed to evaluate child matcher on layer %d: %v", layerIndex, err)
}
@ -698,19 +753,19 @@ type matchSequenceInstance struct {
}
// getMatchesForLayer implements matcherInstance.
func (m *matchSequenceInstance) getMatchesForLayer(ctx context.Context, layerIndex uint32) (matchedResults []matcherResult, err error) {
func (m *matchSequenceInstance) getMatchesForLayer(layerIndex uint32) (matchedResults []matcherResult, err error) {
// decide whether to evaluate base or next matcher first
baseFirst := m.baseFirst()
if baseFirst {
if err := m.evalBase(ctx, layerIndex); err != nil {
if err := m.evalBase(layerIndex); err != nil {
return nil, err
}
}
if err := m.evalNext(ctx, layerIndex); err != nil {
if err := m.evalNext(layerIndex); err != nil {
return nil, err
}
if !baseFirst {
if err := m.evalBase(ctx, layerIndex); err != nil {
if err := m.evalBase(layerIndex); err != nil {
return nil, err
}
}
@ -753,8 +808,8 @@ func (m *matchSequenceInstance) dropIndices(dropIndices []uint32) {
// evalBase evaluates the base child matcher and drops map indices from the
// next matcher if possible.
func (m *matchSequenceInstance) evalBase(ctx context.Context, layerIndex uint32) error {
results, err := m.baseInstance.getMatchesForLayer(ctx, layerIndex)
func (m *matchSequenceInstance) evalBase(layerIndex uint32) error {
results, err := m.baseInstance.getMatchesForLayer(layerIndex)
if err != nil {
return fmt.Errorf("failed to evaluate base matcher on layer %d: %v", layerIndex, err)
}
@ -781,8 +836,8 @@ func (m *matchSequenceInstance) evalBase(ctx context.Context, layerIndex uint32)
// evalNext evaluates the next child matcher and drops map indices from the
// base matcher if possible.
func (m *matchSequenceInstance) evalNext(ctx context.Context, layerIndex uint32) error {
results, err := m.nextInstance.getMatchesForLayer(ctx, layerIndex)
func (m *matchSequenceInstance) evalNext(layerIndex uint32) error {
results, err := m.nextInstance.getMatchesForLayer(layerIndex)
if err != nil {
return fmt.Errorf("failed to evaluate next matcher on layer %d: %v", layerIndex, err)
}

201
core/filtermaps/matcher_backend.go
View file

@ -1,201 +0,0 @@
// Copyright 2024 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package filtermaps
import (
"context"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
)
// FilterMapsMatcherBackend implements MatcherBackend.
type FilterMapsMatcherBackend struct {
f *FilterMaps
// these fields should be accessed under f.matchersLock mutex.
validBlocks common.Range[uint64]
syncCh chan SyncRange
}
// NewMatcherBackend returns a FilterMapsMatcherBackend after registering it in
// the active matcher set.
// Note that Close should always be called when the matcher is no longer used.
func (f *FilterMaps) NewMatcherBackend() *FilterMapsMatcherBackend {
f.indexLock.RLock()
f.matchersLock.Lock()
defer func() {
f.matchersLock.Unlock()
f.indexLock.RUnlock()
}()
fm := &FilterMapsMatcherBackend{f: f}
if f.indexedRange.initialized {
fm.validBlocks = f.indexedRange.blocks
}
f.matchers[fm] = struct{}{}
return fm
}
// GetParams returns the filtermaps parameters.
// GetParams implements MatcherBackend.
func (fm *FilterMapsMatcherBackend) GetParams() *Params {
return &fm.f.Params
}
// Close removes the matcher from the set of active matchers and ensures that
// any SyncLogIndex calls are cancelled.
// Close implements MatcherBackend.
func (fm *FilterMapsMatcherBackend) Close() {
fm.f.matchersLock.Lock()
defer fm.f.matchersLock.Unlock()
delete(fm.f.matchers, fm)
}
// GetFilterMapRows returns the given row of the given map. If the row is empty
// then a non-nil zero length row is returned. If baseLayerOnly is true then
// only the first baseRowLength entries of the row are guaranteed to be
// returned.
// Note that the returned slices should not be modified, they should be copied
// on write.
// GetFilterMapRows implements MatcherBackend.
func (fm *FilterMapsMatcherBackend) GetFilterMapRows(ctx context.Context, mapIndices []uint32, rowIndex uint32, baseLayerOnly bool) ([]FilterRow, error) {
return fm.f.getFilterMapRows(mapIndices, rowIndex, baseLayerOnly)
}
// GetBlockLvPointer returns the starting log value index where the log values
// generated by the given block are located. If blockNumber is beyond the last
// indexed block then the pointer will point right after this block, ensuring
// that the matcher does not fail and can return a set of results where the
// valid range is correct.
// GetBlockLvPointer implements MatcherBackend.
func (fm *FilterMapsMatcherBackend) GetBlockLvPointer(ctx context.Context, blockNumber uint64) (uint64, error) {
fm.f.indexLock.RLock()
defer fm.f.indexLock.RUnlock()
if blockNumber >= fm.f.indexedRange.blocks.AfterLast() {
if fm.f.indexedRange.headIndexed {
// return index after head block
return fm.f.indexedRange.headDelimiter + 1, nil
}
if fm.f.indexedRange.blocks.Count() > 0 {
// return index at the beginning of the last, partially indexed
// block (after the last fully indexed one)
blockNumber = fm.f.indexedRange.blocks.Last()
}
}
return fm.f.getBlockLvPointer(blockNumber)
}
// GetLogByLvIndex returns the log at the given log value index.
// Note that this function assumes that the log index structure is consistent
// with the canonical chain at the point where the given log value index points.
// If this is not the case then an invalid result may be returned or certain
// logs might not be returned at all.
// No error is returned though because of an inconsistency between the chain and
// the log index. It is the caller's responsibility to verify this consistency
// using SyncLogIndex and re-process certain blocks if necessary.
// GetLogByLvIndex implements MatcherBackend.
func (fm *FilterMapsMatcherBackend) GetLogByLvIndex(ctx context.Context, lvIndex uint64) (*types.Log, error) {
fm.f.indexLock.RLock()
defer fm.f.indexLock.RUnlock()
return fm.f.getLogByLvIndex(lvIndex)
}
// synced signals to the matcher that has triggered a synchronisation that it
// has been finished and the log index is consistent with the chain head passed
// as a parameter.
//
// Note that if the log index head was far behind the chain head then it might not
// be synced up to the given head in a single step. Still, the latest chain head
// should be passed as a parameter and the existing log index should be consistent
// with that chain.
//
// Note: acquiring the indexLock read lock is unnecessary here, as this function
// is always called within the indexLoop.
func (fm *FilterMapsMatcherBackend) synced() {
fm.f.matchersLock.Lock()
defer fm.f.matchersLock.Unlock()
indexedBlocks := fm.f.indexedRange.blocks
if !fm.f.indexedRange.headIndexed && !indexedBlocks.IsEmpty() {
indexedBlocks.SetAfterLast(indexedBlocks.Last()) // remove partially indexed last block
}
fm.syncCh <- SyncRange{
IndexedView: fm.f.indexedView,
ValidBlocks: fm.validBlocks,
IndexedBlocks: indexedBlocks,
}
fm.validBlocks = indexedBlocks
fm.syncCh = nil
}
// SyncLogIndex ensures that the log index is consistent with the current state
// of the chain and is synced up to the current head. It blocks until this state
// is achieved or the context is cancelled.
// If successful, it returns a SyncRange that contains the latest chain head,
// the indexed range that is currently consistent with the chain and the valid
// range that has not been changed and has been consistent with all states of the
// chain since the previous SyncLogIndex or the creation of the matcher backend.
func (fm *FilterMapsMatcherBackend) SyncLogIndex(ctx context.Context) (SyncRange, error) {
syncCh := make(chan SyncRange, 1)
fm.f.matchersLock.Lock()
fm.syncCh = syncCh
fm.f.matchersLock.Unlock()
select {
case fm.f.matcherSyncCh <- fm:
case <-ctx.Done():
return SyncRange{}, ctx.Err()
case <-fm.f.disabledCh:
// Note: acquiring the indexLock read lock is unnecessary here,
// as the indexer has already been terminated.
return SyncRange{IndexedView: fm.f.indexedView}, nil
}
select {
case vr := <-syncCh:
return vr, nil
case <-ctx.Done():
return SyncRange{}, ctx.Err()
case <-fm.f.disabledCh:
// Note: acquiring the indexLock read lock is unnecessary here,
// as the indexer has already been terminated.
return SyncRange{IndexedView: fm.f.indexedView}, nil
}
}
// updateMatchersValidRange iterates through active matchers and limits their
// valid range with the current indexed range. This function should be called
// whenever a part of the log index has been removed, before adding new blocks
// to it.
//
// Note: acquiring the indexLock read lock is unnecessary here, as this function
// is always called within the indexLoop.
func (f *FilterMaps) updateMatchersValidRange() {
f.matchersLock.Lock()
defer f.matchersLock.Unlock()
for fm := range f.matchers {
if !f.indexedRange.initialized {
fm.validBlocks = common.Range[uint64]{}
continue
}
fm.validBlocks = fm.validBlocks.Intersection(f.indexedRange.blocks)
}
}

2
core/filtermaps/matcher_test.go
View file

@ -16,6 +16,7 @@
package filtermaps
/*
import (
"context"
crand "crypto/rand"
@ -85,3 +86,4 @@ func TestMatcher(t *testing.T) {
}
}
}
*/

327
core/filtermaps/math.go
View file

@ -21,66 +21,59 @@ import (
"encoding/binary"
"fmt"
"hash/fnv"
"iter"
"math"
"slices"
"sort"
"github.com/ethereum/go-ethereum/common"
)
// FilterRow encodes a single row of a filter map as a list of column indices.
// Note that the values are always stored in the same order as they were added
// and if the same column index is added twice, it is also stored twice.
// Order of column indices and potential duplications do not matter when searching
// for a value but leaving the original order makes reverting to a previous state
// simpler.
type FilterRow []uint32
// Params defines the basic parameters of the log index structure.
type Params struct {
logMapHeight uint // The number of bits required to represent the map height
logMapWidth uint // The number of bits required to represent the map width
logMapsPerEpoch uint // The number of bits required to represent the number of maps per epoch
logValuesPerMap uint // The number of bits required to represent the number of log values per map
// baseRowLengthRatio represents the ratio of base row length
// to the average row length.
baseRowLengthRatio uint
// logLayerDiff defines the logarithmic growth factor (base 2) of
// the maximum row length per layer. It indicates how much the maximum
// row length increases as the layer depth increases.
//
// Specifically:
// - the row length in base layer (layer == 0) is baseRowLength
// - the row length in layer x is baseRowLength << (logLayerDiff * x)
logLayerDiff uint
logMapHeight uint // The number of bits required to represent the map height
logMapWidth uint // The number of bits required to represent the map width
logMapsPerEpoch uint // The number of bits required to represent the number of maps per epoch
logValuesPerMap uint // The number of bits required to represent the number of log values per map
logMappingFrequency []uint
maxRowLength []uint32
// These fields can be derived with the information above
mapHeight uint32 // The number of rows in the filter map
mapsPerEpoch uint32 // The number of maps in an epoch
valuesPerMap uint64 // The number of log values marked on each filter map
baseRowLength uint32 // maximum number of log values per row on layer 0
mapHeight uint32 // The number of rows in the filter map
mapsPerEpoch uint32 // The number of maps in an epoch
valuesPerMap uint64 // The number of log values marked on each filter map
// baseRowGroupSize defines the number of base row entries grouped together
// as a single database entry in the local database to optimize storage
// and retrieval efficiency.
//
// This value can be configured based on the specific implementation.
baseRowGroupSize uint32
rowGroupSize []uint32
}
// DefaultParams is the set of parameters used on mainnet.
var DefaultParams = Params{
logMapHeight: 16,
logMapWidth: 24,
logMapsPerEpoch: 10,
logValuesPerMap: 16,
baseRowGroupSize: 32,
baseRowLengthRatio: 8,
logLayerDiff: 4,
logMapHeight: 16,
logMapWidth: 24,
logMapsPerEpoch: 10,
logValuesPerMap: 16,
logMappingFrequency: []uint{10, 6, 2, 0},
maxRowLength: []uint32{8, 168, 2728, 10920},
rowGroupSize: []uint32{256, 16, 1, 1},
}
// RangeTestParams puts one log value per epoch, ensuring block exact tail unindexing for testing
var RangeTestParams = Params{
logMapHeight: 4,
logMapWidth: 24,
logMapsPerEpoch: 0,
logValuesPerMap: 0,
baseRowGroupSize: 32,
baseRowLengthRatio: 16, // baseRowLength >= 1
logLayerDiff: 4,
logMapHeight: 4,
logMapWidth: 24,
logMapsPerEpoch: 0,
logValuesPerMap: 0,
logMappingFrequency: []uint{10, 6, 2, 0},
maxRowLength: []uint32{8, 168, 2728, 10920},
rowGroupSize: []uint32{16, 4, 1, 1},
}
// deriveFields calculates the derived fields of the parameter set.
@ -88,7 +81,25 @@ func (p *Params) deriveFields() {
p.mapHeight = uint32(1) << p.logMapHeight
p.mapsPerEpoch = uint32(1) << p.logMapsPerEpoch
p.valuesPerMap = uint64(1) << p.logValuesPerMap
p.baseRowLength = uint32(p.valuesPerMap * uint64(p.baseRowLengthRatio) / uint64(p.mapHeight))
}
// mapRowIndex calculates the unified storage index where the given row of the
// given map is stored. Note that this indexing scheme is the same as the one
// proposed in EIP-7745 for tree-hashing the filter map structure and for the
// same data proximity reasons it is also suitable for database representation.
// See also:
// https://eips.ethereum.org/EIPS/eip-7745#hash-tree-structure
func (p *Params) mapRowIndex(mapIndex, rowIndex uint32) uint64 {
epochIndex, mapSubIndex := mapIndex>>p.logMapsPerEpoch, mapIndex&(p.mapsPerEpoch-1)
return (uint64(epochIndex)<<p.logMapHeight+uint64(rowIndex))<<p.logMapsPerEpoch + uint64(mapSubIndex)
}
func (p *Params) getLogMappingFrequency(layerIndex uint32) uint {
return p.logMappingFrequency[min(layerIndex, uint32(len(p.logMappingFrequency)-1))]
}
func (p *Params) getMaxRowLength(layerIndex uint32) uint32 {
return p.maxRowLength[min(layerIndex, uint32(len(p.maxRowLength)-1))]
}
// addressValue returns the log value hash of a log emitting address.
@ -115,21 +126,37 @@ func (p *Params) sanitize() error {
if p.logMapWidth%8 != 0 {
return fmt.Errorf("invalid configuration: logMapWidth (%d) must be a multiple of 8", p.logMapWidth)
}
if p.baseRowGroupSize == 0 || (p.baseRowGroupSize&(p.baseRowGroupSize-1)) != 0 {
return fmt.Errorf("invalid configuration: baseRowGroupSize (%d) must be a power of 2", p.baseRowGroupSize)
if p.logMapWidth > 32 { // column index stored as uint32
return fmt.Errorf("invalid configuration: logMapWidth (%d) should not exceed 32", p.logMapWidth)
}
if p.logMapHeight > 16 { // row index stored as uint16 in finishedMap
return fmt.Errorf("invalid configuration: logMapHeight (%d) should not exceed 32", p.logMapHeight)
}
for _, maxRowLength := range p.maxRowLength {
if maxRowLength >= 0x10000 { // index wrap-around issue in finishedMap
return fmt.Errorf("invalid configuration: maxRowLength entry (%d) should not exceed 2**16", maxRowLength)
}
}
for _, groupSize := range p.rowGroupSize {
if groupSize == 0 || (groupSize&(groupSize-1)) != 0 {
return fmt.Errorf("invalid configuration: rowGroupSize entry (%d) must be a power of 2", groupSize)
}
}
if len(p.maxRowLength) != len(p.rowGroupSize) {
return fmt.Errorf("invalid configuration: length of maxRowLength (%d) and rowGroupSize entry (%d) should be equal", len(p.maxRowLength), len(p.rowGroupSize))
}
return nil
}
// mapGroupIndex returns the start index of the base row group that contains the
// given map index. Assumes baseRowGroupSize is a power of 2.
func (p *Params) mapGroupIndex(index uint32) uint32 {
return index & ^(p.baseRowGroupSize - 1)
func (p *Params) mapGroupIndex(index, dbLayer uint32) uint32 {
return index & ^(p.rowGroupSize[dbLayer] - 1)
}
// mapGroupOffset returns the offset of the given map index within its base row group.
func (p *Params) mapGroupOffset(index uint32) uint32 {
return index & (p.baseRowGroupSize - 1)
func (p *Params) mapGroupOffset(index, dbLayer uint32) uint32 {
return index & (p.rowGroupSize[dbLayer] - 1)
}
// mapEpoch returns the epoch number that the given map index belongs to.
@ -181,38 +208,12 @@ func (p *Params) columnIndex(lvIndex uint64, logValue *common.Hash) uint32 {
return uint32(lvIndex%p.valuesPerMap)<<hashBits + (uint32(hash>>(64-hashBits)) ^ uint32(hash)>>(32-hashBits))
}
// maxRowLength returns the maximum length filter rows are populated up to when
// using the given mapping layer.
//
// A log value can be marked on the map according to a given mapping layer if
// the row mapping on that layer points to a row that has not yet reached the
// maxRowLength belonging to that layer.
//
// This means that a row that is considered full on a given layer may still be
// extended further on a higher order layer.
//
// Each value is marked on the lowest order layer possible, assuming that marks
// are added in ascending log value index order. When searching for a log value
// one should consider all layers and process corresponding rows up until the
// first one where the row mapped to the given layer is not full.
func (p *Params) maxRowLength(layerIndex uint32) uint32 {
logLayerDiff := uint(layerIndex) * p.logLayerDiff
if logLayerDiff > p.logMapsPerEpoch {
logLayerDiff = p.logMapsPerEpoch
}
return p.baseRowLength << logLayerDiff
}
// maskedMapIndex returns the index used for row mapping calculation on the
// given layer. On layer zero the mapping changes once per epoch, then the
// frequency of re-mapping increases with every new layer until it reaches
// the frequency where it is different for every mapIndex.
func (p *Params) maskedMapIndex(mapIndex, layerIndex uint32) uint32 {
logLayerDiff := uint(layerIndex) * p.logLayerDiff
if logLayerDiff > p.logMapsPerEpoch {
logLayerDiff = p.logMapsPerEpoch
}
return mapIndex & (uint32(math.MaxUint32) << (p.logMapsPerEpoch - logLayerDiff))
return mapIndex & (uint32(math.MaxUint32) << p.getLogMappingFrequency(layerIndex))
}
// potentialMatches returns the list of log value indices potentially matching
@ -229,7 +230,7 @@ func (p *Params) potentialMatches(rows []FilterRow, mapIndex uint32, logValue co
results := make(potentialMatches, 0, 8)
mapFirst := uint64(mapIndex) << p.logValuesPerMap
for i, row := range rows {
rowLen, maxLen := len(row), int(p.maxRowLength(uint32(i)))
rowLen, maxLen := len(row), int(p.getMaxRowLength(uint32(i)))
if rowLen > maxLen {
rowLen = maxLen // any additional entries are generated by another log value on a higher mapping layer
}
@ -264,3 +265,177 @@ func (p *Params) potentialMatches(rows []FilterRow, mapIndex uint32, logValue co
// indices in the filter map range are potential matches. If there are no
// potential matches in the given map's range then an empty slice should be used.
type potentialMatches []uint64
func (p potentialMatches) Len() int { return len(p) }
func (p potentialMatches) Less(i, j int) bool { return p[i] < p[j] }
func (p potentialMatches) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
type rangeSet[T uint32 | uint64] []common.Range[T]
func (a rangeSet[T]) includes(v T) bool {
for _, r := range a {
if r.Includes(v) {
return true
}
}
return false
}
func (a rangeSet[T]) closestLte(v T) (last T, found bool) {
for _, r := range a {
if r.First() > v {
return
}
if r.AfterLast() > v {
return v, true
}
last, found = r.Last(), true
}
return
}
func (a rangeSet[T]) closestGte(v T) (last T, found bool) {
for _, r := range a {
if r.First() > v {
return r.First(), true
}
if r.AfterLast() > v {
return v, true
}
}
return
}
type rangeBoundary[T uint32 | uint64] struct {
v T
d int
}
type rangeBoundaries[T uint32 | uint64] []rangeBoundary[T]
func (rb *rangeBoundaries[T]) add(r common.Range[T], d int) {
*rb = append((*rb), rangeBoundary[T]{v: r.First(), d: d}, rangeBoundary[T]{v: r.AfterLast(), d: -d})
}
func (rb rangeBoundaries[T]) makeSet(threshold int) rangeSet[T] {
res := make(rangeSet[T], 0, len(rb)/2)
sort.Slice(rb, func(i, j int) bool {
return rb[i].v < rb[j].v
})
var (
sum int
lastCmp bool
start T
)
for i, r := range rb {
sum += r.d
cmp := sum >= threshold
if cmp != lastCmp && (i == len(rb)-1 || rb[i+1].v != r.v) {
if cmp {
start = r.v
} else {
res = append(res, common.NewRange[T](start, r.v-start))
}
lastCmp = cmp
}
}
return res
}
func (a rangeSet[T]) intersection(b rangeSet[T]) rangeSet[T] {
if len(a) == 0 || len(b) == 0 {
return nil
}
rb := make(rangeBoundaries[T], 0, (len(a)+len(b))*2)
for _, r := range a {
rb.add(r, 1)
}
for _, r := range b {
rb.add(r, 1)
}
return rb.makeSet(2)
}
func (a rangeSet[T]) exclude(b rangeSet[T]) rangeSet[T] {
if len(a) == 0 {
return nil
}
if len(b) == 0 {
return a
}
rb := make(rangeBoundaries[T], 0, (len(a)+len(b))*2)
for _, r := range a {
rb.add(r, 1)
}
for _, r := range b {
rb.add(r, -1)
}
return rb.makeSet(1)
}
func (a rangeSet[T]) union(b rangeSet[T]) rangeSet[T] {
if len(a) == 0 {
return b
}
if len(b) == 0 {
return a
}
rb := make(rangeBoundaries[T], 0, (len(a)+len(b))*2)
for _, r := range a {
rb.add(r, 1)
}
for _, r := range b {
rb.add(r, 1)
}
return rb.makeSet(1)
}
// iter iterates all integers in the range set.
func (r rangeSet[T]) iter() iter.Seq[T] {
return func(yield func(T) bool) {
for _, rr := range r {
for i := range rr.Iter() {
if !yield(i) {
break
}
}
}
}
}
func (a rangeSet[T]) count() T {
var count T
for _, r := range a {
count += r.Count()
}
return count
}
func (a rangeSet[T]) singleRange() common.Range[T] {
if len(a) > 1 {
panic("singleRange called for non-continuous rangeSet")
}
if len(a) == 1 {
return a[0]
}
return common.NewRange[T](0, 0)
}
func singleRangeSet[T uint32 | uint64](r common.Range[T]) rangeSet[T] {
if r.IsEmpty() {
return nil
}
return rangeSet[T]{r}
}
func (a rangeSet[T]) equal(b rangeSet[T]) bool {
if len(a) != len(b) {
return false
}
for i, r := range a {
if b[i] != r {
return false
}
}
return true
}

2
core/filtermaps/math_test.go
View file

@ -87,7 +87,7 @@ func TestPotentialMatches(t *testing.T) {
// split up into a list of rows if longer than allowed
var rows []FilterRow
for layerIndex := uint32(0); row != nil; layerIndex++ {
maxLen := int(params.maxRowLength(layerIndex))
maxLen := int(params.getMaxRowLength(layerIndex))
if len(row) > maxLen {
rows = append(rows, row[:maxLen])
row = row[maxLen:]

171
core/filtermaps/memory_map.go Normal file
View file

@ -0,0 +1,171 @@
// Copyright 2025 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package filtermaps
import (
"slices"
"github.com/ethereum/go-ethereum/common"
)
// memoryMap is an in-memory representation of a filter map that represents rows
// as linked lists and stores entries in a single slice.
// memoryMap allows adding new elements and is used for rendering maps. Completed
// maps can be transformed to an immutable finishedMap.
type memoryMap struct {
entries []mmEntry // size = valuesPerMap
rows []mmRow // size = mapHeight
nextEntry uint32
blockPtrs []uint64
lastBlock lastBlockOfMap
}
// mmEntry is a linked list entry. The field next points to the next entry in the
// entries slice. Zero singals the end of the list (entries[0] is always the first
// entry of a list).
type mmEntry struct {
value, next uint32
}
// mmRow stores the index of the first and last entry of a linked list in the
// entries slice. If length is zero then first and last are not initialized.
// Note that length could be determined by traversing the linked list but it is
// explicitly stored because map rendering needs quick access to row length.
type mmRow struct {
first, last, length uint32
}
// newMemoryMap creates an empty memoryMap.
func (p *Params) newMemoryMap() *memoryMap {
return &memoryMap{
entries: make([]mmEntry, p.valuesPerMap),
rows: make([]mmRow, p.mapHeight),
}
}
// clone creates a copy of the map.
func (m *memoryMap) clone() *memoryMap {
return &memoryMap{
//TODO no need to clone entries if one of the maps will never be changed further.
entries: slices.Clone(m.entries),
rows: slices.Clone(m.rows),
nextEntry: m.nextEntry,
blockPtrs: slices.Clone(m.blockPtrs),
lastBlock: m.lastBlock,
}
}
func (m *memoryMap) firstBlock() uint64 {
return m.lastBlock.number + 1 - uint64(len(m.blockPtrs))
}
func (m *memoryMap) blocks() common.Range[uint64] {
l := uint64(len(m.blockPtrs))
return common.NewRange[uint64](m.lastBlock.number+1-l, l)
}
// addToRow adds a new entry to the specified row.
func (m *memoryMap) addToRow(rowIndex, value uint32) {
row := &m.rows[rowIndex]
if row.length == 0 {
row.first = m.nextEntry
} else {
m.entries[row.last].next = m.nextEntry
}
row.last = m.nextEntry
row.length++
m.entries[m.nextEntry].value = value
m.nextEntry++
}
// rowLength returns the length of a row.
func (m *memoryMap) rowLength(rowIndex uint32) uint32 {
return m.rows[rowIndex].length
}
// getRow returns a row of the map, truncated if maxLen is smaller than the actual
// row length.
func (m *memoryMap) getRow(rowIndex, maxLen uint32) FilterRow {
row := m.rows[rowIndex]
length := min(row.length, maxLen)
res := make(FilterRow, length)
next := row.first
for i := range length {
entry := m.entries[next]
res[i], next = entry.value, entry.next
}
return res
}
// finishedMap is an immutable memory representation of a single filter map.
// It is more compact and allows more efficient row lookup than memoryMap.
// Note that it assumes params.mapHeight <= 2**16 which is checked in deriveFields.
type finishedMap struct {
rowPtrs []uint16 // points to rowData index after end of row; 2**16 can wrap around to 0
rowData []uint32
blockPtrs []uint64
lastBlock lastBlockOfMap
}
// finished creates a new finishedMap from a memoryMap.
func (m *memoryMap) finished() *finishedMap {
fm := &finishedMap{
rowPtrs: make([]uint16, len(m.rows)),
rowData: make([]uint32, m.nextEntry),
blockPtrs: slices.Clone(m.blockPtrs),
lastBlock: m.lastBlock,
}
var ptr uint16
for i, row := range m.rows {
next := row.first
for range row.length {
entry := m.entries[next]
fm.rowData[ptr] = entry.value
next = entry.next
ptr++
}
fm.rowPtrs[i] = ptr
}
return fm
}
func (fm *finishedMap) firstBlock() uint64 {
return fm.lastBlock.number + 1 - uint64(len(fm.blockPtrs))
}
func (fm *finishedMap) blocks() common.Range[uint64] {
l := uint64(len(fm.blockPtrs))
return common.NewRange[uint64](fm.lastBlock.number+1-l, l)
}
// getRow returns a row of the map, truncated if maxLen is smaller than the actual
// row length. If the row has zero length then it returns a zero length slice.
func (fm *finishedMap) getRow(rowIndex, maxLen uint32) FilterRow {
var start uint16
if rowIndex > 0 {
start = fm.rowPtrs[rowIndex-1]
}
// Note: uint16 subtraction ensures correct result if
// rowPtrs[rowIndex] has wrapped around to zero. For example if there are
// exactly 2**16 entries and last row has a length of 5 then
// rowPtr[0xFFFE] == 0xFFFB and rowPtr[0xFFFF] == 0x0000 and the uint16
// subtraction is 0x0000-0xFFFB == 0x0005.
// Also note that it is guaranteed by the filter map design that no single
// row can have a length of 0x10000.
length := fm.rowPtrs[rowIndex] - start
return FilterRow(fm.rowData[start : uint32(start)+min(maxLen, uint32(length))])
}

226
core/filtermaps/test_helpers.go Normal file
View file

@ -0,0 +1,226 @@
// Copyright 2025 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package filtermaps
import (
"math"
"math/rand"
"reflect"
"slices"
"testing"
"github.com/ethereum/go-ethereum/common"
)
const (
testFilterMapRowsCount = 1000
testFilterMapsCount = 100
testPointersCount = 1000
testMaxBlocksPerMap = 4
)
type mapReader struct {
getFilterMapRows func(mapIndices []uint32, rowIndex, layerIndex uint32) ([]FilterRow, error)
getFilterMap func(mapIndex uint32) (*finishedMap, error)
getBlockLvPointer func(blockNumber uint64) (uint64, error)
getLastBlockOfMap func(mapIndex uint32) (uint64, common.Hash, error)
}
func equalRows(a, b []FilterRow) bool {
if len(a) != len(b) {
return false
}
for i := range a {
if !slices.Equal(a[i], b[i]) {
return false
}
}
return true
}
func testMapReader(t *testing.T, testCase string, params *Params, reader mapReader, knownEpochs checkpointList, maps []*finishedMap) {
mapRange := common.NewRange[uint32](uint32(len(knownEpochs))*params.mapsPerEpoch, uint32(len(maps)))
for range testFilterMapRowsCount {
rowIndex := uint32(rand.Intn(int(params.mapHeight)))
lastDbLayer := uint32(rand.Intn(len(params.maxRowLength)))
maxLen := params.maxRowLength[lastDbLayer]
mid := mapRange.First() + uint32(rand.Intn(int(mapRange.Count())))
count := uint32(rand.Intn(int(params.mapsPerEpoch)))
first := max(mid, count/2) - count/2
testRange := common.NewRange[uint32](first, count)
mapIndices := make([]uint32, 0, testRange.Count())
expResults := make([]FilterRow, 0, testRange.Count())
for mapIndex := range testRange.Iter() {
if rand.Intn(2) == 1 {
mapIndices = append(mapIndices, mapIndex)
if mapRange.Includes(mapIndex) {
expResults = append(expResults, maps[mapIndex-mapRange.First()].getRow(rowIndex, maxLen))
} else {
expResults = append(expResults, nil)
}
}
}
rows, err := reader.getFilterMapRows(mapIndices, rowIndex, lastDbLayer)
if err != nil {
t.Fatalf("Test case %s: error reading %d map indices in range %d <= i < %d: %v", testCase, len(mapIndices), testRange.First(), testRange.AfterLast(), err)
} else if !equalRows(rows, expResults) {
t.Fatalf("Test case %s: incorrect results when reading %d map indices in range %d <= i < %d", testCase, len(mapIndices), testRange.First(), testRange.AfterLast())
}
}
for range testFilterMapsCount {
mapIndex := mapRange.First() + uint32(rand.Intn(int(mapRange.Count())))
expResults := maps[mapIndex-mapRange.First()]
fm, err := reader.getFilterMap(mapIndex)
if err != nil {
t.Fatalf("Test case %s: error reading map %d: %v", testCase, mapIndex, err)
} else if !reflect.DeepEqual(fm, expResults) {
t.Fatalf("Test case %s: incorrect results when reading map %d", testCase, mapIndex)
}
}
testPointers := func(mapIndex uint32, expLastBlock lastBlockOfMap, blockNumber, expBlockPtr uint64) {
lastBlock, lbHash, err := reader.getLastBlockOfMap(mapIndex)
if err != nil {
t.Fatalf("Test case %s: error reading last block of map %d: %v", testCase, mapIndex, err)
} else if (lastBlockOfMap{number: lastBlock, hash: lbHash}) != expLastBlock {
t.Fatalf("Test case %s: incorrect results when reading last block of map %d (expected %v, got %v)", testCase, mapIndex, expLastBlock, lastBlockOfMap{number: lastBlock, hash: lbHash})
}
if blockNumber != math.MaxUint64 {
blockPtr, err := reader.getBlockLvPointer(blockNumber)
if err != nil {
t.Fatalf("Test case %s: error reading lv pointer of block %d: %v", testCase, blockNumber, err)
} else if blockPtr != expBlockPtr {
t.Fatalf("Test case %s: incorrect results when reading lv pointer of block %d (expected %v, got %v)", testCase, blockNumber, expBlockPtr, blockPtr)
}
}
}
testNoPointer := func(mapIndex uint32) {
if _, _, err := reader.getLastBlockOfMap(mapIndex); err == nil {
t.Fatalf("Test case %s: unexpected last block of map pointer at map %d", testCase, mapIndex)
}
}
testKnownEpoch := func(epoch uint32) {
testNoPointer(params.firstEpochMap(epoch))
testPointers(params.lastEpochMap(epoch), lastBlockOfMap{number: knownEpochs[epoch].BlockNumber, hash: knownEpochs[epoch].BlockHash}, knownEpochs[epoch].BlockNumber, knownEpochs[epoch].FirstIndex)
}
if len(knownEpochs) > 0 {
for range testPointersCount {
epoch := uint32(rand.Intn(len(knownEpochs)))
testKnownEpoch(epoch)
}
testKnownEpoch(uint32(len(knownEpochs) - 1))
}
testMapPointers := func(mapIndex uint32, blockPos int) { // 0: first 1: random 2: last
fm := maps[mapIndex-mapRange.First()]
if len(fm.blockPtrs) == 0 {
testPointers(mapIndex, fm.lastBlock, math.MaxUint64, 0)
} else {
var subIndex int
switch blockPos {
case 1:
subIndex = rand.Intn(len(fm.blockPtrs))
case 2:
subIndex = len(fm.blockPtrs) - 1
}
testPointers(mapIndex, fm.lastBlock, fm.firstBlock()+uint64(subIndex), fm.blockPtrs[subIndex])
}
}
if !mapRange.IsEmpty() {
testMapPointers(mapRange.First(), 0)
for range testPointersCount {
testMapPointers(mapRange.First()+uint32(rand.Intn(int(mapRange.Count()))), 1)
}
testMapPointers(mapRange.Last(), 2)
}
testNoPointer(mapRange.AfterLast())
testNoPointer(params.lastEpochMap(params.mapEpoch(mapRange.AfterLast())))
testNoPointer(params.firstEpochMap(params.mapEpoch(mapRange.AfterLast()) + 1))
}
func generateTestMaps(params *Params, maps []*finishedMap, amount uint32) []*finishedMap {
var lastBlock lastBlockOfMap
genesis := len(maps) == 0
if !genesis {
lastBlock = maps[len(maps)-1].lastBlock
}
for range amount {
blockCount := rand.Intn(testMaxBlocksPerMap + 1)
if genesis && blockCount == 0 {
blockCount = 1
}
fm := &finishedMap{
rowPtrs: make([]uint16, params.mapHeight),
rowData: make([]uint32, params.valuesPerMap),
blockPtrs: make([]uint64, blockCount),
}
addValues := params.valuesPerMap
for addValues > 0 {
addToRow := uint64(rand.Intn(int(min(addValues+1, params.valuesPerMap/2))))
addValues -= addToRow
var rowIndex uint32
for {
rowIndex = uint32(rand.Intn(int(params.mapHeight)))
if uint64(fm.rowPtrs[rowIndex])+addToRow <= params.valuesPerMap/2 {
break
}
}
fm.rowPtrs[rowIndex] += uint16(addToRow)
}
var ptr uint16
for i, r := range fm.rowPtrs {
ptr += r
fm.rowPtrs[i] = ptr
}
for i := range fm.rowData {
fm.rowData[i] = uint32(rand.Intn(1 << params.logMapWidth))
}
if blockCount > 0 {
startPtr := uint64(len(maps)) * params.valuesPerMap
for i := range fm.blockPtrs {
if genesis {
genesis = false
} else {
lastBlock.number++
fm.blockPtrs[i] = startPtr + uint64(i)*params.valuesPerMap/uint64(blockCount) + uint64(rand.Intn(int(params.valuesPerMap)/blockCount/2))
}
}
rand.Read(lastBlock.hash[:])
}
fm.lastBlock = lastBlock
maps = append(maps, fm)
}
return maps
}
func generateEpochCheckpoint(mapIndex uint32, maps []*finishedMap) epochCheckpoint {
for len(maps[mapIndex].blockPtrs) == 0 {
mapIndex--
}
return epochCheckpoint{
BlockNumber: maps[mapIndex].lastBlock.number,
BlockHash: maps[mapIndex].lastBlock.hash,
FirstIndex: maps[mapIndex].blockPtrs[len(maps[mapIndex].blockPtrs)-1],
}
}
func generateTestCheckpoints(params *Params, maps []*finishedMap) checkpointList {
cpList := make(checkpointList, uint32(len(maps))/params.mapsPerEpoch)
for i := range cpList {
cpList[i] = generateEpochCheckpoint(params.lastEpochMap(uint32(i)), maps)
}
return cpList
}

42
core/rawdb/accessors_indexes.go
View file

@ -320,12 +320,12 @@ func ReadCanonicalRawReceipt(db ethdb.Reader, blockHash common.Hash, blockNumber
// same data proximity reasons it is also suitable for database representation.
// See also:
// https://eips.ethereum.org/EIPS/eip-7745#hash-tree-structure
func ReadFilterMapExtRow(db ethdb.KeyValueReader, mapRowIndex uint64, bitLength uint) ([]uint32, error) {
func ReadFilterMapSingleRow(db ethdb.KeyValueReader, mapRowIndex uint64, dbLayer uint32, bitLength uint) ([]uint32, error) {
byteLength := int(bitLength) / 8
if int(bitLength) != byteLength*8 {
panic("invalid bit length")
}
key := filterMapRowKey(mapRowIndex, false)
key := filterMapRowKey(mapRowIndex, dbLayer)
has, err := db.Has(key)
if err != nil {
return nil, err
@ -349,18 +349,19 @@ func ReadFilterMapExtRow(db ethdb.KeyValueReader, mapRowIndex uint64, bitLength
return row, nil
}
func ReadFilterMapBaseRows(db ethdb.KeyValueReader, mapRowIndex uint64, rowCount uint32, bitLength uint) ([][]uint32, error) {
func ReadFilterMapRowGroup(db ethdb.KeyValueReader, mapRowIndex uint64, dbLayer, rowCount uint32, bitLength uint) ([][]uint32, error) {
byteLength := int(bitLength) / 8
if int(bitLength) != byteLength*8 {
panic("invalid bit length")
}
key := filterMapRowKey(mapRowIndex, true)
key := filterMapRowKey(mapRowIndex, dbLayer)
has, err := db.Has(key)
if err != nil {
return nil, err
}
rows := make([][]uint32, rowCount)
if !has {
//fmt.Println(" no entry")
return rows, nil
}
encRows, err := db.Get(key)
@ -368,6 +369,7 @@ func ReadFilterMapBaseRows(db ethdb.KeyValueReader, mapRowIndex uint64, rowCount
return nil, err
}
encLen := len(encRows)
//fmt.Println(" encLen", encLen)
var (
entryCount, entriesInRow, rowIndex, headerLen, headerBits int
headerByte byte
@ -411,7 +413,7 @@ func ReadFilterMapBaseRows(db ethdb.KeyValueReader, mapRowIndex uint64, rowCount
// WriteFilterMapExtRow stores an extended filter map row at the given mapRowIndex
// or deletes any existing entry if the row is empty.
func WriteFilterMapExtRow(db ethdb.KeyValueWriter, mapRowIndex uint64, row []uint32, bitLength uint) {
func WriteFilterMapSingleRow(db ethdb.KeyValueWriter, mapRowIndex uint64, dbLayer uint32, row []uint32, bitLength uint) {
byteLength := int(bitLength) / 8
if int(bitLength) != byteLength*8 {
panic("invalid bit length")
@ -424,16 +426,24 @@ func WriteFilterMapExtRow(db ethdb.KeyValueWriter, mapRowIndex uint64, row []uin
binary.LittleEndian.PutUint32(b[:], c)
copy(encRow[i*byteLength:(i+1)*byteLength], b[:byteLength])
}
err = db.Put(filterMapRowKey(mapRowIndex, false), encRow)
err = db.Put(filterMapRowKey(mapRowIndex, dbLayer), encRow)
} else {
err = db.Delete(filterMapRowKey(mapRowIndex, false))
err = db.Delete(filterMapRowKey(mapRowIndex, dbLayer))
}
if err != nil {
log.Crit("Failed to store extended filter map row", "err", err)
}
}
func WriteFilterMapBaseRows(db ethdb.KeyValueWriter, mapRowIndex uint64, rows [][]uint32, bitLength uint) {
func WriteFilterMapRowGroup(db ethdb.KeyValueWriter, mapRowIndex uint64, dbLayer uint32, rows [][]uint32, bitLength uint) {
/*var totalsize int //TODO
for _, row := range rows {
totalsize += len(row)
}
if totalsize > 0 {
fmt.Println("WriteFilterMapRowGroup", mapRowIndex, dbLayer, totalsize)
}*/
byteLength := int(bitLength) / 8
if int(bitLength) != byteLength*8 {
panic("invalid bit length")
@ -476,9 +486,10 @@ func WriteFilterMapBaseRows(db ethdb.KeyValueWriter, mapRowIndex uint64, rows []
addHeaderBit(false)
zeroBits--
}
err = db.Put(filterMapRowKey(mapRowIndex, true), encRows)
//fmt.Println(" write encLen", len(encRows))
err = db.Put(filterMapRowKey(mapRowIndex, dbLayer), encRows)
} else {
err = db.Delete(filterMapRowKey(mapRowIndex, true))
err = db.Delete(filterMapRowKey(mapRowIndex, dbLayer))
}
if err != nil {
log.Crit("Failed to store base filter map rows", "err", err)
@ -486,7 +497,7 @@ func WriteFilterMapBaseRows(db ethdb.KeyValueWriter, mapRowIndex uint64, rows []
}
func DeleteFilterMapRows(db ethdb.KeyValueStore, mapRows common.Range[uint64], hashScheme bool, stopCallback func(bool) bool) error {
return SafeDeleteRange(db, filterMapRowKey(mapRows.First(), false), filterMapRowKey(mapRows.AfterLast(), false), hashScheme, stopCallback)
return SafeDeleteRange(db, filterMapRowKey(mapRows.First(), 0), filterMapRowKey(mapRows.AfterLast(), 0), hashScheme, stopCallback)
}
// ReadFilterMapLastBlock retrieves the number of the block that generated the
@ -565,12 +576,9 @@ func DeleteBlockLvPointers(db ethdb.KeyValueStore, blocks common.Range[uint64],
// FilterMapsRange is a storage representation of the block range covered by the
// filter maps structure and the corresponting log value index range.
type FilterMapsRange struct {
Version uint32
HeadIndexed bool
HeadDelimiter uint64
BlocksFirst, BlocksAfterLast uint64
MapsFirst, MapsAfterLast uint32
TailPartialEpoch uint32
Version uint32
ValidMaps, DirtyMaps []uint32
KnownEpochs uint32
}
// ReadFilterMapsRange retrieves the filter maps range data. Note that if the

11
core/rawdb/schema.go
View file

@ -152,7 +152,7 @@ var (
SyncCommitteeKey = []byte("committee-") // bigEndian64(syncPeriod) -> serialized committee
// new log index
filterMapsPrefix = "fm-"
filterMapsPrefix = "fm*" //TODO fm-
filterMapsRangeKey = []byte(filterMapsPrefix + "R")
filterMapRowPrefix = []byte(filterMapsPrefix + "r") // filterMapRowPrefix + mapRowIndex (uint64 big endian) -> filter row
filterMapLastBlockPrefix = []byte(filterMapsPrefix + "b") // filterMapLastBlockPrefix + mapIndex (uint32 big endian) -> block number (uint64 big endian)
@ -353,15 +353,12 @@ func IsStorageTrieNode(key []byte) bool {
}
// filterMapRowKey = filterMapRowPrefix + mapRowIndex (uint64 big endian)
func filterMapRowKey(mapRowIndex uint64, base bool) []byte {
extLen := 8
if base {
extLen = 9
}
func filterMapRowKey(mapRowIndex uint64, dbLayer uint32) []byte {
l := len(filterMapRowPrefix)
key := make([]byte, l+extLen)
key := make([]byte, l+9)
copy(key[:l], filterMapRowPrefix)
binary.BigEndian.PutUint64(key[l:l+8], mapRowIndex)
key[l+8] = byte(dbLayer)
return key
}

6
eth/api_backend.go
View file

@ -459,7 +459,7 @@ func (b *EthAPIBackend) RPCTxFeeCap() float64 {
return b.eth.config.RPCTxFeeCap
}
func (b *EthAPIBackend) CurrentView() *filtermaps.ChainView {
func (b *EthAPIBackend) CurrentChainView() *filtermaps.ChainView {
head := b.eth.blockchain.CurrentBlock()
if head == nil {
return nil
@ -467,8 +467,8 @@ func (b *EthAPIBackend) CurrentView() *filtermaps.ChainView {
return filtermaps.NewChainView(b.eth.blockchain, head.Number.Uint64(), head.Hash())
}
func (b *EthAPIBackend) NewMatcherBackend() filtermaps.MatcherBackend {
return b.eth.filterMaps.NewMatcherBackend()
func (b *EthAPIBackend) GetIndexView(headBlockHash common.Hash) *filtermaps.IndexView {
return b.eth.logIndexer.GetIndexView(headBlockHash)
}
func (b *EthAPIBackend) Engine() consensus.Engine {

83
eth/backend.go
View file

@ -108,8 +108,7 @@ type Ethereum struct {
engine consensus.Engine
accountManager *accounts.Manager
filterMaps *filtermaps.FilterMaps
closeFilterMaps chan chan struct{}
logIndexer *filtermaps.Indexer
APIBackend *EthAPIBackend
@ -285,18 +284,8 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
ExportFileName: config.LogExportCheckpoints,
HashScheme: scheme == rawdb.HashScheme,
}
chainView := eth.newChainView(eth.blockchain.CurrentBlock())
historyCutoff, _ := eth.blockchain.HistoryPruningCutoff()
var finalBlock uint64
if fb := eth.blockchain.CurrentFinalBlock(); fb != nil {
finalBlock = fb.Number.Uint64()
}
filterMaps, err := filtermaps.NewFilterMaps(chainDb, chainView, historyCutoff, finalBlock, filtermaps.DefaultParams, fmConfig)
if err != nil {
return nil, err
}
eth.filterMaps = filterMaps
eth.closeFilterMaps = make(chan chan struct{})
eth.logIndexer = filtermaps.NewIndexer(chainDb, filtermaps.DefaultParams, fmConfig)
eth.blockchain.RegisterIndexer(eth.logIndexer, "log index")
// TxPool
if config.TxPool.Journal != "" {
@ -452,71 +441,9 @@ func (s *Ethereum) Start() error {
// Start the connection manager
s.dropper.Start(s.p2pServer, func() bool { return !s.Synced() })
// start log indexer
s.filterMaps.Start()
go s.updateFilterMapsHeads()
return nil
}
func (s *Ethereum) newChainView(head *types.Header) *filtermaps.ChainView {
if head == nil {
return nil
}
return filtermaps.NewChainView(s.blockchain, head.Number.Uint64(), head.Hash())
}
func (s *Ethereum) updateFilterMapsHeads() {
headEventCh := make(chan core.ChainEvent, 10)
blockProcCh := make(chan bool, 10)
sub := s.blockchain.SubscribeChainEvent(headEventCh)
sub2 := s.blockchain.SubscribeBlockProcessingEvent(blockProcCh)
defer func() {
sub.Unsubscribe()
sub2.Unsubscribe()
for {
select {
case <-headEventCh:
case <-blockProcCh:
default:
return
}
}
}()
var head *types.Header
setHead := func(newHead *types.Header) {
if newHead == nil {
return
}
if head == nil || newHead.Hash() != head.Hash() {
head = newHead
chainView := s.newChainView(head)
historyCutoff, _ := s.blockchain.HistoryPruningCutoff()
var finalBlock uint64
if fb := s.blockchain.CurrentFinalBlock(); fb != nil {
finalBlock = fb.Number.Uint64()
}
s.filterMaps.SetTarget(chainView, historyCutoff, finalBlock)
}
}
setHead(s.blockchain.CurrentBlock())
for {
select {
case ev := <-headEventCh:
setHead(ev.Header)
case blockProc := <-blockProcCh:
s.filterMaps.SetBlockProcessing(blockProc)
case <-time.After(time.Second * 10):
setHead(s.blockchain.CurrentBlock())
case ch := <-s.closeFilterMaps:
close(ch)
return
}
}
}
func (s *Ethereum) setupDiscovery() error {
eth.StartENRUpdater(s.blockchain, s.p2pServer.LocalNode())
@ -575,10 +502,6 @@ func (s *Ethereum) Stop() error {
s.handler.Stop()
// Then stop everything else.
ch := make(chan struct{})
s.closeFilterMaps <- ch
<-ch
s.filterMaps.Stop()
s.txPool.Close()
s.blockchain.Stop()
s.engine.Close()

347
eth/filters/filter.go
View file

@ -19,9 +19,11 @@ package filters
import (
"context"
"errors"
"fmt"
"math"
"math/big"
"slices"
"sync"
"time"
"github.com/ethereum/go-ethereum/common"
@ -43,7 +45,12 @@ type Filter struct {
block *common.Hash // Block hash if filtering a single block
begin, end int64 // Range interval if filtering multiple blocks
rangeLogsTestHook chan rangeLogsTestEvent
testFilterRanges []testFilterRange
}
type testFilterRange struct {
begin, end uint64
indexed bool
}
// NewRangeFilter creates a new filter which uses a bloom filter on blocks to
@ -146,272 +153,101 @@ func (f *Filter) Logs(ctx context.Context) ([]*types.Log, error) {
return f.rangeLogs(ctx, begin, end)
}
const (
rangeLogsTestDone = iota // zero range
rangeLogsTestSync // before sync; zero range
rangeLogsTestSynced // after sync; valid blocks range
rangeLogsTestIndexed // individual search range
rangeLogsTestUnindexed // individual search range
rangeLogsTestResults // results range after search iteration
rangeLogsTestReorg // results range trimmed by reorg
)
type rangeLogsTestEvent struct {
event int
blocks common.Range[uint64]
}
// searchSession represents a single search session.
type searchSession struct {
ctx context.Context
filter *Filter
mb filtermaps.MatcherBackend
syncRange filtermaps.SyncRange // latest synchronized state with the matcher
chainView *filtermaps.ChainView // can be more recent than the indexed view in syncRange
// block ranges always refer to the current chainView
firstBlock, lastBlock uint64 // specified search range; MaxUint64 means latest block
searchRange common.Range[uint64] // actual search range; end trimmed to latest head
matchRange common.Range[uint64] // range in which we have results (subset of searchRange)
matches []*types.Log // valid set of matches in matchRange
forceUnindexed bool // revert to unindexed search
}
// newSearchSession returns a new searchSession.
func newSearchSession(ctx context.Context, filter *Filter, mb filtermaps.MatcherBackend, firstBlock, lastBlock uint64) (*searchSession, error) {
s := &searchSession{
ctx: ctx,
filter: filter,
mb: mb,
firstBlock: firstBlock,
lastBlock: lastBlock,
}
// enforce a consistent state before starting the search in order to be able
// to determine valid range later
var err error
s.syncRange, err = s.mb.SyncLogIndex(s.ctx)
if err != nil {
return nil, err
}
if err := s.updateChainView(); err != nil {
return nil, err
}
return s, nil
}
// updateChainView updates to the latest view of the underlying chain and sets
// searchRange by replacing MaxUint64 (meaning latest block) with actual head
// number in the specified search range.
// If the session already had an existing chain view and set of matches then
// it also trims part of the match set that a chain reorg might have invalidated.
func (s *searchSession) updateChainView() error {
// update chain view based on current chain head (might be more recent than
// the indexed view of syncRange as the indexer updates it asynchronously
// with some delay
newChainView := s.filter.sys.backend.CurrentView()
if newChainView == nil {
return errors.New("head block not available")
}
head := newChainView.HeadNumber()
// update actual search range based on current head number
firstBlock, lastBlock := s.firstBlock, s.lastBlock
if firstBlock == math.MaxUint64 {
firstBlock = head
}
if lastBlock == math.MaxUint64 {
lastBlock = head
}
if firstBlock > lastBlock || lastBlock > head {
return errInvalidBlockRange
}
s.searchRange = common.NewRange(firstBlock, lastBlock+1-firstBlock)
// Trim existing match set in case a reorg may have invalidated some results
if !s.matchRange.IsEmpty() {
trimRange := newChainView.SharedRange(s.chainView).Intersection(s.searchRange)
s.matchRange, s.matches = s.trimMatches(trimRange, s.matchRange, s.matches)
}
s.chainView = newChainView
return nil
}
// trimMatches removes any entries from the specified set of matches that is
// outside the given range.
func (s *searchSession) trimMatches(trimRange, matchRange common.Range[uint64], matches []*types.Log) (common.Range[uint64], []*types.Log) {
newRange := matchRange.Intersection(trimRange)
if newRange == matchRange {
return matchRange, matches
}
if newRange.IsEmpty() {
return newRange, nil
}
for len(matches) > 0 && matches[0].BlockNumber < newRange.First() {
matches = matches[1:]
}
for len(matches) > 0 && matches[len(matches)-1].BlockNumber > newRange.Last() {
matches = matches[:len(matches)-1]
}
return newRange, matches
}
// searchInRange performs a single range search, either indexed or unindexed.
func (s *searchSession) searchInRange(r common.Range[uint64], indexed bool) (common.Range[uint64], []*types.Log, error) {
if indexed {
if s.filter.rangeLogsTestHook != nil {
s.filter.rangeLogsTestHook <- rangeLogsTestEvent{rangeLogsTestIndexed, r}
}
results, err := s.filter.indexedLogs(s.ctx, s.mb, r.First(), r.Last())
if err != nil && !errors.Is(err, filtermaps.ErrMatchAll) {
return common.Range[uint64]{}, nil, err
}
if err == nil {
// sync with filtermaps matcher
if s.filter.rangeLogsTestHook != nil {
s.filter.rangeLogsTestHook <- rangeLogsTestEvent{rangeLogsTestSync, common.Range[uint64]{}}
}
var syncErr error
if s.syncRange, syncErr = s.mb.SyncLogIndex(s.ctx); syncErr != nil {
return common.Range[uint64]{}, nil, syncErr
}
if s.filter.rangeLogsTestHook != nil {
s.filter.rangeLogsTestHook <- rangeLogsTestEvent{rangeLogsTestSynced, s.syncRange.ValidBlocks}
}
// discard everything that might be invalid
trimRange := s.syncRange.ValidBlocks.Intersection(s.chainView.SharedRange(s.syncRange.IndexedView))
matchRange, matches := s.trimMatches(trimRange, r, results)
return matchRange, matches, nil
}
// "match all" filters are not supported by filtermaps; fall back to
// unindexed search which is the most efficient in this case
s.forceUnindexed = true
// fall through to unindexed case
}
if s.filter.rangeLogsTestHook != nil {
s.filter.rangeLogsTestHook <- rangeLogsTestEvent{rangeLogsTestUnindexed, r}
}
matches, err := s.filter.unindexedLogs(s.ctx, s.chainView, r.First(), r.Last())
if err != nil {
return common.Range[uint64]{}, nil, err
}
return r, matches, nil
}
// doSearchIteration performs a search on a range missing from an incomplete set
// of results, adds the new section and removes invalidated entries.
func (s *searchSession) doSearchIteration() error {
switch {
case s.matchRange.IsEmpty():
// no results yet; try search in entire range
indexedSearchRange := s.searchRange.Intersection(s.syncRange.IndexedBlocks)
if s.forceUnindexed = indexedSearchRange.IsEmpty(); !s.forceUnindexed {
// indexed search on the intersection of indexed and searched range
matchRange, matches, err := s.searchInRange(indexedSearchRange, true)
if err != nil {
return err
}
s.matchRange = matchRange
s.matches = matches
return nil
} else {
// no intersection of indexed and searched range; unindexed search on
// the whole searched range
matchRange, matches, err := s.searchInRange(s.searchRange, false)
if err != nil {
return err
}
s.matchRange = matchRange
s.matches = matches
return nil
}
case !s.matchRange.IsEmpty() && s.matchRange.First() > s.searchRange.First():
// Results are available, but the tail section is missing. Perform an unindexed
// search for the missing tail, while still allowing indexed search for the head.
//
// The unindexed search is necessary because the tail portion of the indexes
// has been pruned.
tailRange := common.NewRange(s.searchRange.First(), s.matchRange.First()-s.searchRange.First())
_, tailMatches, err := s.searchInRange(tailRange, false)
if err != nil {
return err
}
s.matches = append(tailMatches, s.matches...)
s.matchRange = tailRange.Union(s.matchRange)
return nil
case !s.matchRange.IsEmpty() && s.matchRange.First() == s.searchRange.First() && s.searchRange.AfterLast() > s.matchRange.AfterLast():
// Results are available, but the head section is missing. Try to perform
// the indexed search for the missing head, or fallback to unindexed search
// if the tail portion of indexed range has been pruned.
headRange := common.NewRange(s.matchRange.AfterLast(), s.searchRange.AfterLast()-s.matchRange.AfterLast())
if !s.forceUnindexed {
indexedHeadRange := headRange.Intersection(s.syncRange.IndexedBlocks)
if !indexedHeadRange.IsEmpty() && indexedHeadRange.First() == headRange.First() {
headRange = indexedHeadRange
} else {
// The tail portion of the indexes has been pruned, falling back
// to unindexed search.
s.forceUnindexed = true
}
}
headMatchRange, headMatches, err := s.searchInRange(headRange, !s.forceUnindexed)
if err != nil {
return err
}
if headMatchRange.First() != s.matchRange.AfterLast() {
// improbable corner case, first part of new head range invalidated by tail unindexing
s.matches, s.matchRange = headMatches, headMatchRange
return nil
}
s.matches = append(s.matches, headMatches...)
s.matchRange = s.matchRange.Union(headMatchRange)
return nil
default:
panic("invalid search session state")
}
}
func (f *Filter) rangeLogs(ctx context.Context, firstBlock, lastBlock uint64) ([]*types.Log, error) {
if f.rangeLogsTestHook != nil {
defer func() {
f.rangeLogsTestHook <- rangeLogsTestEvent{rangeLogsTestDone, common.Range[uint64]{}}
close(f.rangeLogsTestHook)
}()
}
if firstBlock > lastBlock {
return nil, nil
}
mb := f.sys.backend.NewMatcherBackend()
defer mb.Close()
chainView := f.sys.backend.CurrentChainView()
session, err := newSearchSession(ctx, f, mb, firstBlock, lastBlock)
if firstBlock > chainView.HeadNumber() {
firstBlock = chainView.HeadNumber()
}
if lastBlock > chainView.HeadNumber() {
lastBlock = chainView.HeadNumber()
}
indexView := f.sys.backend.GetIndexView(chainView.BlockHash(chainView.HeadNumber()))
if indexView == nil {
return f.unindexedLogs(ctx, chainView, firstBlock, lastBlock)
}
searchRange := common.NewRange[uint64](firstBlock, lastBlock+1-firstBlock)
indexedRange := indexView.BlockRange().Intersection(searchRange)
if indexedRange.IsEmpty() {
indexView.Release()
return f.unindexedLogs(ctx, chainView, firstBlock, lastBlock)
}
var (
res1, res2, res3 []*types.Log
err error
)
res2, err = f.indexedLogs(ctx, chainView, indexView, indexedRange.First(), indexedRange.Last())
indexView.Release()
if err == filtermaps.ErrMatchAll {
return f.unindexedLogs(ctx, chainView, firstBlock, lastBlock)
}
if err != nil {
return nil, err
}
for session.searchRange != session.matchRange {
if err := session.doSearchIteration(); err != nil {
if searchRange.First() < indexedRange.First() {
res1, err = f.unindexedLogs(ctx, chainView, searchRange.First(), indexedRange.First()-1)
if err != nil {
return nil, err
}
if f.rangeLogsTestHook != nil {
f.rangeLogsTestHook <- rangeLogsTestEvent{rangeLogsTestResults, session.matchRange}
}
mr := session.matchRange
if err := session.updateChainView(); err != nil {
return nil, err
}
if f.rangeLogsTestHook != nil && session.matchRange != mr {
f.rangeLogsTestHook <- rangeLogsTestEvent{rangeLogsTestReorg, session.matchRange}
}
}
return session.matches, nil
if indexedRange.Last() < searchRange.Last() {
res3, err = f.unindexedLogs(ctx, chainView, indexedRange.AfterLast(), searchRange.Last())
if err != nil {
return nil, err
}
}
return append(append(res1, res2...), res3...), nil
}
func (f *Filter) indexedLogs(ctx context.Context, mb filtermaps.MatcherBackend, begin, end uint64) ([]*types.Log, error) {
func (f *Filter) indexedLogs(ctx context.Context, chainView *filtermaps.ChainView, indexView *filtermaps.IndexView, begin, end uint64) ([]*types.Log, error) {
if f.testFilterRanges != nil {
f.testFilterRanges = append(f.testFilterRanges, testFilterRange{begin: begin, end: end, indexed: true})
}
start := time.Now()
potentialMatches, err := filtermaps.GetPotentialMatches(ctx, mb, begin, end, f.addresses, f.topics)
matches := filterLogs(potentialMatches, nil, nil, f.addresses, f.topics)
potentialMatches, err := filtermaps.GetPotentialMatches(ctx, indexView, begin, end, f.addresses, f.topics)
potentialLogs := make([]*types.Log, len(potentialMatches))
indexCh := make(chan int)
errCh := make(chan error, 1)
var wg sync.WaitGroup
worker := func() {
defer wg.Done()
for i := range indexCh {
logPosition := potentialMatches[i]
receipts := chainView.Receipts(logPosition.BlockNumber)
if receipts == nil {
select {
case errCh <- fmt.Errorf("receipts for block #%d not found", logPosition.BlockNumber):
default:
}
return
}
log, err := logPosition.GetLog(receipts)
if err != nil {
select {
case errCh <- err:
default:
}
return
}
potentialLogs[i] = log
}
}
for range 4 { //TODO
wg.Add(1)
go worker()
}
for i := range potentialMatches {
indexCh <- i
}
close(indexCh)
wg.Wait()
matches := filterLogs(potentialLogs, nil, nil, f.addresses, f.topics)
log.Trace("Performed indexed log search", "begin", begin, "end", end, "true matches", len(matches), "false positives", len(potentialMatches)-len(matches), "elapsed", common.PrettyDuration(time.Since(start)))
return matches, err
}
@ -419,6 +255,9 @@ func (f *Filter) indexedLogs(ctx context.Context, mb filtermaps.MatcherBackend,
// unindexedLogs returns the logs matching the filter criteria based on raw block
// iteration and bloom matching.
func (f *Filter) unindexedLogs(ctx context.Context, chainView *filtermaps.ChainView, begin, end uint64) ([]*types.Log, error) {
if f.testFilterRanges != nil {
f.testFilterRanges = append(f.testFilterRanges, testFilterRange{begin: begin, end: end, indexed: false})
}
start := time.Now()
log.Debug("Performing unindexed log search", "begin", begin, "end", end)
var matches []*types.Log
@ -517,7 +356,7 @@ func filterLogs(logs []*types.Log, fromBlock, toBlock *big.Int, addresses []comm
}
var ret []*types.Log
for _, log := range logs {
if check(log) {
if log != nil && check(log) {
ret = append(ret, log)
}
}

4
eth/filters/filter_system.go
View file

@ -72,8 +72,8 @@ type Backend interface {
SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription
SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription
CurrentView() *filtermaps.ChainView
NewMatcherBackend() filtermaps.MatcherBackend
CurrentChainView() *filtermaps.ChainView
GetIndexView(headBlockHash common.Hash) *filtermaps.IndexView
}
// FilterSystem holds resources shared by all filters.

40
eth/filters/filter_system_test.go
View file

@ -42,7 +42,7 @@ import (
type testBackend struct {
db ethdb.Database
fm *filtermaps.FilterMaps
logIndexer *filtermaps.Indexer
txFeed event.Feed
logsFeed event.Feed
rmLogsFeed event.Feed
@ -160,31 +160,51 @@ func (b *testBackend) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subsc
return b.chainFeed.Subscribe(ch)
}
func (b *testBackend) CurrentView() *filtermaps.ChainView {
func (b *testBackend) CurrentChainView() *filtermaps.ChainView {
head := b.CurrentBlock()
return filtermaps.NewChainView(b, head.Number.Uint64(), head.Hash())
}
func (b *testBackend) NewMatcherBackend() filtermaps.MatcherBackend {
return b.fm.NewMatcherBackend()
func (b *testBackend) GetIndexView(headBlockHash common.Hash) *filtermaps.IndexView {
return b.logIndexer.GetIndexView(headBlockHash)
}
func (b *testBackend) startFilterMaps(history uint64, disabled bool, params filtermaps.Params) {
head := b.CurrentBlock()
chainView := filtermaps.NewChainView(b, head.Number.Uint64(), head.Hash())
config := filtermaps.Config{
History: history,
Disabled: disabled,
ExportFileName: "",
}
b.fm, _ = filtermaps.NewFilterMaps(b.db, chainView, 0, 0, params, config)
b.fm.Start()
b.fm.WaitIdle()
b.logIndexer = filtermaps.NewIndexer(b.db, params, config)
if !disabled {
b.indexHead(head.Number.Uint64())
}
}
func (b *testBackend) indexHead(head uint64) {
header, _ := b.HeaderByNumber(context.Background(), rpc.BlockNumber(head))
receipts, _ := b.GetReceipts(context.Background(), header.Hash())
b.logIndexer.AddBlockData([]*types.Header{header}, []types.Receipts{receipts})
b.backfillBlocks()
}
func (b *testBackend) backfillBlocks() {
ready, needBlocks := b.logIndexer.Status()
for !ready || !needBlocks.IsEmpty() {
for !ready {
time.Sleep(time.Millisecond)
ready, needBlocks = b.logIndexer.Status()
}
header, _ := b.HeaderByNumber(context.Background(), rpc.BlockNumber(needBlocks.First()))
receipts, _ := b.GetReceipts(context.Background(), header.Hash())
ready, needBlocks = b.logIndexer.AddBlockData([]*types.Header{header}, []types.Receipts{receipts})
}
}
func (b *testBackend) stopFilterMaps() {
b.fm.Stop()
b.fm = nil
b.logIndexer.Stop()
b.logIndexer = nil
}
func (b *testBackend) setPending(block *types.Block, receipts types.Receipts) {

185
eth/filters/filter_test.go
View file

@ -20,6 +20,7 @@ import (
"context"
"encoding/json"
"math/big"
"slices"
"strings"
"testing"
"time"
@ -431,173 +432,45 @@ func TestRangeLogs(t *testing.T) {
t.Fatal(err)
}
chain, _ := core.GenerateChain(gspec.Config, gspec.ToBlock(), ethash.NewFaker(), db, 1000, func(i int, gen *core.BlockGen) {})
options := core.DefaultConfig().WithStateScheme(rawdb.HashScheme)
options.TxLookupLimit = 0 // index all txs
bc, err := core.NewBlockChain(db, gspec, ethash.NewFaker(), options)
if err != nil {
t.Fatal(err)
}
_, err = bc.InsertChain(chain[:600])
if err != nil {
t.Fatal(err)
}
backend.startFilterMaps(200, false, filtermaps.RangeTestParams)
defer backend.stopFilterMaps()
var (
testCase, event int
filter *Filter
addresses = []common.Address{{}}
)
expEvent := func(expEvent int, expFirst, expAfterLast uint64) {
exp := rangeLogsTestEvent{expEvent, common.NewRange[uint64](expFirst, expAfterLast-expFirst)}
event++
ev := <-filter.rangeLogsTestHook
if ev != exp {
t.Fatalf("Test case #%d: wrong test event #%d received (got %v, expected %v)", testCase, event, ev, exp)
rangeTest := func(testCase int, begin, end int64, expRanges []testFilterRange) {
filter := sys.NewRangeFilter(begin, end, []common.Address{{}}, nil)
filter.testFilterRanges = []testFilterRange{}
filter.Logs(context.Background())
if !slices.Equal(filter.testFilterRanges, expRanges) {
t.Fatalf("Test case #%d: wrong list of filtered ranges received (got %v, expected %v)", testCase, filter.testFilterRanges, expRanges)
}
}
newFilter := func(begin, end int64) {
testCase++
event = 0
filter = sys.NewRangeFilter(begin, end, addresses, nil)
filter.rangeLogsTestHook = make(chan rangeLogsTestEvent)
go func(filter *Filter) {
filter.Logs(context.Background())
// ensure that filter will not be blocked if we exit early
for range filter.rangeLogsTestHook {
}
}(filter)
}
bc.InsertChain(chain[:600])
backend.startFilterMaps(200, false, filtermaps.RangeTestParams)
defer backend.stopFilterMaps()
updateHead := func() {
head := bc.CurrentBlock()
backend.fm.SetTarget(filtermaps.NewChainView(backend, head.Number.Uint64(), head.Hash()), 0, 0)
backend.fm.WaitIdle()
}
rangeTest(1, 0, 300, []testFilterRange{{0, 300, false}})
rangeTest(2, 300, 500, []testFilterRange{{400, 500, true}, {300, 399, false}})
rangeTest(3, 400, 10000, []testFilterRange{{400, 600, true}})
bc.SetCanonical(chain[99])
backend.logIndexer.Revert(100)
rangeTest(4, 0, 10000, []testFilterRange{{0, 100, false}})
backend.backfillBlocks()
bc.InsertChain(chain[100:120])
backend.indexHead(120)
rangeTest(5, 0, 10000, []testFilterRange{{0, 120, true}})
bc.InsertChain(chain[120:800])
backend.indexHead(800)
rangeTest(6, 400, 800, []testFilterRange{{600, 800, true}, {400, 599, false}})
backend.stopFilterMaps()
// test case #1
newFilter(300, 500)
expEvent(rangeLogsTestIndexed, 401, 501)
expEvent(rangeLogsTestSync, 0, 0)
expEvent(rangeLogsTestSynced, 401, 601)
expEvent(rangeLogsTestResults, 401, 501)
expEvent(rangeLogsTestUnindexed, 300, 401)
if _, err := bc.InsertChain(chain[600:700]); err != nil {
t.Fatal(err)
}
updateHead()
expEvent(rangeLogsTestResults, 300, 501)
expEvent(rangeLogsTestDone, 0, 0)
backend.startFilterMaps(0, true, filtermaps.RangeTestParams)
rangeTest(7, 0, 10000, []testFilterRange{{0, 800, false}})
backend.stopFilterMaps()
// test case #2
newFilter(400, int64(rpc.LatestBlockNumber))
expEvent(rangeLogsTestIndexed, 501, 701)
if _, err := bc.InsertChain(chain[700:800]); err != nil {
t.Fatal(err)
}
updateHead()
expEvent(rangeLogsTestSync, 0, 0)
expEvent(rangeLogsTestSynced, 601, 699)
expEvent(rangeLogsTestResults, 601, 699)
expEvent(rangeLogsTestUnindexed, 400, 601)
expEvent(rangeLogsTestResults, 400, 699)
expEvent(rangeLogsTestIndexed, 699, 801)
if _, err := bc.SetCanonical(chain[749]); err != nil { // set head to block 750
t.Fatal(err)
}
updateHead()
expEvent(rangeLogsTestSync, 0, 0)
expEvent(rangeLogsTestSynced, 601, 749)
expEvent(rangeLogsTestResults, 400, 749)
expEvent(rangeLogsTestIndexed, 749, 751)
expEvent(rangeLogsTestSync, 0, 0)
expEvent(rangeLogsTestSynced, 551, 751)
expEvent(rangeLogsTestResults, 400, 751)
expEvent(rangeLogsTestDone, 0, 0)
// test case #3
newFilter(int64(rpc.LatestBlockNumber), int64(rpc.LatestBlockNumber))
expEvent(rangeLogsTestIndexed, 750, 751)
if _, err := bc.SetCanonical(chain[739]); err != nil {
t.Fatal(err)
}
updateHead()
expEvent(rangeLogsTestSync, 0, 0)
expEvent(rangeLogsTestSynced, 551, 739)
expEvent(rangeLogsTestResults, 0, 0)
expEvent(rangeLogsTestIndexed, 740, 741)
if _, err := bc.InsertChain(chain[740:750]); err != nil {
t.Fatal(err)
}
updateHead()
expEvent(rangeLogsTestSync, 0, 0)
expEvent(rangeLogsTestSynced, 551, 739)
expEvent(rangeLogsTestResults, 0, 0)
expEvent(rangeLogsTestIndexed, 750, 751)
expEvent(rangeLogsTestSync, 0, 0)
expEvent(rangeLogsTestSynced, 551, 751)
expEvent(rangeLogsTestResults, 750, 751)
expEvent(rangeLogsTestDone, 0, 0)
// test case #4
if _, err := bc.SetCanonical(chain[499]); err != nil {
t.Fatal(err)
}
updateHead()
newFilter(400, int64(rpc.LatestBlockNumber))
expEvent(rangeLogsTestIndexed, 400, 501)
if _, err := bc.InsertChain(chain[500:650]); err != nil {
t.Fatal(err)
}
updateHead()
expEvent(rangeLogsTestSync, 0, 0)
expEvent(rangeLogsTestSynced, 451, 499)
expEvent(rangeLogsTestResults, 451, 499)
expEvent(rangeLogsTestUnindexed, 400, 451)
expEvent(rangeLogsTestResults, 400, 499)
// indexed head extension seems possible
expEvent(rangeLogsTestIndexed, 499, 651)
// further head extension causes tail unindexing in searched range
if _, err := bc.InsertChain(chain[650:750]); err != nil {
t.Fatal(err)
}
updateHead()
expEvent(rangeLogsTestSync, 0, 0)
expEvent(rangeLogsTestSynced, 551, 649)
// tail trimmed to 551; cannot merge with existing results
expEvent(rangeLogsTestResults, 551, 649)
expEvent(rangeLogsTestUnindexed, 400, 551)
expEvent(rangeLogsTestResults, 400, 649)
expEvent(rangeLogsTestIndexed, 649, 751)
expEvent(rangeLogsTestSync, 0, 0)
expEvent(rangeLogsTestSynced, 551, 751)
expEvent(rangeLogsTestResults, 400, 751)
expEvent(rangeLogsTestDone, 0, 0)
// test case #5
newFilter(400, int64(rpc.LatestBlockNumber))
expEvent(rangeLogsTestIndexed, 551, 751)
expEvent(rangeLogsTestSync, 0, 0)
expEvent(rangeLogsTestSynced, 551, 751)
expEvent(rangeLogsTestResults, 551, 751)
expEvent(rangeLogsTestUnindexed, 400, 551)
if _, err := bc.InsertChain(chain[750:1000]); err != nil {
t.Fatal(err)
}
updateHead()
expEvent(rangeLogsTestResults, 400, 751)
// indexed tail already beyond results head; revert to unindexed head search
expEvent(rangeLogsTestUnindexed, 751, 1001)
if _, err := bc.SetCanonical(chain[899]); err != nil {
t.Fatal(err)
}
updateHead()
expEvent(rangeLogsTestResults, 400, 1001)
expEvent(rangeLogsTestReorg, 400, 901)
expEvent(rangeLogsTestDone, 0, 0)
backend.startFilterMaps(0, false, filtermaps.RangeTestParams)
rangeTest(8, 0, 10000, []testFilterRange{{0, 800, true}})
}

4
internal/ethapi/api_test.go
View file

@ -690,10 +690,10 @@ func (b testBackend) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent)
func (b testBackend) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription {
panic("implement me")
}
func (b testBackend) CurrentView() *filtermaps.ChainView {
func (b testBackend) CurrentChainView() *filtermaps.ChainView {
panic("implement me")
}
func (b testBackend) NewMatcherBackend() filtermaps.MatcherBackend {
func (b testBackend) GetIndexView(headBlockHash common.Hash) *filtermaps.IndexView {
panic("implement me")
}

4
internal/ethapi/backend.go
View file

@ -99,8 +99,8 @@ type Backend interface {
SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription
SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription
CurrentView() *filtermaps.ChainView
NewMatcherBackend() filtermaps.MatcherBackend
CurrentChainView() *filtermaps.ChainView
GetIndexView(headBlockHash common.Hash) *filtermaps.IndexView
}
func GetAPIs(apiBackend Backend) []rpc.API {

4
internal/ethapi/transaction_args_test.go
View file

@ -407,7 +407,7 @@ func (b *backendMock) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent)
func (b *backendMock) Engine() consensus.Engine { return nil }
func (b *backendMock) CurrentView() *filtermaps.ChainView { return nil }
func (b *backendMock) NewMatcherBackend() filtermaps.MatcherBackend { return nil }
func (b *backendMock) CurrentChainView() *filtermaps.ChainView { return nil }
func (b *backendMock) GetIndexView(headBlockHash common.Hash) *filtermaps.IndexView { return nil }
func (b *backendMock) HistoryPruningCutoff() uint64 { return 0 }