// Copyright 2025 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // The go-ethereum library is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see = indexBlockEntriesCap } // encode packs index block descriptor into byte stream. func (d *indexBlockDesc) encode() []byte { var buf [indexBlockDescSize]byte binary.BigEndian.PutUint64(buf[0:8], d.max) binary.BigEndian.PutUint16(buf[8:10], d.entries) binary.BigEndian.PutUint32(buf[10:14], d.id) return buf[:] } // decode unpacks index block descriptor from byte stream. func (d *indexBlockDesc) decode(blob []byte) { d.max = binary.BigEndian.Uint64(blob[:8]) d.entries = binary.BigEndian.Uint16(blob[8:10]) d.id = binary.BigEndian.Uint32(blob[10:14]) } // parseIndexBlock parses the index block with the supplied byte stream. // The index block format can be illustrated as below: // // +---->+------------------+ // | | Chunk1 | // | +------------------+ // | | ...... | // | +-->+------------------+ // | | | ChunkN | // | | +------------------+ // +-|---| Restart1 | // | | Restart... | 2N bytes // +---| RestartN | // +------------------+ // | Restart count | 1 byte // +------------------+ // // - Chunk list: A list of data chunks // - Restart list: A list of 2-byte pointers, each pointing to the start position of a chunk // - Restart count: The number of restarts in the block, stored at the end of the block (1 byte) // // Note: the pointer is encoded as a uint16, which is sufficient within a chunk. // A uint16 can cover offsets in the range [0, 65536), which is more than enough // to store 4096 integers. // // Each chunk begins with the full value of the first integer, followed by // subsequent integers representing the differences between the current value // and the preceding one. Integers are encoded with variable-size for best // storage efficiency. Each chunk can be illustrated as below. // // Restart ---> +----------------+ // | Full integer | // +----------------+ // | Diff with prev | // +----------------+ // | ... | // +----------------+ // | Diff with prev | // +----------------+ // // Empty index block is regarded as invalid. func parseIndexBlock(blob []byte) ([]uint16, []byte, error) { if len(blob) < 1 { return nil, nil, fmt.Errorf("corrupted index block, len: %d", len(blob)) } restartLen := int(blob[len(blob)-1]) if restartLen == 0 { return nil, nil, errors.New("corrupted index block, no restart") } tailLen := restartLen*2 + 1 if len(blob) < tailLen { return nil, nil, fmt.Errorf("truncated restarts, size: %d, restarts: %d", len(blob), restartLen) } restarts := make([]uint16, restartLen) dataEnd := len(blob) - tailLen // Extract and validate that restart points are strictly ordered and within the valid // data range. for i := 0; i < restartLen; i++ { off := dataEnd + 2*i restarts[i] = binary.BigEndian.Uint16(blob[off : off+2]) if i > 0 && restarts[i] <= restarts[i-1] { return nil, nil, fmt.Errorf("restart out of order, prev: %d, next: %d", restarts[i-1], restarts[i]) } if int(restarts[i]) >= dataEnd { return nil, nil, fmt.Errorf("invalid restart position, restart: %d, size: %d", restarts[i], dataEnd) } } return restarts, blob[:dataEnd], nil } // blockReader is the reader to access the element within a block. type blockReader struct { restarts []uint16 data []byte } // newBlockReader constructs the block reader with the supplied block data. func newBlockReader(blob []byte) (*blockReader, error) { restarts, data, err := parseIndexBlock(blob) if err != nil { return nil, err } return &blockReader{ restarts: restarts, data: data, // safe to own the slice }, nil } // readGreaterThan locates the first element in the block that is greater than // the specified value. If no such element is found, MaxUint64 is returned. func (br *blockReader) readGreaterThan(id uint64) (uint64, error) { var err error index := sort.Search(len(br.restarts), func(i int) bool { item, n := binary.Uvarint(br.data[br.restarts[i]:]) if n <= 0 { err = fmt.Errorf("failed to decode item at restart %d", br.restarts[i]) } return item > id }) if err != nil { return 0, err } if index == 0 { item, _ := binary.Uvarint(br.data[br.restarts[0]:]) return item, nil } var ( start int limit int result uint64 ) if index == len(br.restarts) { // The element being searched falls within the last restart section, // there is no guarantee such element can be found. start = int(br.restarts[len(br.restarts)-1]) limit = len(br.data) } else { // The element being searched falls within the non-last restart section, // such element can be found for sure. start = int(br.restarts[index-1]) limit = int(br.restarts[index]) } pos := start for pos < limit { x, n := binary.Uvarint(br.data[pos:]) if pos == start { result = x } else { result += x } if result > id { return result, nil } pos += n } // The element which is greater than specified id is not found. if index == len(br.restarts) { return math.MaxUint64, nil } // The element which is the first one greater than the specified id // is exactly the one located at the restart point. item, _ := binary.Uvarint(br.data[br.restarts[index]:]) return item, nil } type blockWriter struct { desc *indexBlockDesc // Descriptor of the block restarts []uint16 // Offsets into the data slice, marking the start of each section data []byte // Aggregated encoded data slice } func newBlockWriter(blob []byte, desc *indexBlockDesc) (*blockWriter, error) { if len(blob) == 0 { return &blockWriter{ desc: desc, data: make([]byte, 0, 1024), }, nil } restarts, data, err := parseIndexBlock(blob) if err != nil { return nil, err } return &blockWriter{ desc: desc, restarts: restarts, data: data, // safe to own the slice }, nil } // append adds a new element to the block. The new element must be greater than // the previous one. The provided ID is assumed to always be greater than 0. func (b *blockWriter) append(id uint64) error { if id == 0 { return errors.New("invalid zero id") } if id <= b.desc.max { return fmt.Errorf("append element out of order, last: %d, this: %d", b.desc.max, id) } // Rotate the current restart section if it's full if b.desc.entries%indexBlockRestartLen == 0 { // Save the offset within the data slice as the restart point // for the next section. b.restarts = append(b.restarts, uint16(len(b.data))) // The restart point item can either be encoded in variable // size or fixed size. Although variable-size encoding is // slightly slower (2ns per operation), it is still relatively // fast, therefore, it's picked for better space efficiency. // // The first element in a restart range is encoded using its // full value. b.data = binary.AppendUvarint(b.data, id) } else { // The element which is not the first one in the section // is encoded using the value difference from the preceding // element. b.data = binary.AppendUvarint(b.data, id-b.desc.max) } b.desc.entries++ b.desc.max = id return nil } // scanSection traverses the specified section and terminates if fn returns true. func (b *blockWriter) scanSection(section int, fn func(uint64, int) bool) { var ( value uint64 start = int(b.restarts[section]) pos = start limit int ) if section == len(b.restarts)-1 { limit = len(b.data) } else { limit = int(b.restarts[section+1]) } for pos < limit { x, n := binary.Uvarint(b.data[pos:]) if pos == start { value = x } else { value += x } if fn(value, pos) { return } pos += n } } // sectionLast returns the last element in the specified section. func (b *blockWriter) sectionLast(section int) uint64 { var n uint64 b.scanSection(section, func(v uint64, _ int) bool { n = v return false }) return n } // sectionSearch looks up the specified value in the given section, // the position and the preceding value will be returned if found. func (b *blockWriter) sectionSearch(section int, n uint64) (found bool, prev uint64, pos int) { b.scanSection(section, func(v uint64, p int) bool { if n == v { pos = p found = true return true // terminate iteration } prev = v return false // continue iteration }) return found, prev, pos } // pop removes the last element from the block. The assumption is held that block // writer must be non-empty. func (b *blockWriter) pop(id uint64) error { if id == 0 { return errors.New("invalid zero id") } if id != b.desc.max { return fmt.Errorf("pop element out of order, last: %d, this: %d", b.desc.max, id) } // If there is only one entry left, the entire block should be reset if b.desc.entries == 1 { //b.desc.min = 0 b.desc.max = 0 b.desc.entries = 0 b.restarts = nil b.data = b.data[:0] return nil } // Pop the last restart section if the section becomes empty after removing // one element. if b.desc.entries%indexBlockRestartLen == 1 { b.data = b.data[:b.restarts[len(b.restarts)-1]] b.restarts = b.restarts[:len(b.restarts)-1] b.desc.max = b.sectionLast(len(b.restarts) - 1) b.desc.entries -= 1 return nil } // Look up the element preceding the one to be popped, in order to update // the maximum element in the block. found, prev, pos := b.sectionSearch(len(b.restarts)-1, id) if !found { return fmt.Errorf("pop element is not found, last: %d, this: %d", b.desc.max, id) } b.desc.max = prev b.data = b.data[:pos] b.desc.entries -= 1 return nil } func (b *blockWriter) empty() bool { return b.desc.empty() } func (b *blockWriter) full() bool { return b.desc.full() } // finish finalizes the index block encoding by appending the encoded restart points // and the restart counter to the end of the block. // // This function is safe to be called multiple times. func (b *blockWriter) finish() []byte { buf := make([]byte, len(b.restarts)*2+1) for i, restart := range b.restarts { binary.BigEndian.PutUint16(buf[2*i:], restart) } buf[len(buf)-1] = byte(len(b.restarts)) return append(b.data, buf...) }