Skip to content

triedb/pathdb: add history index config #32875

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
+143 −62
Closed

triedb/pathdb: add history index config #32875

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b6da120
triedb/pathdb: add history index config
rjl493456442 Oct 11, 2025
882c135
triedb/pathdb: make batch with pre-allocated size
rjl493456442 Oct 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 18 additions & 11 deletions triedb/pathdb/history_index.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,9 +107,7 @@ func (r *indexReader) refresh() error {
// may have been modified by additional elements written to the disk.
if len(r.descList) != 0 {
last := r.descList[len(r.descList)-1]
if !last.full() {
delete(r.readers, last.id)
}
delete(r.readers, last.id)
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In theory, if the last resolved reader is full, it's unnecessary to reload it for refreshing.

In practice, it's very rare that the last reader is full. In order to avoid passing the block capacity from the top down to the indexReader only for this purpose, this condition is removed.

The correctness shouldn't be affected.

}
descList, err := loadIndexData(r.db, r.state)
if err != nil {
Expand Down Expand Up @@ -158,19 +156,23 @@ type indexWriter struct {
lastID uint64 // The ID of the latest tracked history
state stateIdent
db ethdb.KeyValueReader

// configs
blockCap uint16 // Maximum number of entries grouped into a single index block
}

// newIndexWriter constructs the index writer for the specified state.
func newIndexWriter(db ethdb.KeyValueReader, state stateIdent) (*indexWriter, error) {
func newIndexWriter(db ethdb.KeyValueReader, state stateIdent, blockCap uint16) (*indexWriter, error) {
blob := readStateIndex(state, db)
if len(blob) == 0 {
desc := newIndexBlockDesc(0)
bw, _ := newBlockWriter(nil, desc)
bw, _ := newBlockWriter(nil, desc, blockCap)
return &indexWriter{
descList: []*indexBlockDesc{desc},
bw: bw,
state: state,
db: db,
blockCap: blockCap,
}, nil
}
descList, err := parseIndex(blob)
Expand All @@ -179,7 +181,7 @@ func newIndexWriter(db ethdb.KeyValueReader, state stateIdent) (*indexWriter, er
}
lastDesc := descList[len(descList)-1]
indexBlock := readStateIndexBlock(state, db, lastDesc.id)
bw, err := newBlockWriter(indexBlock, lastDesc)
bw, err := newBlockWriter(indexBlock, lastDesc, blockCap)
if err != nil {
return nil, err
}
Expand All @@ -189,6 +191,7 @@ func newIndexWriter(db ethdb.KeyValueReader, state stateIdent) (*indexWriter, er
bw: bw,
state: state,
db: db,
blockCap: blockCap,
}, nil
}

Expand Down Expand Up @@ -218,7 +221,7 @@ func (w *indexWriter) rotate() error {
desc = newIndexBlockDesc(w.bw.desc.id + 1)
)
w.frozen = append(w.frozen, w.bw)
w.bw, err = newBlockWriter(nil, desc)
w.bw, err = newBlockWriter(nil, desc, w.blockCap)
if err != nil {
return err
}
Expand Down Expand Up @@ -265,21 +268,25 @@ type indexDeleter struct {
lastID uint64 // The ID of the latest tracked history
state stateIdent
db ethdb.KeyValueReader

// configs
blockCap uint16 // Maximum number of entries grouped into a single index block
}

// newIndexDeleter constructs the index deleter for the specified state.
func newIndexDeleter(db ethdb.KeyValueReader, state stateIdent) (*indexDeleter, error) {
func newIndexDeleter(db ethdb.KeyValueReader, state stateIdent, blockCap uint16) (*indexDeleter, error) {
blob := readStateIndex(state, db)
if len(blob) == 0 {
// TODO(rjl493456442) we can probably return an error here,
// deleter with no data is meaningless.
desc := newIndexBlockDesc(0)
bw, _ := newBlockWriter(nil, desc)
bw, _ := newBlockWriter(nil, desc, blockCap)
return &indexDeleter{
descList: []*indexBlockDesc{desc},
bw: bw,
state: state,
db: db,
blockCap: blockCap,
}, nil
}
descList, err := parseIndex(blob)
Expand All @@ -288,7 +295,7 @@ func newIndexDeleter(db ethdb.KeyValueReader, state stateIdent) (*indexDeleter,
}
lastDesc := descList[len(descList)-1]
indexBlock := readStateIndexBlock(state, db, lastDesc.id)
bw, err := newBlockWriter(indexBlock, lastDesc)
bw, err := newBlockWriter(indexBlock, lastDesc, blockCap)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -334,7 +341,7 @@ func (d *indexDeleter) pop(id uint64) error {
// Open the previous block writer for deleting
lastDesc := d.descList[len(d.descList)-1]
indexBlock := readStateIndexBlock(d.state, d.db, lastDesc.id)
bw, err := newBlockWriter(indexBlock, lastDesc)
bw, err := newBlockWriter(indexBlock, lastDesc, d.blockCap)
if err != nil {
return err
}
Expand Down
21 changes: 11 additions & 10 deletions triedb/pathdb/history_index_block.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,8 @@ import (
)

const (
indexBlockDescSize = 14 // The size of index block descriptor
indexBlockEntriesCap = 4096 // The maximum number of entries can be grouped in a block
indexBlockRestartLen = 256 // The restart interval length of index block
historyIndexBatch = 1_000_000 // The number of state history indexes for constructing or deleting as batch
indexBlockDescSize = 14 // The size of index block descriptor
indexBlockRestartLen = 256 // The restart interval length of index block
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

While in theory, the restart size can also be configured, it's not moved into the config in this pull request yet

)

// indexBlockDesc represents a descriptor for an index block, which contains a
Expand All @@ -51,8 +49,8 @@ func (d *indexBlockDesc) empty() bool {

// full indicates whether the number of elements in the block exceeds the
// preconfigured limit.
func (d *indexBlockDesc) full() bool {
return d.entries >= indexBlockEntriesCap
func (d *indexBlockDesc) full(capacity uint16) bool {
return d.entries >= capacity
}

// encode packs index block descriptor into byte stream.
Expand Down Expand Up @@ -222,13 +220,15 @@ type blockWriter struct {
desc *indexBlockDesc // Descriptor of the block
restarts []uint16 // Offsets into the data slice, marking the start of each section
data []byte // Aggregated encoded data slice
capacity uint16 // Maximum number of entries grouped into a single block
}

func newBlockWriter(blob []byte, desc *indexBlockDesc) (*blockWriter, error) {
func newBlockWriter(blob []byte, desc *indexBlockDesc, capacity uint16) (*blockWriter, error) {
if len(blob) == 0 {
return &blockWriter{
desc: desc,
data: make([]byte, 0, 1024),
desc: desc,
data: make([]byte, 0, 1024),
capacity: capacity,
}, nil
}
restarts, data, err := parseIndexBlock(blob)
Expand All @@ -239,6 +239,7 @@ func newBlockWriter(blob []byte, desc *indexBlockDesc) (*blockWriter, error) {
desc: desc,
restarts: restarts,
data: data, // safe to own the slice
capacity: capacity,
}, nil
}

Expand Down Expand Up @@ -372,7 +373,7 @@ func (b *blockWriter) empty() bool {
}

func (b *blockWriter) full() bool {
return b.desc.full()
return b.desc.full(b.capacity)
}

// finish finalizes the index block encoding by appending the encoded restart points
Expand Down
34 changes: 21 additions & 13 deletions triedb/pathdb/history_index_block_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,11 @@ import (
)

func TestBlockReaderBasic(t *testing.T) {
blockCap := uint16(4096)
elements := []uint64{
1, 5, 10, 11, 20,
}
bw, _ := newBlockWriter(nil, newIndexBlockDesc(0))
bw, _ := newBlockWriter(nil, newIndexBlockDesc(0), blockCap)
for i := 0; i < len(elements); i++ {
bw.append(elements[i])
}
Expand Down Expand Up @@ -60,13 +61,14 @@ func TestBlockReaderBasic(t *testing.T) {
}

func TestBlockReaderLarge(t *testing.T) {
blockCap := uint16(4096)
var elements []uint64
for i := 0; i < 1000; i++ {
elements = append(elements, rand.Uint64())
}
slices.Sort(elements)

bw, _ := newBlockWriter(nil, newIndexBlockDesc(0))
bw, _ := newBlockWriter(nil, newIndexBlockDesc(0), blockCap)
for i := 0; i < len(elements); i++ {
bw.append(elements[i])
}
Expand Down Expand Up @@ -95,7 +97,8 @@ func TestBlockReaderLarge(t *testing.T) {
}

func TestBlockWriterBasic(t *testing.T) {
bw, _ := newBlockWriter(nil, newIndexBlockDesc(0))
blockCap := uint16(4096)
bw, _ := newBlockWriter(nil, newIndexBlockDesc(0), blockCap)
if !bw.empty() {
t.Fatal("expected empty block")
}
Expand All @@ -107,7 +110,7 @@ func TestBlockWriterBasic(t *testing.T) {
bw.append(uint64(i + 3))
}

bw, err := newBlockWriter(bw.finish(), newIndexBlockDesc(0))
bw, err := newBlockWriter(bw.finish(), newIndexBlockDesc(0), blockCap)
if err != nil {
t.Fatalf("Failed to construct the block writer, %v", err)
}
Expand All @@ -120,7 +123,8 @@ func TestBlockWriterBasic(t *testing.T) {
}

func TestBlockWriterDelete(t *testing.T) {
bw, _ := newBlockWriter(nil, newIndexBlockDesc(0))
blockCap := uint16(4096)
bw, _ := newBlockWriter(nil, newIndexBlockDesc(0), blockCap)
for i := 0; i < 10; i++ {
bw.append(uint64(i + 1))
}
Expand All @@ -144,10 +148,11 @@ func TestBlockWriterDelete(t *testing.T) {
}

func TestBlcokWriterDeleteWithData(t *testing.T) {
blockCap := uint16(4096)
elements := []uint64{
1, 5, 10, 11, 20,
}
bw, _ := newBlockWriter(nil, newIndexBlockDesc(0))
bw, _ := newBlockWriter(nil, newIndexBlockDesc(0), blockCap)
for i := 0; i < len(elements); i++ {
bw.append(elements[i])
}
Expand All @@ -158,7 +163,7 @@ func TestBlcokWriterDeleteWithData(t *testing.T) {
max: 20,
entries: 5,
}
bw, err := newBlockWriter(bw.finish(), desc)
bw, err := newBlockWriter(bw.finish(), desc, blockCap)
if err != nil {
t.Fatalf("Failed to construct block writer %v", err)
}
Expand Down Expand Up @@ -201,15 +206,16 @@ func TestBlcokWriterDeleteWithData(t *testing.T) {
}

func TestCorruptedIndexBlock(t *testing.T) {
bw, _ := newBlockWriter(nil, newIndexBlockDesc(0))
blockCap := uint16(4096)
bw, _ := newBlockWriter(nil, newIndexBlockDesc(0), blockCap)
for i := 0; i < 10; i++ {
bw.append(uint64(i + 1))
}
buf := bw.finish()

// Mutate the buffer manually
buf[len(buf)-1]++
_, err := newBlockWriter(buf, newIndexBlockDesc(0))
_, err := newBlockWriter(buf, newIndexBlockDesc(0), blockCap)
if err == nil {
t.Fatal("Corrupted index block data is not detected")
}
Expand All @@ -218,8 +224,9 @@ func TestCorruptedIndexBlock(t *testing.T) {
// BenchmarkParseIndexBlock benchmarks the performance of parseIndexBlock.
func BenchmarkParseIndexBlock(b *testing.B) {
// Generate a realistic index block blob
bw, _ := newBlockWriter(nil, newIndexBlockDesc(0))
for i := 0; i < 4096; i++ {
blockCap := uint16(4096)
bw, _ := newBlockWriter(nil, newIndexBlockDesc(0), blockCap)
for i := 0; i < int(blockCap); i++ {
bw.append(uint64(i * 2))
}
blob := bw.finish()
Expand All @@ -238,13 +245,14 @@ func BenchmarkBlockWriterAppend(b *testing.B) {
b.ReportAllocs()
b.ResetTimer()

blockCap := uint16(4096)
desc := newIndexBlockDesc(0)
writer, _ := newBlockWriter(nil, desc)
writer, _ := newBlockWriter(nil, desc, blockCap)

for i := 0; i < b.N; i++ {
if writer.full() {
desc = newIndexBlockDesc(0)
writer, _ = newBlockWriter(nil, desc)
writer, _ = newBlockWriter(nil, desc, blockCap)
}
if err := writer.append(writer.desc.max + 1); err != nil {
b.Error(err)
Expand Down
Loading