Skip to content

Commit a7d9b52

Browse files
s1narjl493456442fjllightclient
authored
core/rawdb: integrate eradb backend for RPC (#31604)
This implements a backing store for chain history based on era1 files. The new store is integrated with the freezer. Queries for blocks and receipts below the current freezer tail are handled by the era store. --------- Co-authored-by: Gary Rong <garyrong0905@gmail.com> Co-authored-by: Felix Lange <fjl@twurst.com> Co-authored-by: lightclient <lightclient@protonmail.com>
1 parent c87b856 commit a7d9b52

34 files changed

+773
-270
lines changed

‎cmd/geth/chaincmd.go

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ import (
4545
"github.com/ethereum/go-ethereum/internal/era/eradl"
4646
"github.com/ethereum/go-ethereum/internal/flags"
4747
"github.com/ethereum/go-ethereum/log"
48+
"github.com/ethereum/go-ethereum/node"
4849
"github.com/ethereum/go-ethereum/params"
4950
"github.com/urfave/cli/v2"
5051
)
@@ -277,10 +278,7 @@ func initGenesis(ctx *cli.Context) error {
277278
overrides.OverrideVerkle = &v
278279
}
279280

280-
chaindb, err := stack.OpenDatabaseWithFreezer("chaindata", 0, 0, ctx.String(utils.AncientFlag.Name), "", false)
281-
if err != nil {
282-
utils.Fatalf("Failed to open database: %v", err)
283-
}
281+
chaindb := utils.MakeChainDatabase(ctx, stack, false)
284282
defer chaindb.Close()
285283

286284
triedb := utils.MakeTrieDatabase(ctx, chaindb, ctx.Bool(utils.CachePreimagesFlag.Name), false, genesis.IsVerkle())
@@ -317,7 +315,7 @@ func dumpGenesis(ctx *cli.Context) error {
317315
// dump whatever already exists in the datadir
318316
stack, _ := makeConfigNode(ctx)
319317

320-
db, err := stack.OpenDatabase("chaindata", 0, 0, "", true)
318+
db, err := stack.OpenDatabaseWithOptions("chaindata", node.DatabaseOptions{ReadOnly: true})
321319
if err != nil {
322320
return err
323321
}

‎cmd/utils/flags.go

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,11 @@ var (
111111
Usage: "Root directory for ancient data (default = inside chaindata)",
112112
Category: flags.EthCategory,
113113
}
114+
EraFlag = &flags.DirectoryFlag{
115+
Name: "datadir.era",
116+
Usage: "Root directory for era1 history (default = inside ancient/chain)",
117+
Category: flags.EthCategory,
118+
}
114119
MinFreeDiskSpaceFlag = &flags.DirectoryFlag{
115120
Name: "datadir.minfreedisk",
116121
Usage: "Minimum free disk space in MB, once reached triggers auto shut down (default = --cache.gc converted to MB, 0 = disabled)",
@@ -977,6 +982,7 @@ var (
977982
DatabaseFlags = []cli.Flag{
978983
DataDirFlag,
979984
AncientFlag,
985+
EraFlag,
980986
RemoteDBFlag,
981987
DBEngineFlag,
982988
StateSchemeFlag,
@@ -1613,6 +1619,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
16131619
if ctx.IsSet(AncientFlag.Name) {
16141620
cfg.DatabaseFreezer = ctx.String(AncientFlag.Name)
16151621
}
1622+
if ctx.IsSet(EraFlag.Name) {
1623+
cfg.DatabaseEra = ctx.String(EraFlag.Name)
1624+
}
16161625

16171626
if gcmode := ctx.String(GCModeFlag.Name); gcmode != "full" && gcmode != "archive" {
16181627
Fatalf("--%s must be either 'full' or 'archive'", GCModeFlag.Name)
@@ -2082,7 +2091,15 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node, readonly bool) ethdb.
20822091
}
20832092
chainDb = remotedb.New(client)
20842093
default:
2085-
chainDb, err = stack.OpenDatabaseWithFreezer("chaindata", cache, handles, ctx.String(AncientFlag.Name), "eth/db/chaindata/", readonly)
2094+
options := node.DatabaseOptions{
2095+
ReadOnly: readonly,
2096+
Cache: cache,
2097+
Handles: handles,
2098+
AncientsDirectory: ctx.String(AncientFlag.Name),
2099+
MetricsNamespace: "eth/db/chaindata/",
2100+
EraDirectory: ctx.String(EraFlag.Name),
2101+
}
2102+
chainDb, err = stack.OpenDatabaseWithOptions("chaindata", options)
20862103
}
20872104
if err != nil {
20882105
Fatalf("Could not open database: %v", err)

‎cmd/utils/history_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -158,7 +158,7 @@ func TestHistoryImportAndExport(t *testing.T) {
158158
}
159159

160160
// Now import Era.
161-
db2, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
161+
db2, err := rawdb.Open(rawdb.NewMemoryDatabase(), rawdb.OpenOptions{})
162162
if err != nil {
163163
panic(err)
164164
}

‎common/lru/basiclru.go

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -47,30 +47,37 @@ func NewBasicLRU[K comparable, V any](capacity int) BasicLRU[K, V] {
4747

4848
// Add adds a value to the cache. Returns true if an item was evicted to store the new item.
4949
func (c *BasicLRU[K, V]) Add(key K, value V) (evicted bool) {
50+
_, _, evicted = c.Add3(key, value)
51+
return evicted
52+
}
53+
54+
// Add3 adds a value to the cache. If an item was evicted to store the new one, it returns the evicted item.
55+
func (c *BasicLRU[K, V]) Add3(key K, value V) (ek K, ev V, evicted bool) {
5056
item, ok := c.items[key]
5157
if ok {
52-
// Already exists in cache.
5358
item.value = value
5459
c.items[key] = item
5560
c.list.moveToFront(item.elem)
56-
return false
61+
return ek, ev, false
5762
}
5863

5964
var elem *listElem[K]
6065
if c.Len() >= c.cap {
6166
elem = c.list.removeLast()
62-
delete(c.items, elem.v)
6367
evicted = true
68+
ek = elem.v
69+
ev = c.items[ek].value
70+
delete(c.items, ek)
6471
} else {
6572
elem = new(listElem[K])
6673
}
6774

6875
// Store the new item.
69-
// Note that, if another item was evicted, we re-use its list element here.
76+
// Note that if another item was evicted, we re-use its list element here.
7077
elem.v = key
7178
c.items[key] = cacheItem[K, V]{elem, value}
7279
c.list.pushElem(elem)
73-
return evicted
80+
return ek, ev, evicted
7481
}
7582

7683
// Contains reports whether the given key exists in the cache.

‎core/blockchain_repair_test.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1769,7 +1769,7 @@ func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme s
17691769
if err != nil {
17701770
t.Fatalf("Failed to create persistent key-value database: %v", err)
17711771
}
1772-
db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false)
1772+
db, err := rawdb.Open(pdb, rawdb.OpenOptions{Ancient: ancient})
17731773
if err != nil {
17741774
t.Fatalf("Failed to create persistent freezer database: %v", err)
17751775
}
@@ -1854,7 +1854,7 @@ func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme s
18541854
if err != nil {
18551855
t.Fatalf("Failed to reopen persistent key-value database: %v", err)
18561856
}
1857-
db, err = rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false)
1857+
db, err = rawdb.Open(pdb, rawdb.OpenOptions{Ancient: ancient})
18581858
if err != nil {
18591859
t.Fatalf("Failed to reopen persistent freezer database: %v", err)
18601860
}
@@ -1919,7 +1919,7 @@ func testIssue23496(t *testing.T, scheme string) {
19191919
if err != nil {
19201920
t.Fatalf("Failed to create persistent key-value database: %v", err)
19211921
}
1922-
db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false)
1922+
db, err := rawdb.Open(pdb, rawdb.OpenOptions{Ancient: ancient})
19231923
if err != nil {
19241924
t.Fatalf("Failed to create persistent freezer database: %v", err)
19251925
}
@@ -1979,7 +1979,7 @@ func testIssue23496(t *testing.T, scheme string) {
19791979
if err != nil {
19801980
t.Fatalf("Failed to reopen persistent key-value database: %v", err)
19811981
}
1982-
db, err = rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false)
1982+
db, err = rawdb.Open(pdb, rawdb.OpenOptions{Ancient: ancient})
19831983
if err != nil {
19841984
t.Fatalf("Failed to reopen persistent freezer database: %v", err)
19851985
}

‎core/blockchain_sethead_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1973,7 +1973,7 @@ func testSetHeadWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme
19731973
if err != nil {
19741974
t.Fatalf("Failed to create persistent key-value database: %v", err)
19751975
}
1976-
db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false)
1976+
db, err := rawdb.Open(pdb, rawdb.OpenOptions{Ancient: ancient})
19771977
if err != nil {
19781978
t.Fatalf("Failed to create persistent freezer database: %v", err)
19791979
}

‎core/blockchain_snapshot_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo
7070
if err != nil {
7171
t.Fatalf("Failed to create persistent key-value database: %v", err)
7272
}
73-
db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false)
73+
db, err := rawdb.Open(pdb, rawdb.OpenOptions{Ancient: ancient})
7474
if err != nil {
7575
t.Fatalf("Failed to create persistent freezer database: %v", err)
7676
}
@@ -265,7 +265,7 @@ func (snaptest *crashSnapshotTest) test(t *testing.T) {
265265
if err != nil {
266266
t.Fatalf("Failed to create persistent key-value database: %v", err)
267267
}
268-
newdb, err := rawdb.NewDatabaseWithFreezer(pdb, snaptest.ancient, "", false)
268+
newdb, err := rawdb.Open(pdb, rawdb.OpenOptions{Ancient: snaptest.ancient})
269269
if err != nil {
270270
t.Fatalf("Failed to create persistent freezer database: %v", err)
271271
}

‎core/blockchain_test.go

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -738,7 +738,7 @@ func testFastVsFullChains(t *testing.T, scheme string) {
738738
t.Fatalf("failed to insert receipt %d: %v", n, err)
739739
}
740740
// Freezer style fast import the chain.
741-
ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
741+
ancientDb, err := rawdb.Open(rawdb.NewMemoryDatabase(), rawdb.OpenOptions{})
742742
if err != nil {
743743
t.Fatalf("failed to create temp freezer db: %v", err)
744744
}
@@ -824,7 +824,7 @@ func testLightVsFastVsFullChainHeads(t *testing.T, scheme string) {
824824

825825
// makeDb creates a db instance for testing.
826826
makeDb := func() ethdb.Database {
827-
db, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
827+
db, err := rawdb.Open(rawdb.NewMemoryDatabase(), rawdb.OpenOptions{})
828828
if err != nil {
829829
t.Fatalf("failed to create temp freezer db: %v", err)
830830
}
@@ -1623,7 +1623,7 @@ func testLargeReorgTrieGC(t *testing.T, scheme string) {
16231623
competitor, _ := GenerateChain(genesis.Config, shared[len(shared)-1], engine, genDb, 2*state.TriesInMemory+1, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{3}) })
16241624

16251625
// Import the shared chain and the original canonical one
1626-
db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
1626+
db, _ := rawdb.Open(rawdb.NewMemoryDatabase(), rawdb.OpenOptions{})
16271627
defer db.Close()
16281628

16291629
chain, err := NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), genesis, nil, engine, vm.Config{}, nil)
@@ -1689,7 +1689,7 @@ func testBlockchainRecovery(t *testing.T, scheme string) {
16891689
_, blocks, receipts := GenerateChainWithGenesis(gspec, ethash.NewFaker(), int(height), nil)
16901690

16911691
// Import the chain as a ancient-first node and ensure all pointers are updated
1692-
ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false)
1692+
ancientDb, err := rawdb.Open(rawdb.NewMemoryDatabase(), rawdb.OpenOptions{Ancient: t.TempDir()})
16931693
if err != nil {
16941694
t.Fatalf("failed to create temp freezer db: %v", err)
16951695
}
@@ -1747,7 +1747,7 @@ func testLowDiffLongChain(t *testing.T, scheme string) {
17471747
})
17481748

17491749
// Import the canonical chain
1750-
diskdb, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
1750+
diskdb, _ := rawdb.Open(rawdb.NewMemoryDatabase(), rawdb.OpenOptions{})
17511751
defer diskdb.Close()
17521752

17531753
chain, err := NewBlockChain(diskdb, DefaultCacheConfigWithScheme(scheme), genesis, nil, engine, vm.Config{}, nil)
@@ -1959,7 +1959,7 @@ func testInsertKnownChainData(t *testing.T, typ string, scheme string) {
19591959
b.OffsetTime(-9) // A higher difficulty
19601960
})
19611961
// Import the shared chain and the original canonical one
1962-
chaindb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
1962+
chaindb, err := rawdb.Open(rawdb.NewMemoryDatabase(), rawdb.OpenOptions{})
19631963
if err != nil {
19641964
t.Fatalf("failed to create temp freezer db: %v", err)
19651965
}
@@ -2122,7 +2122,7 @@ func testInsertKnownChainDataWithMerging(t *testing.T, typ string, mergeHeight i
21222122
}
21232123
})
21242124
// Import the shared chain and the original canonical one
2125-
chaindb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
2125+
chaindb, err := rawdb.Open(rawdb.NewMemoryDatabase(), rawdb.OpenOptions{})
21262126
if err != nil {
21272127
t.Fatalf("failed to create temp freezer db: %v", err)
21282128
}
@@ -2496,7 +2496,7 @@ func testSideImportPrunedBlocks(t *testing.T, scheme string) {
24962496
if err != nil {
24972497
t.Fatalf("Failed to create persistent key-value database: %v", err)
24982498
}
2499-
db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false)
2499+
db, err := rawdb.Open(pdb, rawdb.OpenOptions{Ancient: ancient})
25002500
if err != nil {
25012501
t.Fatalf("Failed to create persistent freezer database: %v", err)
25022502
}
@@ -3403,7 +3403,7 @@ func testSetCanonical(t *testing.T, scheme string) {
34033403
}
34043404
gen.AddTx(tx)
34053405
})
3406-
diskdb, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
3406+
diskdb, _ := rawdb.Open(rawdb.NewMemoryDatabase(), rawdb.OpenOptions{})
34073407
defer diskdb.Close()
34083408

34093409
chain, err := NewBlockChain(diskdb, DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil)
@@ -4199,7 +4199,7 @@ func testChainReorgSnapSync(t *testing.T, ancientLimit uint64) {
41994199
gen.SetCoinbase(common.Address{0: byte(0xb), 19: byte(i)})
42004200
})
42014201

4202-
db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
4202+
db, _ := rawdb.Open(rawdb.NewMemoryDatabase(), rawdb.OpenOptions{})
42034203
defer db.Close()
42044204

42054205
chain, _ := NewBlockChain(db, DefaultCacheConfigWithScheme(rawdb.PathScheme), gspec, nil, beacon.New(ethash.NewFaker()), vm.Config{}, nil)
@@ -4315,7 +4315,7 @@ func testInsertChainWithCutoff(t *testing.T, cutoff uint64, ancientLimit uint64,
43154315
config := DefaultCacheConfigWithScheme(rawdb.PathScheme)
43164316
config.ChainHistoryMode = history.KeepPostMerge
43174317

4318-
db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
4318+
db, _ := rawdb.Open(rawdb.NewMemoryDatabase(), rawdb.OpenOptions{})
43194319
defer db.Close()
43204320
chain, _ := NewBlockChain(db, DefaultCacheConfigWithScheme(rawdb.PathScheme), genesis, nil, beacon.New(ethash.NewFaker()), vm.Config{}, nil)
43214321
defer chain.Stop()

‎core/rawdb/accessors_chain_test.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -416,7 +416,7 @@ func checkReceiptsRLP(have, want types.Receipts) error {
416416
func TestAncientStorage(t *testing.T) {
417417
// Freezer style fast import the chain.
418418
frdir := t.TempDir()
419-
db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), frdir, "", false)
419+
db, err := Open(NewMemoryDatabase(), OpenOptions{Ancient: frdir})
420420
if err != nil {
421421
t.Fatalf("failed to create database with ancient backend")
422422
}
@@ -469,7 +469,7 @@ func TestAncientStorage(t *testing.T) {
469469
}
470470

471471
func TestWriteAncientHeaderChain(t *testing.T) {
472-
db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), t.TempDir(), "", false)
472+
db, err := Open(NewMemoryDatabase(), OpenOptions{Ancient: t.TempDir()})
473473
if err != nil {
474474
t.Fatalf("failed to create database with ancient backend")
475475
}
@@ -586,7 +586,7 @@ func TestHashesInRange(t *testing.T) {
586586
func BenchmarkWriteAncientBlocks(b *testing.B) {
587587
// Open freezer database.
588588
frdir := b.TempDir()
589-
db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), frdir, "", false)
589+
db, err := Open(NewMemoryDatabase(), OpenOptions{Ancient: frdir})
590590
if err != nil {
591591
b.Fatalf("failed to create database with ancient backend")
592592
}
@@ -890,7 +890,7 @@ func TestHeadersRLPStorage(t *testing.T) {
890890
// Have N headers in the freezer
891891
frdir := t.TempDir()
892892

893-
db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), frdir, "", false)
893+
db, err := Open(NewMemoryDatabase(), OpenOptions{Ancient: frdir})
894894
if err != nil {
895895
t.Fatalf("failed to create database with ancient backend")
896896
}

‎core/rawdb/ancienttest/testsuite.go

Lines changed: 0 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -77,13 +77,6 @@ func basicRead(t *testing.T, newFn func(kinds []string) ethdb.AncientStore) {
7777
}
7878
for _, c := range cases {
7979
for i := c.start; i < c.limit; i++ {
80-
exist, err := db.HasAncient("a", uint64(i))
81-
if err != nil {
82-
t.Fatalf("Failed to check presence, %v", err)
83-
}
84-
if exist {
85-
t.Fatalf("Item %d is already truncated", uint64(i))
86-
}
8780
_, err = db.Ancient("a", uint64(i))
8881
if err == nil {
8982
t.Fatal("Error is expected for non-existent item")
@@ -93,13 +86,6 @@ func basicRead(t *testing.T, newFn func(kinds []string) ethdb.AncientStore) {
9386

9487
// Test the items in range should be reachable
9588
for i := 10; i < 90; i++ {
96-
exist, err := db.HasAncient("a", uint64(i))
97-
if err != nil {
98-
t.Fatalf("Failed to check presence, %v", err)
99-
}
100-
if !exist {
101-
t.Fatalf("Item %d is missing", uint64(i))
102-
}
10389
blob, err := db.Ancient("a", uint64(i))
10490
if err != nil {
10591
t.Fatalf("Failed to retrieve item, %v", err)
@@ -110,13 +96,6 @@ func basicRead(t *testing.T, newFn func(kinds []string) ethdb.AncientStore) {
11096
}
11197

11298
// Test the items in unknown table shouldn't be reachable
113-
exist, err := db.HasAncient("b", uint64(0))
114-
if err != nil {
115-
t.Fatalf("Failed to check presence, %v", err)
116-
}
117-
if exist {
118-
t.Fatal("Item in unknown table shouldn't be found")
119-
}
12099
_, err = db.Ancient("b", uint64(0))
121100
if err == nil {
122101
t.Fatal("Error is expected for unknown table")

0 commit comments

Comments
 (0)