Skip to content

Commit 7491587

Browse files
guybrushguybrush
committed
feat: configurable bigtable tables
1 parent b6d1d75 commit 7491587

File tree

5 files changed

+69
-32
lines changed

5 files changed

+69
-32
lines changed

‎backend/pkg/commons/db/bigtable.go‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -113,14 +113,14 @@ func InitBigtable(project, instance, chainId, redisAddress string) (*Bigtable, e
113113

114114
bt := &Bigtable{
115115
client: btClient,
116-
tableData: btClient.Open("data"),
117-
tableBlocks: btClient.Open("blocks"),
118-
tableMetadataUpdates: btClient.Open("metadata_updates"),
119-
tableMetadata: btClient.Open("metadata"),
120-
tableBeaconchain: btClient.Open("beaconchain"),
121-
tableMachineMetrics: btClient.Open("machine_metrics"),
122-
tableValidators: btClient.Open("beaconchain_validators"),
123-
tableValidatorsHistory: btClient.Open("beaconchain_validators_history"),
116+
tableData: btClient.Open(utils.Config.Bigtable.TableNameData),
117+
tableBlocks: btClient.Open(utils.Config.Bigtable.TableNameBlocks),
118+
tableMetadataUpdates: btClient.Open(utils.Config.Bigtable.TableNameMetadataUpdates),
119+
tableMetadata: btClient.Open(utils.Config.Bigtable.TableNameMetadata),
120+
tableBeaconchain: btClient.Open(utils.Config.Bigtable.TableNameBeaconchain),
121+
tableMachineMetrics: btClient.Open(utils.Config.Bigtable.TableNameMachineMetrics),
122+
tableValidators: btClient.Open(utils.Config.Bigtable.TableNameValidators),
123+
tableValidatorsHistory: btClient.Open(utils.Config.Bigtable.TableNameValidatorsHistory),
124124
chainId: chainId,
125125
redisCache: rdc,
126126
LastAttestationCacheMux: &sync.Mutex{},

‎backend/pkg/commons/db/bigtable_common.go‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -83,21 +83,21 @@ func (bigtable *Bigtable) ClearByPrefix(table string, family, columns, prefix st
8383
var btTable *gcp_bigtable.Table
8484

8585
switch table {
86-
case "data":
86+
case utils.Config.Bigtable.TableNameData:
8787
btTable = bigtable.tableData
88-
case "blocks":
88+
case utils.Config.Bigtable.TableNameBlocks:
8989
btTable = bigtable.tableBlocks
90-
case "metadata_updates":
90+
case utils.Config.Bigtable.TableNameMetadataUpdates:
9191
btTable = bigtable.tableMetadataUpdates
92-
case "metadata":
92+
case utils.Config.Bigtable.TableNameMetadata:
9393
btTable = bigtable.tableMetadata
94-
case "beaconchain":
94+
case utils.Config.Bigtable.TableNameBeaconchain:
9595
btTable = bigtable.tableBeaconchain
96-
case "machine_metrics":
96+
case utils.Config.Bigtable.TableNameMachineMetrics:
9797
btTable = bigtable.tableMachineMetrics
98-
case "beaconchain_validators":
98+
case utils.Config.Bigtable.TableNameValidators:
9999
btTable = bigtable.tableValidators
100-
case "beaconchain_validators_history":
100+
case utils.Config.Bigtable.TableNameValidatorsHistory:
101101
btTable = bigtable.tableValidatorsHistory
102102
default:
103103
return fmt.Errorf("unknown table %v provided", table)
@@ -192,7 +192,7 @@ func (bigtable *Bigtable) ClearRawByPrefix(prefix string, dryRun bool) error {
192192

193193
rowRange := gcp_bigtable.PrefixRange(prefix)
194194

195-
btTable := bigtable.client.Open("blocks-raw")
195+
btTable := bigtable.client.Open(utils.Config.Bigtable.TableNameBlocksRaw)
196196

197197
mutsDelete := types.NewBulkMutations(MAX_BATCH_MUTATIONS)
198198

‎backend/pkg/commons/db/bigtable_init.go‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -25,36 +25,36 @@ func InitBigtableSchema() error {
2525
func InitBigtableSchemaIndexed() error {
2626
tables := make(map[string]map[string]gcp_bigtable.GCPolicy)
2727

28-
tables["beaconchain_validators"] = map[string]gcp_bigtable.GCPolicy{
28+
tables[utils.Config.Bigtable.TableNameValidators] = map[string]gcp_bigtable.GCPolicy{
2929
ATTESTATIONS_FAMILY: gcp_bigtable.MaxVersionsGCPolicy(1),
3030
}
31-
tables["beaconchain_validators_history"] = map[string]gcp_bigtable.GCPolicy{
31+
tables[utils.Config.Bigtable.TableNameValidatorsHistory] = map[string]gcp_bigtable.GCPolicy{
3232
VALIDATOR_BALANCES_FAMILY: nil,
3333
VALIDATOR_HIGHEST_ACTIVE_INDEX_FAMILY: nil,
3434
ATTESTATIONS_FAMILY: gcp_bigtable.MaxAgeGCPolicy(utils.Day * 31),
3535
SYNC_COMMITTEES_FAMILY: nil,
3636
INCOME_DETAILS_COLUMN_FAMILY: gcp_bigtable.MaxAgeGCPolicy(utils.Day * 31),
3737
STATS_COLUMN_FAMILY: nil,
3838
}
39-
tables["blocks"] = map[string]gcp_bigtable.GCPolicy{
39+
tables[utils.Config.Bigtable.TableNameBlocks] = map[string]gcp_bigtable.GCPolicy{
4040
DEFAULT_FAMILY_BLOCKS: gcp_bigtable.MaxVersionsGCPolicy(1),
4141
}
42-
tables["data"] = map[string]gcp_bigtable.GCPolicy{
42+
tables[utils.Config.Bigtable.TableNameData] = map[string]gcp_bigtable.GCPolicy{
4343
CONTRACT_METADATA_FAMILY: gcp_bigtable.MaxAgeGCPolicy(utils.Day),
4444
DEFAULT_FAMILY: nil,
4545
}
46-
tables["machine_metrics"] = map[string]gcp_bigtable.GCPolicy{
46+
tables[utils.Config.Bigtable.TableNameMachineMetrics] = map[string]gcp_bigtable.GCPolicy{
4747
MACHINE_METRICS_COLUMN_FAMILY: gcp_bigtable.MaxAgeGCPolicy(utils.Day * 31),
4848
}
49-
tables["metadata"] = map[string]gcp_bigtable.GCPolicy{
49+
tables[utils.Config.Bigtable.TableNameMetadata] = map[string]gcp_bigtable.GCPolicy{
5050
ACCOUNT_METADATA_FAMILY: nil,
5151
CONTRACT_METADATA_FAMILY: nil,
5252
ERC20_METADATA_FAMILY: nil,
5353
ERC721_METADATA_FAMILY: nil,
5454
ERC1155_METADATA_FAMILY: nil,
5555
SERIES_FAMILY: gcp_bigtable.MaxVersionsGCPolicy(1),
5656
}
57-
tables["metadata_updates"] = map[string]gcp_bigtable.GCPolicy{
57+
tables[utils.Config.Bigtable.TableNameMetadataUpdates] = map[string]gcp_bigtable.GCPolicy{
5858
METADATA_UPDATES_FAMILY_BLOCKS: gcp_bigtable.MaxAgeGCPolicy(utils.Day),
5959
DEFAULT_FAMILY: nil,
6060
}
@@ -115,7 +115,7 @@ func InitBigtableSchemaRaw() error {
115115
tables := make(map[string]map[string]gcp_bigtable.GCPolicy)
116116

117117
// these are the families in current production-bigtable: b, v, r, a, s, t, u - not sure why some are missing in the raw-go-module
118-
tables["blocks-raw"] = map[string]gcp_bigtable.GCPolicy{
118+
tables[utils.Config.Bigtable.TableNameBlocksRaw] = map[string]gcp_bigtable.GCPolicy{
119119
raw.BT_COLUMNFAMILY_BLOCK: nil,
120120
"v": nil,
121121
raw.BT_COLUMNFAMILY_RECEIPTS: nil,

‎backend/pkg/commons/types/config.go‎

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,22 @@ import (
88
)
99

1010
type Bigtable struct {
11-
Project string `yaml:"project" env:"PROJECT"`
12-
Instance string `yaml:"instance" env:"INSTANCE"`
13-
Emulator bool `yaml:"emulator" env:"EMULATOR"`
14-
EmulatorPort int `yaml:"emulatorPort" env:"EMULATOR_PORT"`
15-
EmulatorHost string `yaml:"emulatorHost" env:"EMULATOR_HOST"`
16-
V2SchemaCutOffEpoch uint64 `yaml:"v2SchemaCutOffEpoch" env:"V2_SCHEMA_CUTT_OFF_EPOCH"`
17-
Remote string `yaml:"remote"`
11+
Project string `yaml:"project" env:"PROJECT"`
12+
Instance string `yaml:"instance" env:"INSTANCE"`
13+
Emulator bool `yaml:"emulator" env:"EMULATOR"`
14+
EmulatorPort int `yaml:"emulatorPort" env:"EMULATOR_PORT"`
15+
EmulatorHost string `yaml:"emulatorHost" env:"EMULATOR_HOST"`
16+
V2SchemaCutOffEpoch uint64 `yaml:"v2SchemaCutOffEpoch" env:"V2_SCHEMA_CUTT_OFF_EPOCH"`
17+
Remote string `yaml:"remote"`
18+
TableNameBeaconchain string `yaml:"tableNameBeaconchain" env:"TABLE_NAME_BEACONCHAIN"`
19+
TableNameValidators string `yaml:"tableNameBeaconchainValidators" env:"TABLE_NAME_BEACONCHAIN_VALIDATORS"`
20+
TableNameValidatorsHistory string `yaml:"tableNameBeaconchainValidatorsHistory" env:"TABLE_NAME_BEACONCHAIN_VALIDATORS_HISTORY"`
21+
TableNameBlocks string `yaml:"tableNameBlocks" env:"TABLE_NAME_BLOCKS"`
22+
TableNameData string `yaml:"tableNameData" env:"TABLE_NAME_DATA"`
23+
TableNameMachineMetrics string `yaml:"tableNameMachineMetrics" env:"TABLE_NAME_MACHINE_METRICS"`
24+
TableNameMetadata string `yaml:"tableNameMetadata" env:"TABLE_NAME_METADATA"`
25+
TableNameMetadataUpdates string `yaml:"tableNameMetadataUpdates" env:"TABLE_NAME_METADATA_UPDATES"`
26+
TableNameBlocksRaw string `yaml:"tableNameBlocksRaw" env:"TABLE_NAME_BLOCKS_RAW"`
1827
}
1928

2029
// Config is a struct to hold the configuration data

‎backend/pkg/commons/utils/config.go‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -279,6 +279,34 @@ func ReadConfig(cfg *types.Config, path string) error {
279279
cfg.RedisSessionStoreEndpoint = cfg.RedisCacheEndpoint
280280
}
281281

282+
if cfg.Bigtable.TableNameBeaconchain == "" {
283+
cfg.Bigtable.TableNameBeaconchain = "beaconchain"
284+
}
285+
if cfg.Bigtable.TableNameValidators == "" {
286+
cfg.Bigtable.TableNameValidators = "beaconchain_validators"
287+
}
288+
if cfg.Bigtable.TableNameValidatorsHistory == "" {
289+
cfg.Bigtable.TableNameValidatorsHistory = "beaconchain_validators_history"
290+
}
291+
if cfg.Bigtable.TableNameBlocks == "" {
292+
cfg.Bigtable.TableNameBlocks = "blocks"
293+
}
294+
if cfg.Bigtable.TableNameData == "" {
295+
cfg.Bigtable.TableNameData = "data"
296+
}
297+
if cfg.Bigtable.TableNameMachineMetrics == "" {
298+
cfg.Bigtable.TableNameMachineMetrics = "machine_metrics"
299+
}
300+
if cfg.Bigtable.TableNameMetadata == "" {
301+
cfg.Bigtable.TableNameMetadata = "metadata"
302+
}
303+
if cfg.Bigtable.TableNameMetadataUpdates == "" {
304+
cfg.Bigtable.TableNameMetadataUpdates = "metadata_updates"
305+
}
306+
if cfg.Bigtable.TableNameBlocksRaw == "" {
307+
cfg.Bigtable.TableNameBlocksRaw = "blocks-raw"
308+
}
309+
282310
confSanityCheck(cfg)
283311

284312
log.InfoWithFields(log.Fields{

0 commit comments

Comments
 (0)