using local redis cache for development

remoterami · remoterami · commit a6d5ab2bd05d · 2025-05-26T16:12:33.000+02:00
diff --git a/backend/pkg/api/data_access/data_access.go b/backend/pkg/api/data_access/data_access.go
@@ -57,6 +57,7 @@ type DataAccessService struct {
 	userWriter              *sqlx.DB
 	bigtable                *db.Bigtable
 	persistentRedisDbClient *redis.Client
+	localRedisDbClient      *redis.Client
 
 	services *services.Services
 	config   *types.Config
@@ -79,6 +80,7 @@ func NewDataAccessService(cfg *types.Config) *DataAccessService {
 	db.ClickHouseReader = das.clickhouseReader
 	db.BigtableClient = das.bigtable
 	db.PersistentRedisDbClient = das.persistentRedisDbClient
+	db.LocalRedisDbClient = das.localRedisDbClient
 
 	return das
 }
@@ -147,6 +149,21 @@ func createDataAccessService(cfg *types.Config) *DataAccessService {
 		dataAccessService.persistentRedisDbClient = rdc
 	}()
 
+	// Initialize the persistent redis client
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		rdc := redis.NewClient(&redis.Options{
+			Addr:        cfg.RedisLocalCacheEndpoint,
+			ReadTimeout: time.Second * 60,
+		})
+
+		if err := rdc.Ping(context.Background()).Err(); err != nil {
+			log.Fatal(err, "error connecting to local redis cache", 0)
+		}
+		dataAccessService.localRedisDbClient = rdc
+	}()
+
 	wg.Wait()
 
 	if cfg.TieredCacheProvider != "redis" {
@@ -159,7 +176,7 @@ func createDataAccessService(cfg *types.Config) *DataAccessService {
 
 func (d *DataAccessService) StartDataAccessServices() {
 	// Create the services
-	d.services = services.NewServices(d.readerDb, d.writerDb, d.clickhouseReader, d.bigtable, d.persistentRedisDbClient)
+	d.services = services.NewServices(d.readerDb, d.writerDb, d.clickhouseReader, d.bigtable, d.persistentRedisDbClient, d.localRedisDbClient)
 
 	// Initialize repositories
 	d.registerNotificationInterfaceTypes()
diff --git a/backend/pkg/api/services/service.go b/backend/pkg/api/services/service.go
@@ -21,15 +21,17 @@ type Services struct {
 	clickhouseReader        *sqlx.DB
 	bigtable                *db.Bigtable
 	persistentRedisDbClient *redis.Client
+	localRedisDbClient      *redis.Client
 }
 
-func NewServices(readerDb, writerDb, clickhouseReader *sqlx.DB, bigtable *db.Bigtable, persistentRedisDbClient *redis.Client) *Services {
+func NewServices(readerDb, writerDb, clickhouseReader *sqlx.DB, bigtable *db.Bigtable, persistentRedisDbClient, localRedisDbClient *redis.Client) *Services {
 	return &Services{
 		readerDb:                readerDb,
 		writerDb:                writerDb,
 		clickhouseReader:        clickhouseReader,
 		bigtable:                bigtable,
 		persistentRedisDbClient: persistentRedisDbClient,
+		localRedisDbClient:      localRedisDbClient,
 	}
 }
 
diff --git a/backend/pkg/api/services/service_slot_viz.go b/backend/pkg/api/services/service_slot_viz.go
@@ -1,31 +1,17 @@
 package services
 
 import (
-	"bytes"
-	"context"
-	"encoding/gob"
 	"fmt"
-	"strconv"
-	"strings"
 	"sync"
-	"sync/atomic"
 	"time"
 
-	"github.com/gobitfly/beaconchain/pkg/commons/cache"
-	"github.com/gobitfly/beaconchain/pkg/commons/db"
 	"github.com/gobitfly/beaconchain/pkg/commons/log"
-	"github.com/gobitfly/beaconchain/pkg/commons/types"
 	"github.com/gobitfly/beaconchain/pkg/commons/utils"
-	constypes "github.com/gobitfly/beaconchain/pkg/consapi/types"
 	"github.com/gobitfly/beaconchain/pkg/monitoring/constants"
 	"github.com/gobitfly/beaconchain/pkg/monitoring/services"
-	"github.com/juliangruber/go-intersect"
-	"github.com/lib/pq"
-	"github.com/pkg/errors"
-	"golang.org/x/sync/errgroup"
 )
 
-var currentDutiesInfo atomic.Pointer[SyncData]
+// var currentDutiesInfo atomic.Pointer[SyncData]
 
 func (s *Services) startSlotVizDataService(wg *sync.WaitGroup) {
 	o := sync.Once{}
@@ -49,7 +35,8 @@ func (s *Services) startSlotVizDataService(wg *sync.WaitGroup) {
 }
 
 func (s *Services) updateSlotVizData() error {
-	var dutiesInfo *SyncData
+	return nil
+	/*var dutiesInfo *SyncData
 	if currentDutiesInfo.Load() == nil {
 		dutiesInfo = s.initDutiesInfo()
 	} else {
@@ -287,19 +274,39 @@ func (s *Services) updateSlotVizData() error {
 
 	currentDutiesInfo.Store(dutiesInfo)
 
-	return nil
+	return nil*/
 }
 
 // GetCurrentDutiesInfo returns the current duties info and a function to release the lock
 // Call release lock after you are done with accessing the data, otherwise it will block the slot viz service from updating
 func (s *Services) GetCurrentDutiesInfo() (*SyncData, error) {
-	if currentDutiesInfo.Load() == nil {
+	dutiesInfo := SyncData{}
+	// dummy data to avoid downloading duties from redis
+	dutiesInfo.PropAssignmentsForSlot = make(map[uint64]uint64)
+	// validators in megatron
+	dutiesInfo.PropAssignmentsForSlot[3_000_000] = 1
+	dutiesInfo.PropAssignmentsForSlot[3_000_001] = 7
+	dutiesInfo.PropAssignmentsForSlot[3_000_002] = 13
+	dutiesInfo.PropAssignmentsForSlot[3_000_003] = 19
+	dutiesInfo.PropAssignmentsForSlot[3_000_004] = 25
+	dutiesInfo.PropAssignmentsForSlot[3_000_005] = 31
+	// and some others
+	dutiesInfo.PropAssignmentsForSlot[3_000_006] = 29141
+	dutiesInfo.PropAssignmentsForSlot[3_000_007] = 500167
+	dutiesInfo.PropAssignmentsForSlot[3_000_008] = 531705
+	dutiesInfo.PropAssignmentsForSlot[3_000_009] = 542402
+	dutiesInfo.PropAssignmentsForSlot[3_000_010] = 845851
+	dutiesInfo.PropAssignmentsForSlot[3_000_011] = 894749
+
+	return &dutiesInfo, nil
+
+	/*if currentDutiesInfo.Load() == nil {
 		return nil, fmt.Errorf("%w: dutiesInfo", ErrWaiting)
 	}
-	return currentDutiesInfo.Load(), nil
+	return currentDutiesInfo.Load(), nil*/
 }
 
-func (s *Services) initDutiesInfo() *SyncData {
+/*func (s *Services) initDutiesInfo() *SyncData {
 	dutiesInfo := SyncData{}
 	dutiesInfo.LatestSlot = uint64(0)
 	dutiesInfo.LatestProposedSlot = uint64(0)
@@ -433,9 +440,9 @@ func (s *Services) copyAndCleanDutiesInfo() *SyncData {
 		}
 	}
 	return dutiesInfo
-}
+}*/
 
-func (s *Services) getMaxValidatorDutiesInfoSlot() uint64 {
+/*func (s *Services) getMaxValidatorDutiesInfoSlot() uint64 {
 	headEpoch := cache.LatestEpoch.Get()
 	slotsPerEpoch := utils.Config.Chain.ClConfig.SlotsPerEpoch
 
@@ -451,15 +458,15 @@ func (s *Services) getMaxValidatorDutiesInfoSlot() uint64 {
 		- Attestation data amount is the main culprit for the database call since it returns huge amounts of data
 		- Other fields used by slotviz do not change as well (sync bits, exec block). If we at some point include changing fields for headEpoch -2
 		  we should consider making this a separate call to avoid loading all attestation data again
-	*/
+
 	if err == nil && p.AssignmentsFetchedForEpoch > 0 && headEpoch > 0 { // if we have fetched epoch assignments before
 		minEpoch = headEpoch - 1
 	}
 
 	maxValidatorDutiesInfoSlot := minEpoch * slotsPerEpoch
 
 	return maxValidatorDutiesInfoSlot
-}
+}*/
 
 type SyncData struct {
 	LatestSlot                   uint64
diff --git a/backend/pkg/api/services/service_validator_mapping.go b/backend/pkg/api/services/service_validator_mapping.go
@@ -91,7 +91,7 @@ func (s *Services) updateValidatorMapping() error {
 	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
 	defer cancel()
 	key := fmt.Sprintf("%d:%s", utils.Config.Chain.ClConfig.DepositChainID, "vm")
-	compressed, err := s.persistentRedisDbClient.Get(ctx, key).Bytes()
+	compressed, err := s.localRedisDbClient.Get(ctx, key).Bytes()
 	if err != nil {
 		return errors.Wrap(err, "failed to get compressed validator mapping from db")
 	}
diff --git a/backend/pkg/commons/db/db.go b/backend/pkg/commons/db/db.go
@@ -48,6 +48,7 @@ var ClickHouseReader *sqlx.DB
 var ClickHouseWriter *sqlx.DB
 
 var PersistentRedisDbClient *redis.Client
+var LocalRedisDbClient *redis.Client
 
 var FarFutureEpoch = uint64(18446744073709551615)
 var MaxSqlNumber = uint64(9223372036854775807)
diff --git a/backend/pkg/commons/types/config.go b/backend/pkg/commons/types/config.go
@@ -33,6 +33,7 @@ type Config struct {
 	EtherscanAPIKey            string `yaml:"etherscanApiKey" env:"ETHERSCAN_API_KEY"`
 	EtherscanAPIBaseURL        string `yaml:"etherscanApiBaseUrl" env:"ETHERSCAN_API_BASEURL"`
 	RedisCacheEndpoint         string `yaml:"redisCacheEndpoint" env:"REDIS_CACHE_ENDPOINT"`
+	RedisLocalCacheEndpoint    string `yaml:"redisLocalCacheEndpoint" env:"REDIS_LOCAL_CACHE_ENDPOINT"`
 	RedisSessionStoreEndpoint  string `yaml:"redisSessionStoreEndpoint" env:"REDIS_SESSION_STORE_ENDPOINT"`
 	TieredCacheProvider        string `yaml:"tieredCacheProvider" env:"CACHE_PROVIDER"`
 	ReportServiceStatus        bool   `yaml:"reportServiceStatus" env:"REPORT_SERVICE_STATUS"`
diff --git a/backend/pkg/commons/utils/config.go b/backend/pkg/commons/utils/config.go
@@ -279,6 +279,11 @@ func ReadConfig(cfg *types.Config, path string) error {
 		cfg.RedisSessionStoreEndpoint = cfg.RedisCacheEndpoint
 	}
 
+	if cfg.RedisLocalCacheEndpoint == "" && cfg.RedisCacheEndpoint != "" {
+		log.Warnf("using RedisCacheEndpoint %s as RedisLocalCacheEndpoint as no dedicated RedisLocalCacheEndpoint was provided", cfg.RedisCacheEndpoint)
+		cfg.RedisLocalCacheEndpoint = cfg.RedisCacheEndpoint
+	}
+
 	confSanityCheck(cfg)
 
 	log.InfoWithFields(log.Fields{

Original file line number	Diff line number	Diff line change
`@@ -21,15 +21,17 @@ type Services struct {`
`21`	`21`	`clickhouseReader *sqlx.DB`
`22`	`22`	`bigtable *db.Bigtable`
`23`	`23`	`persistentRedisDbClient *redis.Client`
	`24`	`+ localRedisDbClient *redis.Client`
`24`	`25`	`}`
`25`	`26`
`26`		`-func NewServices(readerDb, writerDb, clickhouseReader sqlx.DB, bigtable db.Bigtable, persistentRedisDbClient redis.Client) Services {`
	`27`	`+func NewServices(readerDb, writerDb, clickhouseReader sqlx.DB, bigtable db.Bigtable, persistentRedisDbClient, localRedisDbClient redis.Client) Services {`
`27`	`28`	`return &Services{`
`28`	`29`	`readerDb: readerDb,`
`29`	`30`	`writerDb: writerDb,`
`30`	`31`	`clickhouseReader: clickhouseReader,`
`31`	`32`	`bigtable: bigtable,`
`32`	`33`	`persistentRedisDbClient: persistentRedisDbClient,`
	`34`	`+ localRedisDbClient: localRedisDbClient,`
`33`	`35`	`}`
`34`	`36`	`}`
`35`	`37`
Original file line number	Diff line number	Diff line change
`@@ -91,7 +91,7 @@ func (s *Services) updateValidatorMapping() error {`
`91`	`91`	`ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)`
`92`	`92`	`defer cancel()`
`93`	`93`	`key := fmt.Sprintf("%d:%s", utils.Config.Chain.ClConfig.DepositChainID, "vm")`
`94`		`- compressed, err := s.persistentRedisDbClient.Get(ctx, key).Bytes()`
	`94`	`+ compressed, err := s.localRedisDbClient.Get(ctx, key).Bytes()`
`95`	`95`	`if err != nil {`
`96`	`96`	`return errors.Wrap(err, "failed to get compressed validator mapping from db")`
`97`	`97`	`}`