Merge pull request #881 from gobitfly/staging

merge staging->main

Author: guybrush

backend/cmd/blobindexer/main.go

@@ -16,7 +16,7 @@ import (
 func Run() {
 	fs := flag.NewFlagSet("fs", flag.ExitOnError)
 
-	configFlag := fs.String("config", "config.yml", "path to config")
+	configFlag := fs.String("config", "", "path to config")
 	versionFlag := fs.Bool("version", false, "print version and exit")
 	_ = fs.Parse(os.Args[2:])
 	if *versionFlag {

backend/cmd/misc/main.go

@@ -18,6 +18,7 @@ import (
 	"time"
 
 	"github.com/coocood/freecache"
+	"github.com/davecgh/go-spew/spew"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/go-redis/redis/v8"
 	"github.com/gobitfly/beaconchain/cmd/misc/commands"
@@ -32,6 +33,7 @@ import (
 	edb "github.com/gobitfly/beaconchain/pkg/exporter/db"
 	"github.com/gobitfly/beaconchain/pkg/exporter/modules"
 	"github.com/gobitfly/beaconchain/pkg/exporter/services"
+	"github.com/gobitfly/beaconchain/pkg/notification"
 	_ "github.com/jackc/pgx/v5/stdlib"
 	"github.com/pkg/errors"
 	utilMath "github.com/protolambda/zrnt/eth2/util/math"
@@ -75,7 +77,7 @@ func Run() {
 	}
 
 	configPath := fs.String("config", "config/default.config.yml", "Path to the config file")
-	fs.StringVar(&opts.Command, "command", "", "command to run, available: updateAPIKey, applyDbSchema, initBigtableSchema, epoch-export, debug-rewards, debug-blocks, clear-bigtable, index-old-eth1-blocks, update-aggregation-bits, historic-prices-export, index-missing-blocks, export-epoch-missed-slots, migrate-last-attestation-slot-bigtable, export-genesis-validators, update-block-finalization-sequentially, nameValidatorsByRanges, export-stats-totals, export-sync-committee-periods, export-sync-committee-validator-stats, partition-validator-stats, migrate-app-purchases")
+	fs.StringVar(&opts.Command, "command", "", "command to run, available: updateAPIKey, applyDbSchema, initBigtableSchema, epoch-export, debug-rewards, debug-blocks, clear-bigtable, index-old-eth1-blocks, update-aggregation-bits, historic-prices-export, index-missing-blocks, export-epoch-missed-slots, migrate-last-attestation-slot-bigtable, export-genesis-validators, update-block-finalization-sequentially, nameValidatorsByRanges, export-stats-totals, export-sync-committee-periods, export-sync-committee-validator-stats, partition-validator-stats, migrate-app-purchases, collect-notifications, collect-user-db-notifications")
 	fs.Uint64Var(&opts.StartEpoch, "start-epoch", 0, "start epoch")
 	fs.Uint64Var(&opts.EndEpoch, "end-epoch", 0, "end epoch")
 	fs.Uint64Var(&opts.User, "user", 0, "user id")
@@ -181,27 +183,27 @@ func Run() {
 	defer db.FrontendWriterDB.Close()
 
 	// clickhouse
-	db.ClickHouseWriter, db.ClickHouseReader = db.MustInitDB(&types.DatabaseConfig{
-		Username:     cfg.ClickHouse.WriterDatabase.Username,
-		Password:     cfg.ClickHouse.WriterDatabase.Password,
-		Name:         cfg.ClickHouse.WriterDatabase.Name,
-		Host:         cfg.ClickHouse.WriterDatabase.Host,
-		Port:         cfg.ClickHouse.WriterDatabase.Port,
-		MaxOpenConns: cfg.ClickHouse.WriterDatabase.MaxOpenConns,
-		SSL:          true,
-		MaxIdleConns: cfg.ClickHouse.WriterDatabase.MaxIdleConns,
-	}, &types.DatabaseConfig{
-		Username:     cfg.ClickHouse.ReaderDatabase.Username,
-		Password:     cfg.ClickHouse.ReaderDatabase.Password,
-		Name:         cfg.ClickHouse.ReaderDatabase.Name,
-		Host:         cfg.ClickHouse.ReaderDatabase.Host,
-		Port:         cfg.ClickHouse.ReaderDatabase.Port,
-		MaxOpenConns: cfg.ClickHouse.ReaderDatabase.MaxOpenConns,
-		SSL:          true,
-		MaxIdleConns: cfg.ClickHouse.ReaderDatabase.MaxIdleConns,
-	}, "clickhouse", "clickhouse")
-	defer db.ClickHouseReader.Close()
-	defer db.ClickHouseWriter.Close()
+	// db.ClickHouseWriter, db.ClickHouseReader = db.MustInitDB(&types.DatabaseConfig{
+	// 	Username:     cfg.ClickHouse.WriterDatabase.Username,
+	// 	Password:     cfg.ClickHouse.WriterDatabase.Password,
+	// 	Name:         cfg.ClickHouse.WriterDatabase.Name,
+	// 	Host:         cfg.ClickHouse.WriterDatabase.Host,
+	// 	Port:         cfg.ClickHouse.WriterDatabase.Port,
+	// 	MaxOpenConns: cfg.ClickHouse.WriterDatabase.MaxOpenConns,
+	// 	SSL:          true,
+	// 	MaxIdleConns: cfg.ClickHouse.WriterDatabase.MaxIdleConns,
+	// }, &types.DatabaseConfig{
+	// 	Username:     cfg.ClickHouse.ReaderDatabase.Username,
+	// 	Password:     cfg.ClickHouse.ReaderDatabase.Password,
+	// 	Name:         cfg.ClickHouse.ReaderDatabase.Name,
+	// 	Host:         cfg.ClickHouse.ReaderDatabase.Host,
+	// 	Port:         cfg.ClickHouse.ReaderDatabase.Port,
+	// 	MaxOpenConns: cfg.ClickHouse.ReaderDatabase.MaxOpenConns,
+	// 	SSL:          true,
+	// 	MaxIdleConns: cfg.ClickHouse.ReaderDatabase.MaxIdleConns,
+	// }, "clickhouse", "clickhouse")
+	// defer db.ClickHouseReader.Close()
+	// defer db.ClickHouseWriter.Close()
 
 	// Initialize the persistent redis client
 	rdc := redis.NewClient(&redis.Options{
@@ -216,6 +218,14 @@ func Run() {
 	db.PersistentRedisDbClient = rdc
 	defer db.PersistentRedisDbClient.Close()
 
+	if utils.Config.TieredCacheProvider != "redis" {
+		log.Fatal(nil, "no cache provider set, please set TierdCacheProvider (redis)", 0)
+	}
+	if utils.Config.TieredCacheProvider == "redis" || len(utils.Config.RedisCacheEndpoint) != 0 {
+		cache.MustInitTieredCache(utils.Config.RedisCacheEndpoint)
+		log.Infof("tiered Cache initialized, latest finalized epoch: %v", cache.LatestFinalizedEpoch.Get())
+	}
+
 	switch opts.Command {
 	case "nameValidatorsByRanges":
 		err := nameValidatorsByRanges(opts.ValidatorNameRanges)
@@ -456,6 +466,10 @@ func Run() {
 		err = fixEns(erigonClient)
 	case "fix-ens-addresses":
 		err = fixEnsAddresses(erigonClient)
+	case "collect-notifications":
+		err = collectNotifications(opts.StartEpoch)
+	case "collect-user-db-notifications":
+		err = collectUserDbNotifications(opts.StartEpoch)
 	default:
 		log.Fatal(nil, fmt.Sprintf("unknown command %s", opts.Command), 0)
 	}
@@ -467,6 +481,35 @@ func Run() {
 	}
 }
 
+func collectNotifications(startEpoch uint64) error {
+	epoch := startEpoch
+
+	log.Infof("collecting notifications for epoch %v", epoch)
+	notifications, err := notification.GetNotificationsForEpoch(utils.Config.Notifications.PubkeyCachePath, epoch)
+	if err != nil {
+		return err
+	}
+
+	log.Infof("found %v notifications for epoch %v with %v notifications for user 0", len(notifications), epoch, len(notifications[0]))
+	if len(notifications[0]) > 0 {
+		spew.Dump(notifications[0])
+	}
+	return nil
+}
+
+func collectUserDbNotifications(startEpoch uint64) error {
+	epoch := startEpoch
+
+	log.Infof("collecting notifications for epoch %v", epoch)
+	notifications, err := notification.GetUserNotificationsForEpoch(utils.Config.Notifications.PubkeyCachePath, epoch)
+	if err != nil {
+		return err
+	}
+
+	log.Infof("found %v notifications for epoch %v", len(notifications), epoch)
+	return nil
+}
+
 func fixEns(erigonClient *rpc.ErigonClient) error {
 	log.Infof("command: fix-ens")
 	addrs := []struct {

backend/cmd/monitoring/main.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/gobitfly/beaconchain/pkg/commons/db"
 	"github.com/gobitfly/beaconchain/pkg/commons/log"
+	"github.com/gobitfly/beaconchain/pkg/commons/metrics"
 	"github.com/gobitfly/beaconchain/pkg/commons/types"
 	"github.com/gobitfly/beaconchain/pkg/commons/utils"
 	"github.com/gobitfly/beaconchain/pkg/commons/version"
@@ -31,6 +32,15 @@ func Run() {
 	}
 	utils.Config = cfg
 
+	if utils.Config.Metrics.Enabled {
+		go func() {
+			log.Infof("serving metrics on %v", utils.Config.Metrics.Address)
+			if err := metrics.Serve(utils.Config.Metrics.Address, utils.Config.Metrics.Pprof, utils.Config.Metrics.PprofExtra); err != nil {
+				log.Fatal(err, "error serving metrics", 0)
+			}
+		}()
+	}
+
 	db.ClickHouseWriter, db.ClickHouseReader = db.MustInitDB(&types.DatabaseConfig{
 		Username:     cfg.ClickHouse.WriterDatabase.Username,
 		Password:     cfg.ClickHouse.WriterDatabase.Password,

backend/cmd/typescript_converter/main.go

@@ -3,6 +3,7 @@ package typescript_converter
 import (
 	"flag"
 	"go/ast"
+	"iter"
 	"os"
 	"path/filepath"
 	"slices"
@@ -18,6 +19,8 @@ const (
 	fallbackType   = "any"
 	commonFileName = "common"
 	lintDisable    = "/* eslint-disable */\n"
+	goFileSuffix   = ".go"
+	tsFileSuffix   = ".ts"
 )
 
 // Files that should not be converted to TypeScript
@@ -65,24 +68,23 @@ func Run() {
 		log.Fatal(nil, "Failed to load package", 0)
 	}
 
-	// Find all common types
+	// Find all common types, i.e. types that are used in multiple files and must be imported in ts
 	commonTypes := getCommonTypes(pkgs)
-	// Find all usages of common types
-	usage := getCommonUsages(pkgs, commonTypes)
-
-	// Generate Tygo for common.go
-	tygos := []*tygo.Tygo{tygo.New(getTygoConfig(out, commonFileName, ""))}
-	// Generate Tygo for each file
-	for file, typesUsed := range usage {
-		importStr := ""
+	// Find imports (usages of common types) for each file
+	imports := getImports(pkgs, commonTypes)
+
+	var configs []*tygo.Tygo
+	// Generate Tygo config for each file
+	for fileName, typesUsed := range imports {
+		var importStr string
 		if len(typesUsed) > 0 {
 			importStr = "import type { " + strings.Join(typesUsed, ", ") + " } from './" + commonFileName + "'\n"
 		}
-		tygos = append(tygos, tygo.New(getTygoConfig(out, file, importStr)))
+		configs = append(configs, tygo.New(getTygoConfig(out, fileName, importStr)))
 	}
 
 	// Generate TypeScript
-	for _, tygo := range tygos {
+	for _, tygo := range configs {
 		err := tygo.Generate()
 		if err != nil {
 			log.Fatal(err, "Failed to generate TypeScript", 0)
@@ -93,7 +95,7 @@ func Run() {
 }
 
 func deleteFiles(out string) error {
-	files, err := filepath.Glob(out + "*.ts")
+	files, err := filepath.Glob(out + "*" + tsFileSuffix)
 	if err != nil {
 		return err
 	}
@@ -106,68 +108,82 @@ func deleteFiles(out string) error {
 	return nil
 }
 
-func getTygoConfig(out, file, frontmatter string) *tygo.Config {
+func getTygoConfig(outDir, fileName, frontmatter string) *tygo.Config {
 	return &tygo.Config{
 		Packages: []*tygo.PackageConfig{
 			{
 				Path:         packagePath,
 				TypeMappings: typeMappings,
 				FallbackType: fallbackType,
-				IncludeFiles: []string{file + ".go"},
-				OutputPath:   out + file + ".ts",
+				IncludeFiles: []string{fileName + goFileSuffix},
+				OutputPath:   outDir + fileName + tsFileSuffix,
 				Frontmatter:  lintDisable + frontmatter,
 			},
 		},
 	}
 }
 
-// Parse common.go to find all common types
-func getCommonTypes(pkgs []*packages.Package) map[string]bool {
-	commonTypes := make(map[string]bool)
-	for _, pkg := range pkgs {
-		for _, file := range pkg.Syntax {
-			filename := strings.TrimSuffix(filepath.Base(pkg.Fset.File(file.Pos()).Name()), ".go")
-			if filepath.Base(filename) != commonFileName {
-				continue
-			}
-			ast.Inspect(file, func(n ast.Node) bool {
-				if typeSpec, ok := n.(*ast.TypeSpec); ok {
-					commonTypes[typeSpec.Name.Name] = true
+// Iterate over all file names and files in the packages
+func allFiles(pkgs []*packages.Package) iter.Seq2[string, *ast.File] {
+	return func(yield func(string, *ast.File) bool) {
+		for _, pkg := range pkgs {
+			for _, file := range pkg.Syntax {
+				fileName := filepath.Base(pkg.Fset.File(file.Pos()).Name())
+				if !yield(fileName, file) {
+					return
 				}
-				return true
-			})
-			return commonTypes
+			}
 		}
 	}
-	return nil
+}
+
+// Parse common.go to find all common types
+func getCommonTypes(pkgs []*packages.Package) map[string]struct{} {
+	var commonFile *ast.File
+	// find common file
+	for fileName, file := range allFiles(pkgs) {
+		fileName = strings.TrimSuffix(fileName, goFileSuffix)
+		if filepath.Base(fileName) == commonFileName {
+			commonFile = file
+			break
+		}
+	}
+	if commonFile == nil {
+		log.Fatal(nil, "common.go not found", 0)
+	}
+	commonTypes := make(map[string]struct{})
+	// iterate over all types in common file and add them to the map
+	for node := range ast.Preorder(commonFile) {
+		if typeSpec, ok := node.(*ast.TypeSpec); ok {
+			commonTypes[typeSpec.Name.Name] = struct{}{}
+		}
+	}
+	return commonTypes
 }
 
 // Parse all files to find used common types for each file
-func getCommonUsages(pkgs []*packages.Package, commonTypes map[string]bool) map[string][]string {
-	usage := make(map[string][]string) // Map from file to list of commonTypes used
-	for _, pkg := range pkgs {
-		for _, file := range pkg.Syntax {
-			filename := strings.TrimSuffix(filepath.Base(pkg.Fset.File(file.Pos()).Name()), ".go")
-			if filepath.Base(filename) == commonFileName || slices.Contains(ignoredFiles, filename) {
+// Returns a map with file name as key and a set of common types used in the file as value
+func getImports(pkgs []*packages.Package, commonTypes map[string]struct{}) map[string][]string {
+	imports := make(map[string][]string) // Map from file to set of commonTypes used
+	imports[commonFileName] = []string{} // Add common file to map with empty set
+	for fileName, file := range allFiles(pkgs) {
+		fileName = strings.TrimSuffix(fileName, goFileSuffix)
+		if filepath.Base(fileName) == commonFileName || slices.Contains(ignoredFiles, fileName) {
+			continue
+		}
+		var currentFileImports []string
+		// iterate over all struct fields in the file
+		for node := range ast.Preorder(file) {
+			ident, ok := node.(*ast.Ident)
+			if !ok {
 				continue
 			}
-			if _, exists := usage[filename]; !exists {
-				usage[filename] = make([]string, 0)
+			_, isCommonType := commonTypes[ident.Name]
+			if isCommonType && !slices.Contains(currentFileImports, ident.Name) {
+				currentFileImports = append(currentFileImports, ident.Name)
 			}
-			ast.Inspect(file, func(n ast.Node) bool {
-				ident, ok := n.(*ast.Ident)
-				if !ok {
-					return true
-				}
-				if !commonTypes[ident.Name] {
-					return true
-				}
-				if !slices.Contains(usage[filename], ident.Name) {
-					usage[filename] = append(usage[filename], ident.Name)
-				}
-				return true
-			})
 		}
+		imports[fileName] = currentFileImports
 	}
-	return usage
+	return imports
 }