Skip to content

Commit 267a0f9

Browse files
peterbitflypeterbitfly
authored
Merge pull request #566 from gobitfly/BIDS-3208/dashboard_notifications_backend
Bids 3208/dashboard notifications backend
2 parents 4e76b22 + 1ef7ac8 commit 267a0f9

File tree

11 files changed

+1020
-1197
lines changed

11 files changed

+1020
-1197
lines changed

‎backend/cmd/misc/main.go‎

Lines changed: 65 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
"time"
1919

2020
"github.com/coocood/freecache"
21+
"github.com/davecgh/go-spew/spew"
2122
"github.com/ethereum/go-ethereum/common"
2223
"github.com/go-redis/redis/v8"
2324
"github.com/gobitfly/beaconchain/cmd/misc/commands"
@@ -32,6 +33,7 @@ import (
3233
edb "github.com/gobitfly/beaconchain/pkg/exporter/db"
3334
"github.com/gobitfly/beaconchain/pkg/exporter/modules"
3435
"github.com/gobitfly/beaconchain/pkg/exporter/services"
36+
"github.com/gobitfly/beaconchain/pkg/notification"
3537
_ "github.com/jackc/pgx/v5/stdlib"
3638
"github.com/pkg/errors"
3739
utilMath "github.com/protolambda/zrnt/eth2/util/math"
@@ -75,7 +77,7 @@ func Run() {
7577
}
7678

7779
configPath := fs.String("config", "config/default.config.yml", "Path to the config file")
78-
fs.StringVar(&opts.Command, "command", "", "command to run, available: updateAPIKey, applyDbSchema, initBigtableSchema, epoch-export, debug-rewards, debug-blocks, clear-bigtable, index-old-eth1-blocks, update-aggregation-bits, historic-prices-export, index-missing-blocks, export-epoch-missed-slots, migrate-last-attestation-slot-bigtable, export-genesis-validators, update-block-finalization-sequentially, nameValidatorsByRanges, export-stats-totals, export-sync-committee-periods, export-sync-committee-validator-stats, partition-validator-stats, migrate-app-purchases")
80+
fs.StringVar(&opts.Command, "command", "", "command to run, available: updateAPIKey, applyDbSchema, initBigtableSchema, epoch-export, debug-rewards, debug-blocks, clear-bigtable, index-old-eth1-blocks, update-aggregation-bits, historic-prices-export, index-missing-blocks, export-epoch-missed-slots, migrate-last-attestation-slot-bigtable, export-genesis-validators, update-block-finalization-sequentially, nameValidatorsByRanges, export-stats-totals, export-sync-committee-periods, export-sync-committee-validator-stats, partition-validator-stats, migrate-app-purchases, collect-notifications, collect-user-db-notifications")
7981
fs.Uint64Var(&opts.StartEpoch, "start-epoch", 0, "start epoch")
8082
fs.Uint64Var(&opts.EndEpoch, "end-epoch", 0, "end epoch")
8183
fs.Uint64Var(&opts.User, "user", 0, "user id")
@@ -181,27 +183,27 @@ func Run() {
181183
defer db.FrontendWriterDB.Close()
182184

183185
// clickhouse
184-
db.ClickHouseWriter, db.ClickHouseReader = db.MustInitDB(&types.DatabaseConfig{
185-
Username: cfg.ClickHouse.WriterDatabase.Username,
186-
Password: cfg.ClickHouse.WriterDatabase.Password,
187-
Name: cfg.ClickHouse.WriterDatabase.Name,
188-
Host: cfg.ClickHouse.WriterDatabase.Host,
189-
Port: cfg.ClickHouse.WriterDatabase.Port,
190-
MaxOpenConns: cfg.ClickHouse.WriterDatabase.MaxOpenConns,
191-
SSL: true,
192-
MaxIdleConns: cfg.ClickHouse.WriterDatabase.MaxIdleConns,
193-
}, &types.DatabaseConfig{
194-
Username: cfg.ClickHouse.ReaderDatabase.Username,
195-
Password: cfg.ClickHouse.ReaderDatabase.Password,
196-
Name: cfg.ClickHouse.ReaderDatabase.Name,
197-
Host: cfg.ClickHouse.ReaderDatabase.Host,
198-
Port: cfg.ClickHouse.ReaderDatabase.Port,
199-
MaxOpenConns: cfg.ClickHouse.ReaderDatabase.MaxOpenConns,
200-
SSL: true,
201-
MaxIdleConns: cfg.ClickHouse.ReaderDatabase.MaxIdleConns,
202-
}, "clickhouse", "clickhouse")
203-
defer db.ClickHouseReader.Close()
204-
defer db.ClickHouseWriter.Close()
186+
// db.ClickHouseWriter, db.ClickHouseReader = db.MustInitDB(&types.DatabaseConfig{
187+
// Username: cfg.ClickHouse.WriterDatabase.Username,
188+
// Password: cfg.ClickHouse.WriterDatabase.Password,
189+
// Name: cfg.ClickHouse.WriterDatabase.Name,
190+
// Host: cfg.ClickHouse.WriterDatabase.Host,
191+
// Port: cfg.ClickHouse.WriterDatabase.Port,
192+
// MaxOpenConns: cfg.ClickHouse.WriterDatabase.MaxOpenConns,
193+
// SSL: true,
194+
// MaxIdleConns: cfg.ClickHouse.WriterDatabase.MaxIdleConns,
195+
// }, &types.DatabaseConfig{
196+
// Username: cfg.ClickHouse.ReaderDatabase.Username,
197+
// Password: cfg.ClickHouse.ReaderDatabase.Password,
198+
// Name: cfg.ClickHouse.ReaderDatabase.Name,
199+
// Host: cfg.ClickHouse.ReaderDatabase.Host,
200+
// Port: cfg.ClickHouse.ReaderDatabase.Port,
201+
// MaxOpenConns: cfg.ClickHouse.ReaderDatabase.MaxOpenConns,
202+
// SSL: true,
203+
// MaxIdleConns: cfg.ClickHouse.ReaderDatabase.MaxIdleConns,
204+
// }, "clickhouse", "clickhouse")
205+
// defer db.ClickHouseReader.Close()
206+
// defer db.ClickHouseWriter.Close()
205207

206208
// Initialize the persistent redis client
207209
rdc := redis.NewClient(&redis.Options{
@@ -216,6 +218,14 @@ func Run() {
216218
db.PersistentRedisDbClient = rdc
217219
defer db.PersistentRedisDbClient.Close()
218220

221+
if utils.Config.TieredCacheProvider != "redis" {
222+
log.Fatal(nil, "no cache provider set, please set TierdCacheProvider (redis)", 0)
223+
}
224+
if utils.Config.TieredCacheProvider == "redis" || len(utils.Config.RedisCacheEndpoint) != 0 {
225+
cache.MustInitTieredCache(utils.Config.RedisCacheEndpoint)
226+
log.Infof("tiered Cache initialized, latest finalized epoch: %v", cache.LatestFinalizedEpoch.Get())
227+
}
228+
219229
switch opts.Command {
220230
case "nameValidatorsByRanges":
221231
err := nameValidatorsByRanges(opts.ValidatorNameRanges)
@@ -456,6 +466,10 @@ func Run() {
456466
err = fixEns(erigonClient)
457467
case "fix-ens-addresses":
458468
err = fixEnsAddresses(erigonClient)
469+
case "collect-notifications":
470+
err = collectNotifications(opts.StartEpoch)
471+
case "collect-user-db-notifications":
472+
err = collectUserDbNotifications(opts.StartEpoch)
459473
default:
460474
log.Fatal(nil, fmt.Sprintf("unknown command %s", opts.Command), 0)
461475
}
@@ -467,6 +481,35 @@ func Run() {
467481
}
468482
}
469483

484+
func collectNotifications(startEpoch uint64) error {
485+
epoch := startEpoch
486+
487+
log.Infof("collecting notifications for epoch %v", epoch)
488+
notifications, err := notification.GetNotificationsForEpoch(utils.Config.Notifications.PubkeyCachePath, epoch)
489+
if err != nil {
490+
return err
491+
}
492+
493+
log.Infof("found %v notifications for epoch %v with %v notifications for user 0", len(notifications), epoch, len(notifications[0]))
494+
if len(notifications[0]) > 0 {
495+
spew.Dump(notifications[0])
496+
}
497+
return nil
498+
}
499+
500+
func collectUserDbNotifications(startEpoch uint64) error {
501+
epoch := startEpoch
502+
503+
log.Infof("collecting notifications for epoch %v", epoch)
504+
notifications, err := notification.GetUserNotificationsForEpoch(utils.Config.Notifications.PubkeyCachePath, epoch)
505+
if err != nil {
506+
return err
507+
}
508+
509+
log.Infof("found %v notifications for epoch %v", len(notifications), epoch)
510+
return nil
511+
}
512+
470513
func fixEns(erigonClient *rpc.ErigonClient) error {
471514
log.Infof("command: fix-ens")
472515
addrs := []struct {

‎backend/go.mod‎

Lines changed: 26 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ require (
66
cloud.google.com/go/bigtable v1.21.0
77
cloud.google.com/go/secretmanager v1.11.5
88
firebase.google.com/go v3.13.0+incompatible
9+
firebase.google.com/go/v4 v4.14.1
910
github.com/ClickHouse/clickhouse-go/v2 v2.17.1
1011
github.com/Gurpartap/storekit-go v0.0.0-20201205024111-36b6cd5c6a21
1112
github.com/alexedwards/scs/redisstore v0.0.0-20240316134038-7e11d57e8885
@@ -30,7 +31,7 @@ require (
3031
github.com/gobitfly/eth.store v0.0.0-20240312111708-b43f13990280
3132
github.com/golang-jwt/jwt v3.2.2+incompatible
3233
github.com/golang-jwt/jwt/v4 v4.5.0
33-
github.com/golang/protobuf v1.5.3
34+
github.com/golang/protobuf v1.5.4
3435
github.com/gomodule/redigo v1.9.2
3536
github.com/google/uuid v1.6.0
3637
github.com/gorilla/csrf v1.7.2
@@ -70,26 +71,27 @@ require (
7071
github.com/wealdtech/go-eth2-types/v2 v2.8.2
7172
github.com/wealdtech/go-eth2-util v1.8.0
7273
github.com/xeipuuv/gojsonschema v1.2.0
73-
golang.org/x/crypto v0.19.0
74+
golang.org/x/crypto v0.21.0
7475
golang.org/x/exp v0.0.0-20240213143201-ec583247a57a
7576
golang.org/x/sync v0.6.0
7677
golang.org/x/text v0.14.0
7778
golang.org/x/time v0.5.0
7879
golang.org/x/tools v0.18.0
79-
google.golang.org/api v0.164.0
80-
google.golang.org/protobuf v1.32.0
80+
google.golang.org/api v0.170.0
81+
google.golang.org/protobuf v1.33.0
8182
gopkg.in/yaml.v2 v2.4.0
8283
)
8384

8485
require (
85-
cloud.google.com/go v0.112.0 // indirect
86-
cloud.google.com/go/compute v1.23.3 // indirect
86+
cloud.google.com/go v0.112.1 // indirect
87+
cloud.google.com/go/compute v1.24.0 // indirect
8788
cloud.google.com/go/compute/metadata v0.2.3 // indirect
88-
cloud.google.com/go/firestore v1.14.0 // indirect
89-
cloud.google.com/go/iam v1.1.5 // indirect
90-
cloud.google.com/go/longrunning v0.5.4 // indirect
91-
cloud.google.com/go/storage v1.36.0 // indirect
89+
cloud.google.com/go/firestore v1.15.0 // indirect
90+
cloud.google.com/go/iam v1.1.7 // indirect
91+
cloud.google.com/go/longrunning v0.5.5 // indirect
92+
cloud.google.com/go/storage v1.40.0 // indirect
9293
github.com/ClickHouse/ch-go v0.58.2 // indirect
94+
github.com/MicahParks/keyfunc v1.9.0 // indirect
9395
github.com/Microsoft/go-winio v0.6.1 // indirect
9496
github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2 // indirect
9597
github.com/ajg/form v1.5.1 // indirect
@@ -153,7 +155,7 @@ require (
153155
github.com/google/go-querystring v1.1.0 // indirect
154156
github.com/google/s2a-go v0.1.7 // indirect
155157
github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
156-
github.com/googleapis/gax-go/v2 v2.12.0 // indirect
158+
github.com/googleapis/gax-go/v2 v2.12.3 // indirect
157159
github.com/gorilla/securecookie v1.1.2 // indirect
158160
github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 // indirect
159161
github.com/herumi/bls-eth-go-binary v1.31.0 // indirect
@@ -246,24 +248,25 @@ require (
246248
github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // indirect
247249
github.com/yusufpapurcu/wmi v1.2.3 // indirect
248250
go.opencensus.io v0.24.0 // indirect
249-
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 // indirect
250-
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 // indirect
251-
go.opentelemetry.io/otel v1.23.0 // indirect
252-
go.opentelemetry.io/otel/metric v1.23.0 // indirect
253-
go.opentelemetry.io/otel/trace v1.23.0 // indirect
251+
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
252+
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
253+
go.opentelemetry.io/otel v1.24.0 // indirect
254+
go.opentelemetry.io/otel/metric v1.24.0 // indirect
255+
go.opentelemetry.io/otel/trace v1.24.0 // indirect
254256
go.uber.org/atomic v1.11.0 // indirect
255257
go.uber.org/multierr v1.11.0 // indirect
256258
go.uber.org/zap v1.27.0 // indirect
257259
golang.org/x/mod v0.15.0 // indirect
258-
golang.org/x/net v0.21.0 // indirect
259-
golang.org/x/oauth2 v0.17.0 // indirect
260-
golang.org/x/sys v0.17.0 // indirect
260+
golang.org/x/net v0.23.0 // indirect
261+
golang.org/x/oauth2 v0.18.0 // indirect
262+
golang.org/x/sys v0.18.0 // indirect
261263
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
262264
google.golang.org/appengine v1.6.8 // indirect
263-
google.golang.org/genproto v0.0.0-20240125205218-1f4bbc51befe // indirect
264-
google.golang.org/genproto/googleapis/api v0.0.0-20240125205218-1f4bbc51befe // indirect
265-
google.golang.org/genproto/googleapis/rpc v0.0.0-20240205150955-31a09d347014 // indirect
266-
google.golang.org/grpc v1.62.0 // indirect
265+
google.golang.org/appengine/v2 v2.0.2 // indirect
266+
google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect
267+
google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c // indirect
268+
google.golang.org/genproto/googleapis/rpc v0.0.0-20240311132316-a219d84964c2 // indirect
269+
google.golang.org/grpc v1.62.1 // indirect
267270
gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect
268271
gopkg.in/yaml.v3 v3.0.1 // indirect
269272
lukechampine.com/blake3 v1.2.1 // indirect

0 commit comments

Comments
 (0)