(BEDS-500) don't throw error when adding more validators than limit allows (#877)

Author: LuccaBitfly

backend/pkg/api/data_access/dummy.go

@@ -253,10 +253,6 @@ func (d *DummyService) GetValidatorDashboardGroupExists(ctx context.Context, das
 	return true, nil
 }
 
-func (d *DummyService) GetValidatorDashboardExistingValidatorCount(ctx context.Context, dashboardId t.VDBIdPrimary, validators []t.VDBValidator) (uint64, error) {
-	return getDummyData[uint64]()
-}
-
 func (d *DummyService) AddValidatorDashboardValidators(ctx context.Context, dashboardId t.VDBIdPrimary, groupId uint64, validators []t.VDBValidator) ([]t.VDBPostValidatorsData, error) {
 	return getDummyData[[]t.VDBPostValidatorsData]()
 }

backend/pkg/api/data_access/vdb.go

@@ -28,7 +28,6 @@ type ValidatorDashboardRepository interface {
 	GetValidatorDashboardGroupCount(ctx context.Context, dashboardId t.VDBIdPrimary) (uint64, error)
 	GetValidatorDashboardGroupExists(ctx context.Context, dashboardId t.VDBIdPrimary, groupId uint64) (bool, error)
 
-	GetValidatorDashboardExistingValidatorCount(ctx context.Context, dashboardId t.VDBIdPrimary, validators []t.VDBValidator) (uint64, error)
 	AddValidatorDashboardValidators(ctx context.Context, dashboardId t.VDBIdPrimary, groupId uint64, validators []t.VDBValidator) ([]t.VDBPostValidatorsData, error)
 	AddValidatorDashboardValidatorsByDepositAddress(ctx context.Context, dashboardId t.VDBIdPrimary, groupId uint64, address string, limit uint64) ([]t.VDBPostValidatorsData, error)
 	AddValidatorDashboardValidatorsByWithdrawalAddress(ctx context.Context, dashboardId t.VDBIdPrimary, groupId uint64, address string, limit uint64) ([]t.VDBPostValidatorsData, error)

backend/pkg/api/data_access/vdb_management.go

@@ -6,6 +6,7 @@ import (
 	"encoding/hex"
 	"fmt"
 	"math/big"
+	"slices"
 	"sort"
 	"strconv"
 	"strings"
@@ -15,7 +16,6 @@ import (
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/gobitfly/beaconchain/pkg/api/enums"
 	t "github.com/gobitfly/beaconchain/pkg/api/types"
-	"github.com/gobitfly/beaconchain/pkg/commons/log"
 	"github.com/gobitfly/beaconchain/pkg/commons/utils"
 	constypes "github.com/gobitfly/beaconchain/pkg/consapi/types"
 	"github.com/lib/pq"
@@ -790,21 +790,6 @@ func (d *DataAccessService) GetValidatorDashboardGroupExists(ctx context.Context
 	return groupExists, err
 }
 
-// return how many of the passed validators are already in the dashboard
-func (d *DataAccessService) GetValidatorDashboardExistingValidatorCount(ctx context.Context, dashboardId t.VDBIdPrimary, validators []t.VDBValidator) (uint64, error) {
-	if len(validators) == 0 {
-		return 0, nil
-	}
-
-	var count uint64
-	err := d.alloyReader.GetContext(ctx, &count, `
-		SELECT COUNT(*)
-		FROM users_val_dashboards_validators
-		WHERE dashboard_id = $1 AND validator_index = ANY($2)
-	`, dashboardId, pq.Array(validators))
-	return count, err
-}
-
 func (d *DataAccessService) AddValidatorDashboardValidators(ctx context.Context, dashboardId t.VDBIdPrimary, groupId uint64, validators []t.VDBValidator) ([]t.VDBPostValidatorsData, error) {
 	if len(validators) == 0 {
 		// No validators to add
@@ -889,191 +874,145 @@ func (d *DataAccessService) AddValidatorDashboardValidators(ctx context.Context,
 	return result, nil
 }
 
+// Updates the group for validators already in the dashboard linked to the deposit address.
+// Adds up to limit new validators associated with the deposit address, if not already in the dashboard.
 func (d *DataAccessService) AddValidatorDashboardValidatorsByDepositAddress(ctx context.Context, dashboardId t.VDBIdPrimary, groupId uint64, address string, limit uint64) ([]t.VDBPostValidatorsData, error) {
-	// for all validators already in the dashboard that are associated with the deposit address, update the group
-	// then add no more than `limit` validators associated with the deposit address to the dashboard
 	addressParsed, err := hex.DecodeString(strings.TrimPrefix(address, "0x"))
 	if err != nil {
 		return nil, err
 	}
 
-	if len(addressParsed) != 20 {
-		return nil, fmt.Errorf("invalid deposit address: %s", address)
-	}
-	var validatorIndicesToAdd []uint64
-	err = d.readerDb.SelectContext(ctx, &validatorIndicesToAdd, "SELECT validatorindex FROM validators WHERE pubkey IN (SELECT publickey FROM eth1_deposits WHERE from_address = $1) ORDER BY validatorindex LIMIT $2;", addressParsed, limit)
-	if err != nil {
-		return nil, err
-	}
+	g, gCtx := errgroup.WithContext(ctx)
 
-	// retrieve the existing validators
-	var existingValidators []uint64
-	err = d.alloyWriter.SelectContext(ctx, &existingValidators, "SELECT validator_index FROM users_val_dashboards_validators WHERE dashboard_id = $1", dashboardId)
-	if err != nil {
-		return nil, err
-	}
-	existingValidatorsMap := make(map[uint64]bool, len(existingValidators))
-	for _, validatorIndex := range existingValidators {
-		existingValidatorsMap[validatorIndex] = true
-	}
-
-	// filter out the validators that are already in the dashboard
+	// fetch validators that are already in the dashboard and associated with the deposit address
 	var validatorIndicesToUpdate []uint64
+
+	g.Go(func() error {
+		return d.readerDb.SelectContext(gCtx, &validatorIndicesToUpdate, `
+		SELECT DISTINCT uvdv.validator_index
+		FROM validators v
+		JOIN eth1_deposits d ON v.pubkey = d.publickey
+		JOIN users_val_dashboards_validators uvdv ON v.validatorindex = uvdv.validator_index
+		WHERE uvdv.dashboard_id = $1 AND d.from_address = $2;
+		`, dashboardId, addressParsed)
+	})
+
+	// fetch validators that are not yet in the dashboard and associated with the deposit address, up to the limit
 	var validatorIndicesToInsert []uint64
-	for _, validatorIndex := range validatorIndicesToAdd {
-		if _, ok := existingValidatorsMap[validatorIndex]; ok {
-			validatorIndicesToUpdate = append(validatorIndicesToUpdate, validatorIndex)
-		} else {
-			validatorIndicesToInsert = append(validatorIndicesToInsert, validatorIndex)
-		}
-	}
+	g.Go(func() error {
+		return d.readerDb.SelectContext(gCtx, &validatorIndicesToInsert, `
+		SELECT DISTINCT v.validatorindex
+		FROM validators v
+		JOIN eth1_deposits d ON v.pubkey = d.publickey
+		LEFT JOIN users_val_dashboards_validators uvdv ON v.validatorindex = uvdv.validator_index AND uvdv.dashboard_id = $1
+		WHERE d.from_address = $2 AND uvdv.validator_index IS NULL
+		ORDER BY v.validatorindex
+		LIMIT $3;
+		`, dashboardId, addressParsed, limit)
+	})
 
-	// update the group for all existing validators
-	validatorIndices := make([]uint64, 0, int(limit))
-	validatorIndices = append(validatorIndices, validatorIndicesToUpdate...)
-
-	// insert the new validators up to the allowed user max limit taking into account how many validators are already in the dashboard
-	if len(validatorIndicesToInsert) > 0 {
-		freeSpace := int(limit) - len(existingValidators)
-		if freeSpace > 0 {
-			if len(validatorIndicesToInsert) > freeSpace { // cap inserts to the amount of free space available
-				log.Infof("limiting the number of validators to insert to %d", freeSpace)
-				validatorIndicesToInsert = validatorIndicesToInsert[:freeSpace]
-			}
-			validatorIndices = append(validatorIndices, validatorIndicesToInsert...)
-		}
+	err = g.Wait()
+	if err != nil {
+		return nil, err
 	}
 
-	if len(validatorIndices) == 0 {
-		// no validators to add
-		return []t.VDBPostValidatorsData{}, nil
-	}
-	log.Infof("inserting %d new validators and updating %d validators of dashboard %d, limit is %d", len(validatorIndicesToInsert), len(validatorIndicesToUpdate), dashboardId, limit)
+	validatorIndices := slices.Concat(validatorIndicesToUpdate, validatorIndicesToInsert)
+
 	return d.AddValidatorDashboardValidators(ctx, dashboardId, groupId, validatorIndices)
 }
 
+// Updates the group for validators already in the dashboard linked to the withdrawal address.
+// Adds up to limit new validators associated with the withdrawal address, if not already in the dashboard.
 func (d *DataAccessService) AddValidatorDashboardValidatorsByWithdrawalAddress(ctx context.Context, dashboardId t.VDBIdPrimary, groupId uint64, address string, limit uint64) ([]t.VDBPostValidatorsData, error) {
-	// for all validators already in the dashboard that are associated with the withdrawal address, update the group
-	// then add no more than `limit` validators associated with the deposit address to the dashboard
 	addressParsed, err := hex.DecodeString(strings.TrimPrefix(address, "0x"))
 	if err != nil {
 		return nil, err
 	}
-	var validatorIndicesToAdd []uint64
-	err = d.readerDb.SelectContext(ctx, &validatorIndicesToAdd, "SELECT validatorindex FROM validators WHERE withdrawalcredentials = $1 ORDER BY validatorindex LIMIT $2;", addressParsed, limit)
-	if err != nil {
-		return nil, err
-	}
 
-	// retrieve the existing validators
-	var existingValidators []uint64
-	err = d.alloyWriter.SelectContext(ctx, &existingValidators, "SELECT validator_index FROM users_val_dashboards_validators WHERE dashboard_id = $1", dashboardId)
-	if err != nil {
-		return nil, err
-	}
-	existingValidatorsMap := make(map[uint64]bool, len(existingValidators))
-	for _, validatorIndex := range existingValidators {
-		existingValidatorsMap[validatorIndex] = true
-	}
+	g, gCtx := errgroup.WithContext(ctx)
 
-	// filter out the validators that are already in the dashboard
+	// fetch validators that are already in the dashboard and associated with the withdrawal address
 	var validatorIndicesToUpdate []uint64
+	g.Go(func() error {
+		return d.readerDb.SelectContext(gCtx, &validatorIndicesToUpdate, `
+		SELECT DISTINCT uvdv.validator_index
+		FROM validators v
+		JOIN users_val_dashboards_validators uvdv ON v.validatorindex = uvdv.validator_index
+		WHERE uvdv.dashboard_id = $1 AND v.withdrawalcredentials = $2 AND uvdv.dashboard_id = $2;
+		`, dashboardId, addressParsed)
+	})
+
+	// fetch validators that are not yet in the dashboard and associated with the withdrawal address, up to the limit
 	var validatorIndicesToInsert []uint64
-	for _, validatorIndex := range validatorIndicesToAdd {
-		if _, ok := existingValidatorsMap[validatorIndex]; ok {
-			validatorIndicesToUpdate = append(validatorIndicesToUpdate, validatorIndex)
-		} else {
-			validatorIndicesToInsert = append(validatorIndicesToInsert, validatorIndex)
-		}
-	}
+	g.Go(func() error {
+		return d.readerDb.SelectContext(gCtx, &validatorIndicesToInsert, `
+		SELECT DISTINCT v.validatorindex
+		FROM validators v
+		LEFT JOIN users_val_dashboards_validators uvdv ON v.validatorindex = uvdv.validator_index AND uvdv.dashboard_id = $1
+		WHERE v.withdrawalcredentials = $2 AND uvdv.validator_index IS NULL
+		ORDER BY v.validatorindex
+		LIMIT $3;
+		`, dashboardId, addressParsed, limit)
+	})
 
-	// update the group for all existing validators
-	validatorIndices := make([]uint64, 0, int(limit))
-	validatorIndices = append(validatorIndices, validatorIndicesToUpdate...)
-
-	// insert the new validators up to the allowed user max limit taking into account how many validators are already in the dashboard
-	if len(validatorIndicesToInsert) > 0 {
-		freeSpace := int(limit) - len(existingValidators)
-		if freeSpace > 0 {
-			if len(validatorIndicesToInsert) > freeSpace { // cap inserts to the amount of free space available
-				log.Infof("limiting the number of validators to insert to %d", freeSpace)
-				validatorIndicesToInsert = validatorIndicesToInsert[:freeSpace]
-			}
-			validatorIndices = append(validatorIndices, validatorIndicesToInsert...)
-		}
+	err = g.Wait()
+	if err != nil {
+		return nil, err
 	}
 
-	if len(validatorIndices) == 0 {
-		// no validators to add
-		return []t.VDBPostValidatorsData{}, nil
-	}
-	log.Infof("inserting %d new validators and updating %d validators of dashboard %d, limit is %d", len(validatorIndicesToInsert), len(validatorIndicesToUpdate), dashboardId, limit)
+	validatorIndices := slices.Concat(validatorIndicesToUpdate, validatorIndicesToInsert)
+
 	return d.AddValidatorDashboardValidators(ctx, dashboardId, groupId, validatorIndices)
 }
 
+// Update the group for validators already in the dashboard linked to the graffiti (via produced block).
+// Add up to limit new validators associated with the graffiti, if not already in the dashboard.
 func (d *DataAccessService) AddValidatorDashboardValidatorsByGraffiti(ctx context.Context, dashboardId t.VDBIdPrimary, groupId uint64, graffiti string, limit uint64) ([]t.VDBPostValidatorsData, error) {
-	// for all validators already in the dashboard that are associated with the graffiti (by produced block), update the group
-	// then add no more than `limit` validators associated with the deposit address to the dashboard
-	var validatorIndicesToAdd []uint64
-	err := d.readerDb.SelectContext(ctx, &validatorIndicesToAdd, "SELECT DISTINCT proposer FROM blocks WHERE graffiti_text = $1 ORDER BY proposer LIMIT $2;", graffiti, limit)
-	if err != nil {
-		return nil, err
-	}
+	g, gCtx := errgroup.WithContext(ctx)
 
-	// retrieve the existing validators
-	var existingValidators []uint64
-	err = d.alloyWriter.SelectContext(ctx, &existingValidators, "SELECT validator_index FROM users_val_dashboards_validators WHERE dashboard_id = $1", dashboardId)
-	if err != nil {
-		return nil, err
-	}
-	existingValidatorsMap := make(map[uint64]bool, len(existingValidators))
-	for _, validatorIndex := range existingValidators {
-		existingValidatorsMap[validatorIndex] = true
-	}
-
-	// filter out the validators that are already in the dashboard
+	// fetch validators that are already in the dashboard and associated with the graffiti
 	var validatorIndicesToUpdate []uint64
+	g.Go(func() error {
+		return d.readerDb.SelectContext(gCtx, &validatorIndicesToUpdate, `
+		SELECT DISTINCT uvdv.validator_index
+		FROM blocks b
+		JOIN users_val_dashboards_validators uvdv ON b.proposer = uvdv.validator_index
+		WHERE uvdv.dashboard_id = $1 AND b.graffiti_text = $2;
+		`, dashboardId, graffiti)
+	})
+
+	// fetch validators that are not yet in the dashboard and associated with the graffiti, up to the limit
 	var validatorIndicesToInsert []uint64
-	for _, validatorIndex := range validatorIndicesToAdd {
-		if _, ok := existingValidatorsMap[validatorIndex]; ok {
-			validatorIndicesToUpdate = append(validatorIndicesToUpdate, validatorIndex)
-		} else {
-			validatorIndicesToInsert = append(validatorIndicesToInsert, validatorIndex)
-		}
-	}
+	g.Go(func() error {
+		return d.readerDb.SelectContext(gCtx, &validatorIndicesToInsert, `
+		SELECT DISTINCT b.proposer
+		FROM blocks b
+		LEFT JOIN users_val_dashboards_validators uvdv ON b.proposer = uvdv.validator_index AND uvdv.dashboard_id = $1
+		WHERE b.graffiti_text = $2 AND uvdv.validator_index IS NULL
+		ORDER BY b.proposer
+		LIMIT $3;
+		`, dashboardId, graffiti, limit)
+	})
 
-	// update the group for all existing validators
-	validatorIndices := make([]uint64, 0, int(limit))
-	validatorIndices = append(validatorIndices, validatorIndicesToUpdate...)
-
-	// insert the new validators up to the allowed user max limit taking into account how many validators are already in the dashboard
-	if len(validatorIndicesToInsert) > 0 {
-		freeSpace := int(limit) - len(existingValidators)
-		if freeSpace > 0 {
-			if len(validatorIndicesToInsert) > freeSpace { // cap inserts to the amount of free space available
-				log.Infof("limiting the number of validators to insert to %d", freeSpace)
-				validatorIndicesToInsert = validatorIndicesToInsert[:freeSpace]
-			}
-			validatorIndices = append(validatorIndices, validatorIndicesToInsert...)
-		}
+	err := g.Wait()
+	if err != nil {
+		return nil, err
 	}
 
-	if len(validatorIndices) == 0 {
-		// no validators to add
-		return []t.VDBPostValidatorsData{}, nil
-	}
-	log.Infof("inserting %d new validators and updating %d validators of dashboard %d, limit is %d", len(validatorIndicesToInsert), len(validatorIndicesToUpdate), dashboardId, limit)
+	validatorIndices := slices.Concat(validatorIndicesToUpdate, validatorIndicesToInsert)
+
 	return d.AddValidatorDashboardValidators(ctx, dashboardId, groupId, validatorIndices)
 }
 
 func (d *DataAccessService) RemoveValidatorDashboardValidators(ctx context.Context, dashboardId t.VDBIdPrimary, validators []t.VDBValidator) error {
 	if len(validators) == 0 {
-		// // Remove all validators for the dashboard
-		// _, err := d.alloyWriter.ExecContext(ctx, `
-		// 	DELETE FROM users_val_dashboards_validators
-		// 	WHERE dashboard_id = $1
-		// `, dashboardId)
-		return fmt.Errorf("calling RemoveValidatorDashboardValidators with empty validators list is not allowed")
+		// Remove all validators for the dashboard
+		// This is usually forbidden by API validation
+		_, err := d.alloyWriter.ExecContext(ctx, `
+			DELETE FROM users_val_dashboards_validators
+			WHERE dashboard_id = $1
+		`, dashboardId)
+		return err
 	}
 
 	//Create the query to delete validators