K8SPSMDB-1325: Add directShardOperation role to clusterAdmin and clusterMonitor (#1887)

egegunes · hors · web-flow · commit ce163e5ff2e2 · 2025-04-17T16:35:42.000+03:00
* K8SPSMDB-1325: Add directShardOperation role to clusterAdmin and clusterMonitor

* K8SPSMDB-1325: Improve removeShard log message

* fix compare

* update compare files

* fix custom-replset-name

* add unit test

* add more tests

* fix version-service

---------

Co-authored-by: Viacheslav Sarzhan &lt;slava.sarzhan@percona.com&gt;
diff --git a/e2e-tests/custom-replset-name/conf/some-name.yml b/e2e-tests/custom-replset-name/conf/some-name.yml
@@ -5,7 +5,7 @@ metadata:
 spec:
   backup:
     enabled: true
-    image: percona/percona-backup-mongodb:2.0.4
+    image:
     pitr:
       enabled: false
     serviceAccountName: percona-server-mongodb-operator
@@ -33,7 +33,7 @@ spec:
           bucket: operator-testing
           prefix: psmdb
           endpointUrl: https://storage.googleapis.com
-  image: percona/percona-server-mongodb:4.4.10-11
+  image:
   imagePullPolicy: Always
   pmm:
     enabled: false
diff --git a/e2e-tests/functions b/e2e-tests/functions
@@ -881,7 +881,11 @@ compare_mongo_cmd() {
 		| $sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' \
 			>$tmp_dir/${command}${postfix}
 
-	diff -u ${test_dir}/compare/${command}${postfix}.json $tmp_dir/${command}${postfix}
+	if [[ ${UPDATE_COMPARE_FILES} -eq 0 ]]; then
+		diff -u ${test_dir}/compare/${command}${postfix}.json $tmp_dir/${command}${postfix}
+	else
+		cp $tmp_dir/${command}${postfix} ${test_dir}/compare/${command}${postfix}.json
+	fi
 }
 
 compare_mongos_cmd() {
@@ -897,7 +901,12 @@ compare_mongos_cmd() {
 		| egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:' \
 		| $sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' \
 			>$tmp_dir/${command}${postfix}
-	diff ${test_dir}/compare/${command}${postfix}.json $tmp_dir/${command}${postfix}
+
+	if [[ ${UPDATE_COMPARE_FILES} -eq 0 ]]; then
+		diff ${test_dir}/compare/${command}${postfix}.json $tmp_dir/${command}${postfix}
+	else
+		cp $tmp_dir/${command}${postfix} ${test_dir}/compare/${command}${postfix}.json
+	fi
 }
 
 get_mongo_primary_endpoint() {
@@ -959,7 +968,12 @@ compare_mongo_user() {
 		| sed -e '$s/,$/}/' \
 		| jq '.authInfo.authenticatedUserPrivileges|=sort_by(.resource.anyResource, .resource.cluster, .resource.db, .resource.collection)|.authInfo.authenticatedUserRoles|=sort_by(.role)' \
 			>$tmp_dir/$user.json
-	diff -u $expected_result $tmp_dir/$user.json
+
+	if [[ ${UPDATE_COMPARE_FILES} -eq 0 ]]; then
+		diff -u $expected_result $tmp_dir/$user.json
+	else
+		cp $tmp_dir/$user.json $expected_result
+	fi
 }
 
 start_gke() {
diff --git a/e2e-tests/init-deploy/compare/clusterAdmin-60.json b/e2e-tests/init-deploy/compare/clusterAdmin-60.json
@@ -10,6 +10,10 @@
       {
         "role": "clusterAdmin",
         "db": "admin"
+      },
+      {
+        "role": "directShardOperations",
+        "db": "admin"
       }
     ],
     "authenticatedUserPrivileges": [
diff --git a/e2e-tests/init-deploy/compare/clusterAdmin-70.json b/e2e-tests/init-deploy/compare/clusterAdmin-70.json
@@ -10,6 +10,10 @@
       {
         "role": "clusterAdmin",
         "db": "admin"
+      },
+      {
+        "role": "directShardOperations",
+        "db": "admin"
       }
     ],
     "authenticatedUserPrivileges": [
@@ -263,6 +267,7 @@
           "hostInfo",
           "inprog",
           "invalidateUserCache",
+          "issueDirectShardOperations",
           "killAnyCursor",
           "killAnySession",
           "killop",
diff --git a/e2e-tests/init-deploy/compare/clusterAdmin-80.json b/e2e-tests/init-deploy/compare/clusterAdmin-80.json
@@ -10,6 +10,10 @@
       {
         "role": "clusterAdmin",
         "db": "admin"
+      },
+      {
+        "role": "directShardOperations",
+        "db": "admin"
       }
     ],
     "authenticatedUserPrivileges": [
@@ -277,6 +281,7 @@
           "hostInfo",
           "inprog",
           "invalidateUserCache",
+          "issueDirectShardOperations",
           "killAnyCursor",
           "killAnySession",
           "killop",
diff --git a/e2e-tests/init-deploy/compare/clusterMonitor-60.json b/e2e-tests/init-deploy/compare/clusterMonitor-60.json
@@ -11,6 +11,10 @@
         "role": "clusterMonitor",
         "db": "admin"
       },
+      {
+        "role": "directShardOperations",
+        "db": "admin"
+      },
       {
         "role": "explainRole",
         "db": "admin"
diff --git a/e2e-tests/init-deploy/compare/clusterMonitor-70.json b/e2e-tests/init-deploy/compare/clusterMonitor-70.json
@@ -11,6 +11,10 @@
         "role": "clusterMonitor",
         "db": "admin"
       },
+      {
+        "role": "directShardOperations",
+        "db": "admin"
+      },
       {
         "role": "explainRole",
         "db": "admin"
@@ -191,6 +195,7 @@
           "getShardMap",
           "hostInfo",
           "inprog",
+          "issueDirectShardOperations",
           "listDatabases",
           "listSampledQueries",
           "listSessions",
diff --git a/e2e-tests/init-deploy/compare/clusterMonitor-80.json b/e2e-tests/init-deploy/compare/clusterMonitor-80.json
@@ -11,6 +11,10 @@
         "role": "clusterMonitor",
         "db": "admin"
       },
+      {
+        "role": "directShardOperations",
+        "db": "admin"
+      },
       {
         "role": "explainRole",
         "db": "admin"
@@ -203,6 +207,7 @@
           "getShardMap",
           "hostInfo",
           "inprog",
+          "issueDirectShardOperations",
           "listDatabases",
           "listSampledQueries",
           "listSessions",
diff --git a/e2e-tests/version-service/conf/version-service-major-rs0.yml b/e2e-tests/version-service/conf/version-service-major-rs0.yml
@@ -13,7 +13,7 @@ spec:
   updateStrategy: SmartUpdate
   upgradeOptions:
     versionServiceEndpoint: http://version-service:11000
-    apply: 5.0-latest
+    apply: 6.0-latest
     schedule: "0 2 * * *"
   replsets:
   - name: rs0
diff --git a/e2e-tests/version-service/run b/e2e-tests/version-service/run
@@ -143,7 +143,7 @@ kubectl_bin delete pod -l run=version-service ${OPERATOR_NS:+-n $OPERATOR_NS}
 check_telemetry_transfer "http://version-service-cr:11000" "disabled" "disabled"
 
 cases=("version-service-exact" "version-service-recommended" "version-service-latest" "version-service-major" "version-service-unreachable")
-expected_images=("percona/percona-server-mongodb:6.0.3-2" "percona/percona-server-mongodb:8.0.4-1-multi" "percona/percona-server-mongodb:8.0.4-1-multi" "percona/percona-server-mongodb:5.0.14-12" "$IMAGE_MONGOD")
+expected_images=("percona/percona-server-mongodb:6.0.3-2" "percona/percona-server-mongodb:8.0.4-1-multi" "percona/percona-server-mongodb:8.0.4-1-multi" "percona/percona-server-mongodb:6.0.4-3" "$IMAGE_MONGOD")
 
 for i in "${!cases[@]}"; do
 	desc "test ${cases[$i]}"
diff --git a/pkg/controller/perconaservermongodb/connections_test.go b/pkg/controller/perconaservermongodb/connections_test.go
@@ -425,7 +425,7 @@ func (c *fakeMongoClient) GetRole(ctx context.Context, db, role string) (*mongo.
 
 func (c *fakeMongoClient) GetUserInfo(ctx context.Context, username, db string) (*mongo.User, error) {
 	return &mongo.User{
-		Roles: []map[string]interface{}{},
+		Roles: []mongo.Role{},
 	}, nil
 }
 
diff --git a/pkg/controller/perconaservermongodb/custom_users.go b/pkg/controller/perconaservermongodb/custom_users.go
@@ -345,23 +345,16 @@ func updatePass(
 	return nil
 }
 
-func updateRoles(
-	ctx context.Context,
-	mongoCli mongo.Client,
-	user *api.User,
-	userInfo *mongo.User) error {
+func updateRoles(ctx context.Context, mongoCli mongo.Client, user *api.User, userInfo *mongo.User) error {
 	log := logf.FromContext(ctx)
 
 	if userInfo == nil {
 		return nil
 	}
 
-	roles := make([]map[string]interface{}, 0)
+	roles := make([]mongo.Role, 0)
 	for _, role := range user.Roles {
-		roles = append(roles, map[string]interface{}{
-			"role": role.Name,
-			"db":   role.DB,
-		})
+		roles = append(roles, mongo.Role{DB: role.DB, Role: role.Name})
 	}
 
 	if reflect.DeepEqual(userInfo.Roles, roles) {
@@ -381,12 +374,9 @@ func updateRoles(
 func createExternalUser(ctx context.Context, mongoCli mongo.Client, user *api.User) error {
 	log := logf.FromContext(ctx)
 
-	roles := make([]map[string]interface{}, 0)
+	roles := make([]mongo.Role, 0)
 	for _, role := range user.Roles {
-		roles = append(roles, map[string]interface{}{
-			"role": role.Name,
-			"db":   role.DB,
-		})
+		roles = append(roles, mongo.Role{DB: role.DB, Role: role.Name})
 	}
 
 	log.Info("Creating user", "user", user.UserID())
@@ -408,12 +398,9 @@ func createUser(
 	annotationKey, passKey string) error {
 	log := logf.FromContext(ctx)
 
-	roles := make([]map[string]interface{}, 0)
+	roles := make([]mongo.Role, 0)
 	for _, role := range user.Roles {
-		roles = append(roles, map[string]interface{}{
-			"role": role.Name,
-			"db":   role.DB,
-		})
+		roles = append(roles, mongo.Role{DB: role.DB, Role: role.Name})
 	}
 
 	log.Info("Creating user", "user", user.UserID())
diff --git a/pkg/controller/perconaservermongodb/mgo.go b/pkg/controller/perconaservermongodb/mgo.go
@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"reflect"
+	"slices"
 	"sort"
 	"strings"
 	"time"
@@ -593,8 +594,13 @@ func (r *ReconcilePerconaServerMongoDB) removeRSFromShard(ctx context.Context, c
 			return nil
 		}
 
-		log.Info(resp.Msg, "shard", rsName,
-			"chunk remaining", resp.Remaining.Chunks, "jumbo chunks remaining", resp.Remaining.JumboChunks)
+		log.Info(resp.Msg,
+			"shard", rsName,
+			"note", resp.Note,
+			"dbsToMove", resp.DBsToMove,
+			"remaining dbs", resp.Remaining.DBs,
+			"remaining chunks", resp.Remaining.Chunks,
+			"remaining jumbo chunks", resp.Remaining.JumboChunks)
 
 		time.Sleep(10 * time.Second)
 	}
@@ -771,34 +777,41 @@ func (r *ReconcilePerconaServerMongoDB) handleReplicaSetNoPrimary(ctx context.Co
 	return errNoRunningMongodContainers
 }
 
-func getRoles(cr *api.PerconaServerMongoDB, role api.SystemUserRole) []map[string]interface{} {
-	roles := make([]map[string]interface{}, 0)
+func getRoles(cr *api.PerconaServerMongoDB, role api.SystemUserRole) []mongo.Role {
+	roles := make([]mongo.Role, 0)
 	switch role {
 	case api.RoleDatabaseAdmin:
-		return []map[string]interface{}{
-			{"role": "readWriteAnyDatabase", "db": "admin"},
-			{"role": "readAnyDatabase", "db": "admin"},
-			{"role": "restore", "db": "admin"},
-			{"role": "backup", "db": "admin"},
-			{"role": "dbAdminAnyDatabase", "db": "admin"},
-			{"role": string(api.RoleClusterMonitor), "db": "admin"},
+		return []mongo.Role{
+			{DB: "admin", Role: "readWriteAnyDatabase"},
+			{DB: "admin", Role: "readAnyDatabase"},
+			{DB: "admin", Role: "restore"},
+			{DB: "admin", Role: "backup"},
+			{DB: "admin", Role: "dbAdminAnyDatabase"},
+			{DB: "admin", Role: string(api.RoleClusterMonitor)},
 		}
 	case api.RoleClusterMonitor:
-		if cr.CompareVersion("1.12.0") >= 0 {
-			roles = []map[string]interface{}{
-				{"db": "admin", "role": "explainRole"},
-				{"db": "local", "role": "read"},
-			}
+		roles = []mongo.Role{
+			{DB: "admin", Role: "explainRole"},
+			{DB: "local", Role: "read"},
+		}
+		if cr.CompareVersion("1.20.0") >= 0 {
+			roles = append(roles, mongo.Role{DB: "admin", Role: "directShardOperations"})
 		}
 	case api.RoleBackup:
-		roles = []map[string]interface{}{
-			{"db": "admin", "role": "readWrite"},
-			{"db": "admin", "role": string(api.RoleClusterMonitor)},
-			{"db": "admin", "role": "restore"},
-			{"db": "admin", "role": "pbmAnyAction"},
+		roles = []mongo.Role{
+			{DB: "admin", Role: "readWrite"},
+			{DB: "admin", Role: string(api.RoleClusterMonitor)},
+			{DB: "admin", Role: "restore"},
+			{DB: "admin", Role: "pbmAnyAction"},
+		}
+	case api.RoleClusterAdmin:
+		if cr.CompareVersion("1.20.0") >= 0 {
+			roles = []mongo.Role{
+				{DB: "admin", Role: "directShardOperations"},
+			}
 		}
 	}
-	roles = append(roles, map[string]interface{}{"db": "admin", "role": string(role)})
+	roles = append(roles, mongo.Role{DB: "admin", Role: string(role)})
 	return roles
 }
 
@@ -882,10 +895,22 @@ func (r *ReconcilePerconaServerMongoDB) createOrUpdateSystemRoles(ctx context.Co
 }
 
 // compareRoles compares 2 role arrays and returns true if they are equal
-func compareRoles(x []map[string]interface{}, y []map[string]interface{}) bool {
+func compareRoles(x []mongo.Role, y []mongo.Role) bool {
 	if len(x) != len(y) {
 		return false
 	}
+
+	sortFunc := func(a mongo.Role, b mongo.Role) int {
+		if a.DB < b.DB || a.Role < b.Role {
+			return -1
+		}
+
+		return 0
+	}
+
+	slices.SortFunc(x, sortFunc)
+	slices.SortFunc(y, sortFunc)
+
 	for i := range x {
 		if !reflect.DeepEqual(x[i], y[i]) {
 			return false
@@ -998,13 +1023,15 @@ func (r *ReconcilePerconaServerMongoDB) createOrUpdateSystemUsers(ctx context.Co
 			return errors.Wrap(err, "get user info")
 		}
 		if user == nil {
+			log.Info("Creating user", "database", "admin", "user", creds.Username)
 			err = cli.CreateUser(ctx, "admin", creds.Username, creds.Password, getRoles(cr, role)...)
 			if err != nil {
 				return errors.Wrapf(err, "failed to create user %s", role)
 			}
 			continue
 		}
 		if !compareRoles(user.Roles, getRoles(cr, role)) {
+			log.Info("Updating user roles", "database", "admin", "user", creds.Username, "currentRoles", user.Roles, "newRoles", getRoles(cr, role))
 			err = cli.UpdateUserRoles(ctx, "admin", creds.Username, getRoles(cr, role))
 			if err != nil {
 				return errors.Wrapf(err, "failed to create user %s", role)
diff --git a/pkg/controller/perconaservermongodb/mgo_test.go b/pkg/controller/perconaservermongodb/mgo_test.go
diff --git a/pkg/psmdb/mongo/fake/client.go b/pkg/psmdb/mongo/fake/client.go
diff --git a/pkg/psmdb/mongo/models.go b/pkg/psmdb/mongo/models.go
diff --git a/pkg/psmdb/mongo/mongo.go b/pkg/psmdb/mongo/mongo.go

Original file line number	Diff line number	Diff line change
`@@ -10,6 +10,10 @@`
`10`	`10`	`{`
`11`	`11`	`"role": "clusterAdmin",`
`12`	`12`	`"db": "admin"`
	`13`	`+ },`
	`14`	`+ {`
	`15`	`+ "role": "directShardOperations",`
	`16`	`+ "db": "admin"`
`13`	`17`	`}`
`14`	`18`	`],`
`15`	`19`	`"authenticatedUserPrivileges": [`