Use pagination when retrieving groups for an okta application (#162)

* When retrieving groups for an okta application, use pagination to retrieve all groups instead of stopping on the first page

* Update documentation around groupLimit now that the behavior is different

* Add missing groupLimit description

Author: dweebo

README.md

@@ -416,7 +416,8 @@ The following table describes the set of configuration options for the Okta prov
 | `appId` | Application ID of App Groups are assigned to | `''`  | Yes |
 | `extractLoginUsername` | Bool to determine if you should extract username from okta login | `false`  | No |
 | `profileKey` | Attribute field on Okta User Profile you would like to use as identity | `'login'` | No |
-| `groupLimit` | Integer to set the maximum number of groups to sync | `1000` | No |
+| `groupLimit` | Integer to set the maximum number of groups to retrieve from OKTA per request. | `1000` | No |
+
 
 The following is an example of a minimal configuration that can be applied to integrate with an Okta provider:
 

api/v1alpha1/groupsync_types.go

@@ -403,7 +403,7 @@ type OktaProvider struct {
 	// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Profile Key",xDescriptors={"urn:alm:descriptor:com.tectonic.ui:text"}
 	// +kubebuilder:validation:Optional
 	ProfileKey string `json:"profileKey"`
-	// GroupLimit is the maximum number of groups that can be synced. Default is "1000"
+	// GroupLimit is the maximum number of groups that are requested from OKTA per request.  Multiple requests will be made using pagination if you have more groups than this limit. Default is "1000"
 	// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Group Limit",xDescriptors={"urn:alm:descriptor:com.tectonic.ui:number"}
 	// +kubebuilder:validation:Optional
 	GroupLimit int `json:"groupLimit"`

config/crd/bases/redhatcop.redhat.io_groupsyncs.yaml

@@ -709,7 +709,7 @@ spec:
                             description: ExtractLoginUsername is true if Okta username's are defaulted to emails and you would like the username only
                             type: boolean
                           groupLimit:
-                            description: GroupLimit is the maximum number of groups that can be synced. Default is "1000"
+                            description: GroupLimit is the maximum number of groups that are requested from OKTA per request.  Multiple requests will be made using pagination if you have more groups than this limit. Default is "1000"
                             type: integer
                           groups:
                             description: Groups represents a filtered list of groups to synchronize

config/manifests/bases/group-sync-operator.clusterserviceversion.yaml

@@ -590,7 +590,7 @@ spec:
         path: providers[0].okta.extractLoginUsername
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:text
-      - description: GroupLimit is the maximum number of groups that can be synced.
+      - description: GroupLimit is the maximum number of groups that are requested from OKTA per request.  Multiple requests will be made using pagination if you have more groups than this limit.
           Default is "1000"
         displayName: Group Limit
         path: providers[0].okta.groupLimit
@@ -1003,6 +1003,7 @@ spec:
       | `appId` | Application ID of App Groups are assigned to | | Yes |
       | `extractLoginUsername` | Bool to determine if you should extract username from okta login | | No |
       | `profileKey` | Attribute field on Okta User Profile you would like to use as identity | `login` | No |
+      | `groupLimit` | Integer to set the maximum number of groups to retrieve from OKTA per request. | `1000` | No |
 
 
     The following is an example of a minimal configuration that can be applied to integrate with an Okta provider:

pkg/syncer/okta.go

@@ -4,11 +4,12 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"github.com/okta/okta-sdk-golang/v2/okta/query"
 	"net/url"
 	"strings"
 	"sync"
 
+	"github.com/okta/okta-sdk-golang/v2/okta/query"
+
 	"github.com/okta/okta-sdk-golang/v2/okta"
 	userv1 "github.com/openshift/api/user/v1"
 	"github.com/redhat-cop/group-sync-operator/api/v1alpha1"
@@ -177,13 +178,25 @@ func (o OktaSyncer) getGroups() ([]*okta.Group, error) {
 		groups []*okta.Group
 	)
 
-	appGroups, _, err := o.goOkta.Application.ListApplicationGroupAssignments(context.TODO(), o.Provider.AppId, query.NewQueryParams(query.WithLimit(int64(o.Provider.GroupLimit))))
+	appGroups, resp, err := o.goOkta.Application.ListApplicationGroupAssignments(context.TODO(), o.Provider.AppId, query.NewQueryParams(query.WithLimit(int64(o.Provider.GroupLimit))))
 
 	if err != nil {
 		oktaLogger.Error(err, "getting groups for specified application")
 		return nil, err
 	}
 
+	for resp.HasNextPage() {
+		var nextAppGroups []*okta.ApplicationGroupAssignment
+		resp, err = resp.Next(context.TODO(), &nextAppGroups)
+
+		if err != nil {
+			oktaLogger.Error(err, "getting groups for specified application")
+			return nil, err
+		}
+
+		appGroups = append(appGroups, nextAppGroups...)
+	}
+
 	groups, err = o.fetchGroupsAsync(appGroups)
 	return groups, err
 }