Skip to content

[Internal] GetRun logic paginates more arrays #1132

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Feb 13, 2025
Merged

[Internal] GetRun logic paginates more arrays #1132

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2a87c50
RunGet also paginates clusters and parameters
gkiko10 Jan 21, 2025
a010077
Add comments
gkiko10 Feb 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 8 additions & 4 deletions service/jobs/ext_api.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@ func (a *JobsAPI) GetRun(ctx context.Context, request GetRunRequest) (*Run, erro
// When querying a ForEach task run, a page token is returned when there are more than 100 iterations. Only a single task is returned, corresponding to the ForEach task itself. Therefore, the client only reads the iterations from the next page and not the tasks.
isPaginatingIterations := len(run.Iterations) > 0

pageToken := run.NextPageToken
for pageToken != "" {
request.PageToken = pageToken
// runs/get response includes next_page_token as long as there are more pages to fetch.
for run.NextPageToken != "" {
request.PageToken = run.NextPageToken
nextRun, err := a.jobsImpl.GetRun(ctx, request)
if err != nil {
return nil, err
Expand All @@ -27,7 +27,11 @@ func (a *JobsAPI) GetRun(ctx context.Context, request GetRunRequest) (*Run, erro
} else {
run.Tasks = append(run.Tasks, nextRun.Tasks...)
}
pageToken = nextRun.NextPageToken
// Each new page of runs/get response includes the next page of the job_clusters, job_parameters, and repair history.
run.JobClusters = append(run.JobClusters, nextRun.JobClusters...)
run.JobParameters = append(run.JobParameters, nextRun.JobParameters...)
run.RepairHistory = append(run.RepairHistory, nextRun.RepairHistory...)
run.NextPageToken = nextRun.NextPageToken
Comment on lines +31 to +34
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you add a comment to explain why we are doing this?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done

}

return run, nil
Expand Down
54 changes: 49 additions & 5 deletions service/jobs/ext_api_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@ func TestGetRun(t *testing.T) {
TaskKey: "task2",
},
},
NextPageToken: "",
},
},
{
Expand Down Expand Up @@ -175,7 +174,7 @@ func TestGetRun(t *testing.T) {
assert.Equal(t, expected, run.Tasks)
})

t.Run("clusters array is not increased when paginated", func(t *testing.T) {
t.Run("clusters array is also paginated", func(t *testing.T) {
var requestMocks qa.HTTPFixtures = []qa.HTTPFixture{
{
Method: "GET",
Expand All @@ -199,6 +198,16 @@ func TestGetRun(t *testing.T) {
JobClusterKey: "cluster2",
},
},
JobParameters: []JobParameter{
{
Name: "key1",
Value: "value1",
},
{
Name: "key2",
Value: "value2",
},
},
NextPageToken: "token1",
},
},
Expand All @@ -218,10 +227,17 @@ func TestGetRun(t *testing.T) {
},
JobClusters: []JobCluster{
{
JobClusterKey: "cluster1",
JobClusterKey: "cluster3",
},
},
JobParameters: []JobParameter{
{
JobClusterKey: "cluster2",
Name: "key3",
Value: "value3",
},
{
Name: "key4",
Value: "value4",
},
},
},
Expand Down Expand Up @@ -253,6 +269,27 @@ func TestGetRun(t *testing.T) {
{
JobClusterKey: "cluster2",
},
{
JobClusterKey: "cluster3",
},
},
JobParameters: []JobParameter{
{
Name: "key1",
Value: "value1",
},
{
Name: "key2",
Value: "value2",
},
{
Name: "key3",
Value: "value3",
},
{
Name: "key4",
Value: "value4",
},
},
},
},
Expand All @@ -269,9 +306,16 @@ func TestGetRun(t *testing.T) {
run, err := api.GetRun(ctx, request)

assert.NoError(t, err)
assert.Equal(t, 2, len(run.JobClusters))
assert.Equal(t, 4, len(run.Tasks))
assert.Equal(t, 3, len(run.JobClusters))
assert.Equal(t, 4, len(run.JobParameters))
assert.Equal(t, "cluster1", run.JobClusters[0].JobClusterKey)
assert.Equal(t, "cluster2", run.JobClusters[1].JobClusterKey)
assert.Equal(t, "cluster3", run.JobClusters[2].JobClusterKey)
assert.Equal(t, "key1", run.JobParameters[0].Name)
assert.Equal(t, "value1", run.JobParameters[0].Value)
assert.Equal(t, "key4", run.JobParameters[3].Name)
assert.Equal(t, "value4", run.JobParameters[3].Value)
})

t.Run("run with two iterations pages", func(t *testing.T) {
Expand Down
Loading