From 2a87c50c9fecfb67cbd4a6b57e6bd81173306fb6 Mon Sep 17 00:00:00 2001 From: Giorgi Kikolashvili Date: Tue, 21 Jan 2025 13:21:09 +0100 Subject: [PATCH 1/2] RunGet also paginates clusters and parameters --- service/jobs/ext_api.go | 10 ++++--- service/jobs/ext_api_test.go | 54 ++++++++++++++++++++++++++++++++---- 2 files changed, 55 insertions(+), 9 deletions(-) diff --git a/service/jobs/ext_api.go b/service/jobs/ext_api.go index 0b8f3ee80..6b7adc47f 100644 --- a/service/jobs/ext_api.go +++ b/service/jobs/ext_api.go @@ -14,9 +14,8 @@ func (a *JobsAPI) GetRun(ctx context.Context, request GetRunRequest) (*Run, erro // When querying a ForEach task run, a page token is returned when there are more than 100 iterations. Only a single task is returned, corresponding to the ForEach task itself. Therefore, the client only reads the iterations from the next page and not the tasks. isPaginatingIterations := len(run.Iterations) > 0 - pageToken := run.NextPageToken - for pageToken != "" { - request.PageToken = pageToken + for run.NextPageToken != "" { + request.PageToken = run.NextPageToken nextRun, err := a.jobsImpl.GetRun(ctx, request) if err != nil { return nil, err @@ -27,7 +26,10 @@ func (a *JobsAPI) GetRun(ctx context.Context, request GetRunRequest) (*Run, erro } else { run.Tasks = append(run.Tasks, nextRun.Tasks...) } - pageToken = nextRun.NextPageToken + run.JobClusters = append(run.JobClusters, nextRun.JobClusters...) + run.JobParameters = append(run.JobParameters, nextRun.JobParameters...) + run.RepairHistory = append(run.RepairHistory, nextRun.RepairHistory...) + run.NextPageToken = nextRun.NextPageToken } return run, nil diff --git a/service/jobs/ext_api_test.go b/service/jobs/ext_api_test.go index e13a437f8..acec84215 100644 --- a/service/jobs/ext_api_test.go +++ b/service/jobs/ext_api_test.go @@ -29,7 +29,6 @@ func TestGetRun(t *testing.T) { TaskKey: "task2", }, }, - NextPageToken: "", }, }, { @@ -175,7 +174,7 @@ func TestGetRun(t *testing.T) { assert.Equal(t, expected, run.Tasks) }) - t.Run("clusters array is not increased when paginated", func(t *testing.T) { + t.Run("clusters array is also paginated", func(t *testing.T) { var requestMocks qa.HTTPFixtures = []qa.HTTPFixture{ { Method: "GET", @@ -199,6 +198,16 @@ func TestGetRun(t *testing.T) { JobClusterKey: "cluster2", }, }, + JobParameters: []JobParameter{ + { + Name: "key1", + Value: "value1", + }, + { + Name: "key2", + Value: "value2", + }, + }, NextPageToken: "token1", }, }, @@ -218,10 +227,17 @@ func TestGetRun(t *testing.T) { }, JobClusters: []JobCluster{ { - JobClusterKey: "cluster1", + JobClusterKey: "cluster3", }, + }, + JobParameters: []JobParameter{ { - JobClusterKey: "cluster2", + Name: "key3", + Value: "value3", + }, + { + Name: "key4", + Value: "value4", }, }, }, @@ -253,6 +269,27 @@ func TestGetRun(t *testing.T) { { JobClusterKey: "cluster2", }, + { + JobClusterKey: "cluster3", + }, + }, + JobParameters: []JobParameter{ + { + Name: "key1", + Value: "value1", + }, + { + Name: "key2", + Value: "value2", + }, + { + Name: "key3", + Value: "value3", + }, + { + Name: "key4", + Value: "value4", + }, }, }, }, @@ -269,9 +306,16 @@ func TestGetRun(t *testing.T) { run, err := api.GetRun(ctx, request) assert.NoError(t, err) - assert.Equal(t, 2, len(run.JobClusters)) + assert.Equal(t, 4, len(run.Tasks)) + assert.Equal(t, 3, len(run.JobClusters)) + assert.Equal(t, 4, len(run.JobParameters)) assert.Equal(t, "cluster1", run.JobClusters[0].JobClusterKey) assert.Equal(t, "cluster2", run.JobClusters[1].JobClusterKey) + assert.Equal(t, "cluster3", run.JobClusters[2].JobClusterKey) + assert.Equal(t, "key1", run.JobParameters[0].Name) + assert.Equal(t, "value1", run.JobParameters[0].Value) + assert.Equal(t, "key4", run.JobParameters[3].Name) + assert.Equal(t, "value4", run.JobParameters[3].Value) }) t.Run("run with two iterations pages", func(t *testing.T) { From a0100773ac14d51bafe177f0e88f4cf42a2e4b72 Mon Sep 17 00:00:00 2001 From: Giorgi Kikolashvili Date: Tue, 4 Feb 2025 10:41:47 +0100 Subject: [PATCH 2/2] Add comments --- service/jobs/ext_api.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/service/jobs/ext_api.go b/service/jobs/ext_api.go index 6b7adc47f..e7ea090c3 100644 --- a/service/jobs/ext_api.go +++ b/service/jobs/ext_api.go @@ -14,6 +14,7 @@ func (a *JobsAPI) GetRun(ctx context.Context, request GetRunRequest) (*Run, erro // When querying a ForEach task run, a page token is returned when there are more than 100 iterations. Only a single task is returned, corresponding to the ForEach task itself. Therefore, the client only reads the iterations from the next page and not the tasks. isPaginatingIterations := len(run.Iterations) > 0 + // runs/get response includes next_page_token as long as there are more pages to fetch. for run.NextPageToken != "" { request.PageToken = run.NextPageToken nextRun, err := a.jobsImpl.GetRun(ctx, request) @@ -26,6 +27,7 @@ func (a *JobsAPI) GetRun(ctx context.Context, request GetRunRequest) (*Run, erro } else { run.Tasks = append(run.Tasks, nextRun.Tasks...) } + // Each new page of runs/get response includes the next page of the job_clusters, job_parameters, and repair history. run.JobClusters = append(run.JobClusters, nextRun.JobClusters...) run.JobParameters = append(run.JobParameters, nextRun.JobParameters...) run.RepairHistory = append(run.RepairHistory, nextRun.RepairHistory...)