From 4fa58c09b5f83a17e46154709af1cf0f77524156 Mon Sep 17 00:00:00 2001 From: Giorgi Kikolashvili Date: Tue, 21 Jan 2025 15:08:49 +0100 Subject: [PATCH 1/4] Add GetJob pagination --- service/jobs/ext_api.go | 24 ++++ service/jobs/ext_api_test.go | 255 +++++++++++++++++++++++++++++++++++ 2 files changed, 279 insertions(+) diff --git a/service/jobs/ext_api.go b/service/jobs/ext_api.go index 0b8f3ee80..c0e6f8fd4 100644 --- a/service/jobs/ext_api.go +++ b/service/jobs/ext_api.go @@ -32,3 +32,27 @@ func (a *JobsAPI) GetRun(ctx context.Context, request GetRunRequest) (*Run, erro return run, nil } + +func (a *JobsAPI) Get(ctx context.Context, request GetJobRequest) (*Job, error) { + job, err := a.jobsImpl.Get(ctx, request) + if err != nil { + return nil, err + } + + pageToken := job.NextPageToken + for pageToken != "" { + request.PageToken = pageToken + nextJob, err := a.jobsImpl.Get(ctx, request) + if err != nil { + return nil, err + } + + job.Settings.Tasks = append(job.Settings.Tasks, nextJob.Settings.Tasks...) + job.Settings.JobClusters = append(job.Settings.JobClusters, nextJob.Settings.JobClusters...) + job.Settings.Parameters = append(job.Settings.Parameters, nextJob.Settings.Parameters...) + job.Settings.Environments = append(job.Settings.Environments, nextJob.Settings.Environments...) + pageToken = nextJob.NextPageToken + } + + return job, nil +} diff --git a/service/jobs/ext_api_test.go b/service/jobs/ext_api_test.go index e13a437f8..b951f371c 100644 --- a/service/jobs/ext_api_test.go +++ b/service/jobs/ext_api_test.go @@ -369,3 +369,258 @@ func TestGetRun(t *testing.T) { assert.EqualValues(t, 999, run.Tasks[0].RunId) }) } + +func TestGetJob(t *testing.T) { + ctx := context.Background() + + t.Run("job with no pagination", func(t *testing.T) { + var requestMocks qa.HTTPFixtures = []qa.HTTPFixture{ + { + Method: "GET", + Resource: "/api/2.2/jobs/get?job_id=514594995218126", + Response: Job{ + Settings: &JobSettings{ + Tasks: []Task{ + { + TaskKey: "task1", + }, + { + TaskKey: "task2", + }, + { + TaskKey: "task3", + }, + { + TaskKey: "task4", + }, + }, + }, + }, + }, + { + Method: "GET", + ReuseRequest: true, + Resource: "/api/2.1/jobs/get?run_id=514594995218126", + Response: Job{ + Settings: &JobSettings{ + Tasks: []Task{ + { + TaskKey: "task1", + }, + { + TaskKey: "task2", + }, + { + TaskKey: "task3", + }, + { + TaskKey: "task4", + }, + }, + }, + }, + }, + } + client, server := requestMocks.Client(t) + defer server.Close() + + mockJobsImpl := &jobsImpl{ + client: client, + } + api := &JobsAPI{jobsImpl: *mockJobsImpl} + + request := GetJobRequest{JobId: 514594995218126} + job, err := api.Get(ctx, request) + + assert.NoError(t, err) + assert.Equal(t, 4, len(job.Settings.Tasks)) + assert.EqualValues(t, "task1", job.Settings.Tasks[0].TaskKey) + assert.EqualValues(t, "task4", job.Settings.Tasks[3].TaskKey) + }) + + t.Run("job with multiple pages", func(t *testing.T) { + var requestMocks qa.HTTPFixtures = []qa.HTTPFixture{ + { + Method: "GET", + Resource: "/api/2.2/jobs/get?job_id=514594995218126", + Response: Job{ + Settings: &JobSettings{ + Tasks: []Task{ + { + TaskKey: "task1", + }, + { + TaskKey: "task2", + }, + }, + JobClusters: []JobCluster{ + { + JobClusterKey: "cluster1", + }, + { + JobClusterKey: "cluster2", + }, + }, + Parameters: []JobParameterDefinition{ + { + Name: "param1", + Default: "default1", + }, + { + Name: "param2", + Default: "default2", + }, + }, + Environments: []JobEnvironment{ + { + EnvironmentKey: "env1", + }, + }, + }, + NextPageToken: "token1", + }, + }, + { + Method: "GET", + Resource: "/api/2.2/jobs/get?job_id=514594995218126&page_token=token1", + Response: Job{ + Settings: &JobSettings{ + Tasks: []Task{ + { + TaskKey: "task3", + }, + { + TaskKey: "task4", + }, + }, + JobClusters: []JobCluster{ + { + JobClusterKey: "cluster3", + }, + { + JobClusterKey: "cluster4", + }, + }, + Parameters: []JobParameterDefinition{ + { + Name: "param3", + Default: "default3", + }, + }, + Environments: []JobEnvironment{ + { + EnvironmentKey: "env2", + }, + }, + }, + NextPageToken: "token2", + }, + }, + { + Method: "GET", + Resource: "/api/2.2/jobs/get?job_id=514594995218126&page_token=token2", + Response: Job{ + Settings: &JobSettings{ + Tasks: []Task{ + { + TaskKey: "task5", + }, + }, + Environments: []JobEnvironment{ + { + EnvironmentKey: "env3", + }, + }, + }, + }, + }, + { + Method: "GET", + ReuseRequest: true, + Resource: "/api/2.1/jobs/get?run_id=514594995218126", + Response: Job{ + Settings: &JobSettings{ + Tasks: []Task{ + { + TaskKey: "task1", + }, + { + TaskKey: "task2", + }, + { + TaskKey: "task3", + }, + { + TaskKey: "task4", + }, + { + TaskKey: "task5", + }, + }, + JobClusters: []JobCluster{ + { + JobClusterKey: "cluster1", + }, + { + JobClusterKey: "cluster2", + }, + { + JobClusterKey: "cluster3", + }, + { + JobClusterKey: "cluster4", + }, + }, + Parameters: []JobParameterDefinition{ + { + Name: "param1", + Default: "default1", + }, + { + Name: "param2", + Default: "default2", + }, + { + Name: "param3", + Default: "default3", + }, + }, + Environments: []JobEnvironment{ + { + EnvironmentKey: "env1", + }, + { + EnvironmentKey: "env2", + }, + { + EnvironmentKey: "env3", + }, + }, + }, + }, + }, + } + client, server := requestMocks.Client(t) + defer server.Close() + + mockJobsImpl := &jobsImpl{ + client: client, + } + api := &JobsAPI{jobsImpl: *mockJobsImpl} + + request := GetJobRequest{JobId: 514594995218126} + job, err := api.Get(ctx, request) + + assert.NoError(t, err) + assert.Equal(t, 5, len(job.Settings.Tasks)) + assert.Equal(t, 4, len(job.Settings.JobClusters)) + assert.Equal(t, 3, len(job.Settings.Parameters)) + assert.Equal(t, 3, len(job.Settings.Environments)) + assert.EqualValues(t, "task1", job.Settings.Tasks[0].TaskKey) + assert.EqualValues(t, "task4", job.Settings.Tasks[3].TaskKey) + assert.EqualValues(t, "task5", job.Settings.Tasks[4].TaskKey) + assert.EqualValues(t, "cluster3", job.Settings.JobClusters[2].JobClusterKey) + assert.EqualValues(t, "param3", job.Settings.Parameters[2].Name) + assert.EqualValues(t, "env3", job.Settings.Environments[2].EnvironmentKey) + }) +} From 97f359cb6100291b9921751289edbb9624bd2579 Mon Sep 17 00:00:00 2001 From: Giorgi Kikolashvili Date: Tue, 21 Jan 2025 15:20:19 +0100 Subject: [PATCH 2/4] Fix test --- service/jobs/ext_api_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/service/jobs/ext_api_test.go b/service/jobs/ext_api_test.go index b951f371c..1e788172d 100644 --- a/service/jobs/ext_api_test.go +++ b/service/jobs/ext_api_test.go @@ -400,7 +400,7 @@ func TestGetJob(t *testing.T) { { Method: "GET", ReuseRequest: true, - Resource: "/api/2.1/jobs/get?run_id=514594995218126", + Resource: "/api/2.1/jobs/get?job_id=514594995218126", Response: Job{ Settings: &JobSettings{ Tasks: []Task{ @@ -537,7 +537,7 @@ func TestGetJob(t *testing.T) { { Method: "GET", ReuseRequest: true, - Resource: "/api/2.1/jobs/get?run_id=514594995218126", + Resource: "/api/2.1/jobs/get?job_id=514594995218126", Response: Job{ Settings: &JobSettings{ Tasks: []Task{ From 25569ae4b45134e54a02905b0fa72a4a0b8333d1 Mon Sep 17 00:00:00 2001 From: Giorgi Kikolashvili Date: Fri, 24 Jan 2025 10:44:54 +0100 Subject: [PATCH 3/4] Simplify pagination logic --- service/jobs/ext_api.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/service/jobs/ext_api.go b/service/jobs/ext_api.go index c0e6f8fd4..653bb59ef 100644 --- a/service/jobs/ext_api.go +++ b/service/jobs/ext_api.go @@ -33,15 +33,16 @@ func (a *JobsAPI) GetRun(ctx context.Context, request GetRunRequest) (*Run, erro return run, nil } +// Get retrieves a job based on the provided request. +// It handles pagination if the job contains multiple tasks, job_clusters, job_parameters or environments. func (a *JobsAPI) Get(ctx context.Context, request GetJobRequest) (*Job, error) { job, err := a.jobsImpl.Get(ctx, request) if err != nil { return nil, err } - pageToken := job.NextPageToken - for pageToken != "" { - request.PageToken = pageToken + for job.NextPageToken != "" { + request.PageToken = job.NextPageToken nextJob, err := a.jobsImpl.Get(ctx, request) if err != nil { return nil, err @@ -51,7 +52,7 @@ func (a *JobsAPI) Get(ctx context.Context, request GetJobRequest) (*Job, error) job.Settings.JobClusters = append(job.Settings.JobClusters, nextJob.Settings.JobClusters...) job.Settings.Parameters = append(job.Settings.Parameters, nextJob.Settings.Parameters...) job.Settings.Environments = append(job.Settings.Environments, nextJob.Settings.Environments...) - pageToken = nextJob.NextPageToken + job.NextPageToken = nextJob.NextPageToken } return job, nil From 61ce55b7efa71c6386e9dad3b5e4bccb19395ba5 Mon Sep 17 00:00:00 2001 From: Giorgi Kikolashvili Date: Thu, 6 Feb 2025 19:11:37 +0100 Subject: [PATCH 4/4] Add comments --- service/jobs/ext_api.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/service/jobs/ext_api.go b/service/jobs/ext_api.go index 653bb59ef..1f4c714cc 100644 --- a/service/jobs/ext_api.go +++ b/service/jobs/ext_api.go @@ -41,6 +41,7 @@ func (a *JobsAPI) Get(ctx context.Context, request GetJobRequest) (*Job, error) return nil, err } + // jobs/get response includes next_page_token as long as there are more pages to fetch. for job.NextPageToken != "" { request.PageToken = job.NextPageToken nextJob, err := a.jobsImpl.Get(ctx, request) @@ -48,6 +49,7 @@ func (a *JobsAPI) Get(ctx context.Context, request GetJobRequest) (*Job, error) return nil, err } + // Each new page of jobs/get response includes the next page of the tasks, job_clusters, job_parameters, and environments. job.Settings.Tasks = append(job.Settings.Tasks, nextJob.Settings.Tasks...) job.Settings.JobClusters = append(job.Settings.JobClusters, nextJob.Settings.JobClusters...) job.Settings.Parameters = append(job.Settings.Parameters, nextJob.Settings.Parameters...)