Skip to content

Commit 99c5a47

Browse files
marrivamarriva
authored
Merge pull request #24 from flant/github-checks
add github client
2 parents 018b48b + 4dec096 commit 99c5a47

File tree

10 files changed

+478
-33
lines changed

10 files changed

+478
-33
lines changed

‎cmd/projects/branches.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111
"github.com/flant/glaball/pkg/limiter"
1212
"github.com/flant/glaball/pkg/sort/v2"
1313
"github.com/flant/glaball/pkg/util"
14+
"github.com/google/go-github/v56/github"
1415
"github.com/hashicorp/go-hclog"
1516
"github.com/spf13/cobra"
1617
"github.com/xanzy/go-gitlab"
@@ -83,6 +84,11 @@ type ProjectBranch struct {
8384
Branches []*gitlab.Branch `json:"branches,omitempty"`
8485
}
8586

87+
type RepositoryBranch struct {
88+
Repository *github.Repository `json:"repository,omitempty"`
89+
Branch *github.Branch `json:"branch,omitempty"`
90+
}
91+
8692
func BranchesListCmd() error {
8793
if !sort.ValidOrderBy(branchOrderBy, ProjectBranch{}) {
8894
branchOrderBy = append(branchOrderBy, branchDefaultField)

‎cmd/projects/files.go

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package projects
33
import (
44
"bufio"
55
"bytes"
6+
"context"
67
"fmt"
78
"os"
89
"regexp"
@@ -12,6 +13,7 @@ import (
1213
"github.com/flant/glaball/pkg/client"
1314
"github.com/flant/glaball/pkg/limiter"
1415
"github.com/flant/glaball/pkg/sort/v2"
16+
"github.com/google/go-github/v58/github"
1517
"gopkg.in/yaml.v3"
1618

1719
"github.com/flant/glaball/cmd/common"
@@ -230,6 +232,38 @@ func listProjectsFiles(h *client.Host, filepath, ref string, re []*regexp.Regexp
230232
}
231233
}
232234

235+
func listProjectsFilesFromGithub(h *client.Host, filepath, ref string, re []*regexp.Regexp, opt github.RepositoryListByOrgOptions,
236+
wg *limiter.Limiter, data chan<- interface{}, options ...gitlab.RequestOptionFunc) {
237+
238+
defer wg.Done()
239+
240+
ctx := context.TODO()
241+
wg.Lock()
242+
list, resp, err := h.GithubClient.Repositories.ListByOrg(ctx, h.Org, &opt)
243+
if err != nil {
244+
if err != nil {
245+
wg.Error(h, err)
246+
wg.Unlock()
247+
return
248+
}
249+
}
250+
wg.Unlock()
251+
252+
for _, v := range list {
253+
wg.Add(1)
254+
// TODO: handle deadlock when no files found
255+
go getRawFileFromGithub(h, v, filepath, ref, re, wg, data)
256+
}
257+
258+
if resp.NextPage > 0 {
259+
wg.Add(1)
260+
opt.Page = resp.NextPage
261+
go listProjectsFilesFromGithub(h, filepath, ref, re, opt, wg, data, options...)
262+
}
263+
264+
return
265+
}
266+
233267
func getRawFile(h *client.Host, project *gitlab.Project, filepath, ref string, re []*regexp.Regexp,
234268
wg *limiter.Limiter, data chan<- interface{}, options ...gitlab.RequestOptionFunc) {
235269

@@ -257,6 +291,46 @@ func getRawFile(h *client.Host, project *gitlab.Project, filepath, ref string, r
257291
}
258292
}
259293

294+
// TODO:
295+
func getRawFileFromGithub(h *client.Host, repository *github.Repository, filepath, ref string, re []*regexp.Regexp,
296+
wg *limiter.Limiter, data chan<- interface{}) {
297+
298+
defer wg.Done()
299+
300+
targetRef := ref
301+
if ref == "" {
302+
targetRef = repository.GetDefaultBranch()
303+
}
304+
// TODO:
305+
ctx := context.TODO()
306+
wg.Lock()
307+
fileContent, _, resp, err := h.GithubClient.Repositories.GetContents(ctx,
308+
repository.Owner.GetLogin(),
309+
repository.GetName(),
310+
filepath,
311+
&github.RepositoryContentGetOptions{Ref: targetRef})
312+
wg.Unlock()
313+
if err != nil {
314+
hclog.L().Named("files").Trace("get raw file error", "repository", repository.GetHTMLURL(), "error", err)
315+
return
316+
}
317+
318+
raw, err := fileContent.GetContent()
319+
if err != nil {
320+
hclog.L().Named("files").Trace("get raw file error", "repository", repository.GetHTMLURL(), "error", err)
321+
return
322+
}
323+
324+
for _, r := range re {
325+
if r.MatchString(raw) {
326+
data <- sort.Element{Host: h, Struct: &RepositoryFile{Repository: repository, Raw: raw}, Cached: resp.Header.Get("X-From-Cache") == "1"}
327+
hclog.L().Named("files").Trace("search pattern was found in file", "team", h.Team, "repository", h.Project, "host", h.URL,
328+
"repo", repository.GetHTMLURL(), "file", filepath, "pattern", r.String(), "content", hclog.Fmt("%s", raw))
329+
return
330+
}
331+
}
332+
}
333+
260334
func getGitlabCIFile(h *client.Host, check bool, project *gitlab.Project, re []*regexp.Regexp,
261335
wg *limiter.Limiter, data chan<- interface{}, options ...gitlab.RequestOptionFunc) {
262336

@@ -300,6 +374,11 @@ type ProjectFile struct {
300374
Raw []byte
301375
}
302376

377+
type RepositoryFile struct {
378+
Repository *github.Repository `json:"repository,omitempty"`
379+
Raw string
380+
}
381+
303382
type ProjectLintResult struct {
304383
Project *gitlab.Project `json:"project,omitempty"`
305384
MergedYaml map[string]interface{} `json:"merged_yaml,omitempty"`
@@ -398,7 +477,17 @@ func ListProjectsFiles(h *client.Host, filepath, ref string, re []*regexp.Regexp
398477
listProjectsFiles(h, filepath, ref, re, opt, wg, data, options...)
399478
}
400479

480+
func ListProjectsFilesFromGithub(h *client.Host, filepath, ref string, re []*regexp.Regexp, opt github.RepositoryListByOrgOptions,
481+
wg *limiter.Limiter, data chan<- interface{}, options ...gitlab.RequestOptionFunc) {
482+
listProjectsFilesFromGithub(h, filepath, ref, re, opt, wg, data, options...)
483+
}
484+
401485
func GetRawFile(h *client.Host, project *gitlab.Project, filepath, ref string, re []*regexp.Regexp,
402486
wg *limiter.Limiter, data chan<- interface{}, options ...gitlab.RequestOptionFunc) {
403487
getRawFile(h, project, filepath, ref, re, wg, data, options...)
404488
}
489+
490+
func GetRawFileFromGithub(h *client.Host, repository *github.Repository, filepath, ref string, re []*regexp.Regexp,
491+
wg *limiter.Limiter, data chan<- interface{}) {
492+
getRawFileFromGithub(h, repository, filepath, ref, re, wg, data)
493+
}

‎cmd/projects/list.go

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package projects
22

33
import (
4+
"context"
45
"encoding/csv"
56
"fmt"
67
"os"
@@ -12,6 +13,7 @@ import (
1213
"github.com/flant/glaball/pkg/limiter"
1314
"github.com/flant/glaball/pkg/sort/v2"
1415
"github.com/flant/glaball/pkg/util"
16+
"github.com/google/go-github/v58/github"
1517

1618
"github.com/flant/glaball/cmd/common"
1719

@@ -345,6 +347,31 @@ func listProjects(h *client.Host, opt gitlab.ListProjectsOptions, wg *limiter.Li
345347

346348
defer wg.Done()
347349

350+
// TODO:
351+
if h.GithubClient != nil {
352+
ctx := context.TODO()
353+
list, resp, err := h.GithubClient.Repositories.ListByOrg(ctx, h.Org,
354+
&github.RepositoryListByOrgOptions{ListOptions: github.ListOptions{PerPage: 100}},
355+
)
356+
if err != nil {
357+
wg.Error(h, err)
358+
return err
359+
}
360+
361+
for _, v := range list {
362+
data <- sort.Element{Host: h, Struct: v, Cached: resp.Header.Get("X-From-Cache") == "1"}
363+
}
364+
365+
if resp.NextPage > 0 {
366+
wg.Add(1)
367+
opt.Page = resp.NextPage
368+
go listProjects(h, opt, wg, data, options...)
369+
}
370+
371+
return nil
372+
373+
}
374+
348375
wg.Lock()
349376

350377
list, resp, err := h.Client.Projects.ListProjects(&opt, options...)
@@ -354,7 +381,7 @@ func listProjects(h *client.Host, opt gitlab.ListProjectsOptions, wg *limiter.Li
354381
return err
355382
}
356383

357-
wg.Unlock()
384+
wg.Unlock() // TODO: ratelimiter
358385

359386
for _, v := range list {
360387
data <- sort.Element{Host: h, Struct: v, Cached: resp.Header.Get("X-From-Cache") == "1"}

‎cmd/projects/mr.go

Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package projects
22

33
import (
4+
"context"
45
"encoding/csv"
56
"fmt"
67
"os"
@@ -10,6 +11,7 @@ import (
1011
"github.com/flant/glaball/pkg/limiter"
1112
"github.com/flant/glaball/pkg/sort/v2"
1213
"github.com/flant/glaball/pkg/util"
14+
"github.com/google/go-github/v58/github"
1315

1416
"github.com/flant/glaball/cmd/common"
1517

@@ -219,6 +221,74 @@ func ListProjectsByNamespace(h *client.Host, namespaces []string, opt gitlab.Lis
219221
return listProjectsByNamespace(h, namespaces, opt, wg, data, options...)
220222
}
221223

224+
func listRepositories(h *client.Host, archived bool, opt github.RepositoryListByOrgOptions,
225+
wg *limiter.Limiter, data chan<- interface{}) error {
226+
defer wg.Done()
227+
228+
ctx := context.TODO()
229+
wg.Lock()
230+
list, resp, err := h.GithubClient.Repositories.ListByOrg(ctx, h.Org, &opt)
231+
if err != nil {
232+
wg.Error(h, err)
233+
wg.Unlock()
234+
return err
235+
}
236+
wg.Unlock()
237+
238+
for _, v := range list {
239+
if v.GetArchived() == archived {
240+
data <- sort.Element{Host: h, Struct: v, Cached: resp.Header.Get("X-From-Cache") == "1"}
241+
}
242+
}
243+
244+
if resp.NextPage > 0 {
245+
wg.Add(1)
246+
opt.Page = resp.NextPage
247+
go listRepositories(h, archived, opt, wg, data)
248+
}
249+
250+
return nil
251+
}
252+
253+
func ListRepositories(h *client.Host, archived bool, opt github.RepositoryListByOrgOptions,
254+
wg *limiter.Limiter, data chan<- interface{}) error {
255+
return listRepositories(h, archived, opt, wg, data)
256+
}
257+
258+
func listRepositoriesByNamespace(h *client.Host, namespaces []string, archived bool, opt github.RepositoryListByOrgOptions,
259+
wg *limiter.Limiter, data chan<- interface{}) error {
260+
defer wg.Done()
261+
262+
ctx := context.TODO()
263+
wg.Lock()
264+
list, resp, err := h.GithubClient.Repositories.ListByOrg(ctx, h.Org, &opt)
265+
if err != nil {
266+
wg.Error(h, err)
267+
wg.Unlock()
268+
return err
269+
}
270+
wg.Unlock()
271+
272+
for _, v := range list {
273+
if v.GetArchived() == archived && (len(namespaces) == 0 || util.ContainsString(namespaces, v.GetName())) {
274+
data <- sort.Element{Host: h, Struct: v, Cached: resp.Header.Get("X-From-Cache") == "1"}
275+
}
276+
}
277+
278+
if resp.NextPage > 0 {
279+
wg.Add(1)
280+
opt.Page = resp.NextPage
281+
go listRepositoriesByNamespace(h, namespaces, archived, opt, wg, data)
282+
}
283+
284+
return nil
285+
}
286+
287+
func ListRepositoriesByNamespace(h *client.Host, namespaces []string, archived bool, opt github.RepositoryListByOrgOptions,
288+
wg *limiter.Limiter, data chan<- interface{}) error {
289+
return listRepositoriesByNamespace(h, namespaces, archived, opt, wg, data)
290+
}
291+
222292
func listMergeRequests(h *client.Host, project *gitlab.Project, opt gitlab.ListProjectMergeRequestsOptions,
223293
wg *limiter.Limiter, data chan<- interface{}, options ...gitlab.RequestOptionFunc) error {
224294
defer wg.Done()
@@ -360,12 +430,93 @@ func listMergeRequestsByAssigneeOrAuthorID(h *client.Host, project *gitlab.Proje
360430
return nil
361431
}
362432

433+
func listPullRequestsByAssigneeOrAuthorID(h *client.Host, repository *github.Repository, IDs []int,
434+
opt github.PullRequestListOptions, wg *limiter.Limiter, data chan<- interface{}) error {
435+
defer wg.Done()
436+
437+
ctx := context.TODO()
438+
wg.Lock()
439+
list, resp, err := h.GithubClient.PullRequests.List(ctx, h.Org, repository.GetName(), &opt)
440+
if err != nil {
441+
wg.Error(h, err)
442+
wg.Unlock()
443+
return err
444+
}
445+
wg.Unlock()
446+
447+
for _, v := range list {
448+
if len(IDs) == 0 {
449+
data <- sort.Element{Host: h, Struct: v, Cached: resp.Header.Get("X-From-Cache") == "1"}
450+
continue
451+
}
452+
453+
// if pr has assignee, then check and continue
454+
if v.Assignee != nil {
455+
if util.ContainsInt(IDs, int(v.Assignee.GetID())) {
456+
data <- sort.Element{Host: h, Struct: v, Cached: resp.Header.Get("X-From-Cache") == "1"}
457+
}
458+
continue
459+
}
460+
461+
// otherwise check the author
462+
if v.User != nil && util.ContainsInt(IDs, int(v.User.GetID())) {
463+
data <- sort.Element{Host: h, Struct: v, Cached: resp.Header.Get("X-From-Cache") == "1"}
464+
}
465+
}
466+
467+
if resp.NextPage > 0 {
468+
wg.Add(1)
469+
opt.Page = resp.NextPage
470+
go listPullRequestsByAssigneeOrAuthorID(h, repository, IDs, opt, wg, data)
471+
}
472+
473+
return nil
474+
}
475+
363476
// authorIDs slice must be sorted in ascending order
364477
func ListMergeRequestsByAuthorOrAssigneeID(h *client.Host, project *gitlab.Project, IDs []int, opt gitlab.ListProjectMergeRequestsOptions,
365478
wg *limiter.Limiter, data chan<- interface{}, options ...gitlab.RequestOptionFunc) error {
366479
return listMergeRequestsByAssigneeOrAuthorID(h, project, IDs, opt, wg, data, options...)
367480
}
368481

482+
// authorIDs slice must be sorted in ascending order
483+
func ListPullRequestsByAuthorOrAssigneeID(h *client.Host, repository *github.Repository, IDs []int,
484+
opt github.PullRequestListOptions, wg *limiter.Limiter, data chan<- interface{}) error {
485+
return listPullRequestsByAssigneeOrAuthorID(h, repository, IDs, opt, wg, data)
486+
}
487+
488+
func listPullRequests(h *client.Host, repository *github.Repository, opt github.PullRequestListOptions,
489+
wg *limiter.Limiter, data chan<- interface{}) error {
490+
defer wg.Done()
491+
492+
ctx := context.TODO()
493+
wg.Lock()
494+
list, resp, err := h.GithubClient.PullRequests.List(ctx, h.Org, repository.GetName(), &opt)
495+
if err != nil {
496+
wg.Error(h, err)
497+
wg.Unlock()
498+
return err
499+
}
500+
wg.Unlock()
501+
502+
for _, v := range list {
503+
data <- sort.Element{Host: h, Struct: v, Cached: resp.Header.Get("X-From-Cache") == "1"}
504+
}
505+
506+
if resp.NextPage > 0 {
507+
wg.Add(1)
508+
opt.Page = resp.NextPage
509+
go listPullRequests(h, repository, opt, wg, data)
510+
}
511+
512+
return nil
513+
}
514+
515+
func ListPullRequests(h *client.Host, repository *github.Repository, opt github.PullRequestListOptions,
516+
wg *limiter.Limiter, data chan<- interface{}, options ...gitlab.RequestOptionFunc) error {
517+
return listPullRequests(h, repository, opt, wg, data)
518+
}
519+
369520
func listMergeRequestsSearch(h *client.Host, project *gitlab.Project, key string, value *regexp.Regexp, opt gitlab.ListProjectMergeRequestsOptions,
370521
wg *limiter.Limiter, data chan<- interface{}, options ...gitlab.RequestOptionFunc) error {
371522
defer wg.Done()

0 commit comments

Comments
 (0)