Skip to content

Commit 37272bd

Browse files
huangsamhuangsam
committed
Apply max constraint on search logic directly
1 parent a2a6b22 commit 37272bd

File tree

6 files changed

+63
-80
lines changed

6 files changed

+63
-80
lines changed

‎cmd/namigo/sub/search.go‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,14 @@ func SearchAction(c *cli.Context) error {
2020
if len(searchTerm) == 0 {
2121
return errors.New("Provide at least one search term")
2222
}
23+
maxResults := c.Int("max")
2324

2425
ptf := newSearchPortfolio()
2526
ptfErrorCount := 0
2627

2728
ptf.run(func(wg *sync.WaitGroup) {
2829
defer wg.Done()
29-
if searchResults, err := golang.SearchByScrape(searchTerm); err == nil {
30+
if searchResults, err := golang.SearchByScrape(searchTerm, maxResults); err == nil {
3031
fmt.Println("🟢 Load Golang results")
3132
ptf.results.golang = searchResults
3233
} else {
@@ -37,7 +38,7 @@ func SearchAction(c *cli.Context) error {
3738

3839
ptf.run(func(wg *sync.WaitGroup) {
3940
defer wg.Done()
40-
if searchResults, err := npm.SearchByScrape(searchTerm); err == nil {
41+
if searchResults, err := npm.SearchByScrape(searchTerm, maxResults); err == nil {
4142
fmt.Println("🟢 Load NPM results")
4243
ptf.results.npm = searchResults
4344
} else {
@@ -48,7 +49,7 @@ func SearchAction(c *cli.Context) error {
4849

4950
ptf.run(func(wg *sync.WaitGroup) {
5051
defer wg.Done()
51-
if searchResults, err := pypi.SearchByAPI(searchTerm); err == nil {
52+
if searchResults, err := pypi.SearchByAPI(searchTerm, maxResults); err == nil {
5253
fmt.Println("🟢 Load PyPI results")
5354
ptf.results.pypi = searchResults
5455
} else {
@@ -59,7 +60,7 @@ func SearchAction(c *cli.Context) error {
5960

6061
ptf.run(func(wg *sync.WaitGroup) {
6162
defer wg.Done()
62-
if probeResults, err := dns.SearchByProbe(searchTerm); err == nil {
63+
if probeResults, err := dns.SearchByProbe(searchTerm, maxResults); err == nil {
6364
fmt.Println("🟢 Load DNS results")
6465
ptf.results.DNS = probeResults
6566
} else {
@@ -84,11 +85,10 @@ func SearchAction(c *cli.Context) error {
8485

8586
f := &searchFormatter{}
8687

87-
maxResultCount := c.Int("max")
88-
util.PrintResults(ptf.results.golang, "Golang", maxResultCount, f.formatGo)
89-
util.PrintResults(ptf.results.npm, "NPM", maxResultCount, f.formatNPM)
90-
util.PrintResults(ptf.results.pypi, "PyPI", maxResultCount, f.formatPyPI)
91-
util.PrintResults(ptf.results.DNS, "DNS", maxResultCount, f.formatDNS)
88+
util.PrintResults(ptf.results.golang, "Golang", f.formatGo)
89+
util.PrintResults(ptf.results.npm, "NPM", f.formatNPM)
90+
util.PrintResults(ptf.results.pypi, "PyPI", f.formatPyPI)
91+
util.PrintResults(ptf.results.DNS, "DNS", f.formatDNS)
9292

9393
return nil
9494
}

‎internal/util/output.go‎

Lines changed: 9 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -6,45 +6,33 @@ import (
66
"github.com/huangsam/namigo/internal/model"
77
)
88

9-
func PrintResults(results any, label string, maxResults int, format func(any) string) {
9+
func PrintResults(results any, label string, format func(any) string) {
1010
switch res := results.(type) {
1111
case []model.GoPackageResult:
1212
if len(res) > 0 {
13-
fmt.Printf("%d %s results found. First %d are:\n", len(res), label, maxResults)
14-
for i, r := range res {
15-
if i >= maxResults {
16-
break
17-
}
13+
fmt.Printf("%d %s results found:\n", len(res), label)
14+
for _, r := range res {
1815
fmt.Println(format(r))
1916
}
2017
}
2118
case []model.NPMPackageResult:
2219
if len(res) > 0 {
23-
fmt.Printf("%d %s results found. First %d are:\n", len(res), label, maxResults)
24-
for i, r := range res {
25-
if i >= maxResults {
26-
break
27-
}
20+
fmt.Printf("%d %s results found:\n", len(res), label)
21+
for _, r := range res {
2822
fmt.Println(format(r))
2923
}
3024
}
3125
case []model.PyPIPackageResult:
3226
if len(res) > 0 {
33-
fmt.Printf("%d %s results found. First %d are:\n", len(res), label, maxResults)
34-
for i, r := range res {
35-
if i >= maxResults {
36-
break
37-
}
27+
fmt.Printf("%d %s results found:\n", len(res), label)
28+
for _, r := range res {
3829
fmt.Println(format(r))
3930
}
4031
}
4132
case []model.DNSResult:
4233
if len(res) > 0 {
43-
fmt.Printf("%d %s results found. First %d are:\n", len(res), label, maxResults)
44-
for i, r := range res {
45-
if i >= maxResults {
46-
break
47-
}
34+
fmt.Printf("%d %s results found:\n", len(res), label)
35+
for _, r := range res {
4836
fmt.Println(format(r))
4937
}
5038
}

‎pkg/dns/search.go‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ import (
88
"github.com/huangsam/namigo/internal/model"
99
)
1010

11-
func SearchByProbe(name string) ([]model.DNSResult, error) {
11+
func SearchByProbe(name string, max int) ([]model.DNSResult, error) {
1212
var wg sync.WaitGroup
1313
domains := []string{"com", "org", "net", "io", "tech", "ai", "me", "shop"}
1414
domainChan := make(chan string)
@@ -22,18 +22,26 @@ func SearchByProbe(name string) ([]model.DNSResult, error) {
2222

2323
result := []model.DNSResult{}
2424
errorCount := 0
25+
var mu sync.Mutex
2526

2627
for i := 0; i < 4; i++ {
2728
wg.Add(1)
2829
go func() {
2930
defer wg.Done()
3031
for domain := range domainChan {
32+
mu.Lock()
33+
if len(result) >= max {
34+
return
35+
}
36+
mu.Unlock()
3137
fullDomain := fmt.Sprintf("%s.%s", name, domain)
3238
ips, err := net.LookupIP(fullDomain)
3339
if err != nil {
3440
errorCount++
3541
}
42+
mu.Lock()
3643
result = append(result, model.DNSResult{FQDN: fullDomain, IPList: ips})
44+
mu.Unlock()
3745
}
3846
}()
3947
}

‎pkg/golang/search.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ import (
1111
)
1212

1313
// SearchByScrape searches for Go packages by scraping pkg.go.dev.
14-
func SearchByScrape(name string) ([]model.GoPackageResult, error) {
14+
func SearchByScrape(name string, max int) ([]model.GoPackageResult, error) {
1515
client := &http.Client{Timeout: 5 * time.Second}
1616
pipeline := util.NewDocumentPipeline(client, listing(name))
1717
doc, err := pipeline.Execute()
@@ -22,6 +22,10 @@ func SearchByScrape(name string) ([]model.GoPackageResult, error) {
2222
result := []model.GoPackageResult{}
2323

2424
doc.Find(".SearchSnippet").Each(func(i int, section *goquery.Selection) {
25+
if len(result) >= max {
26+
return
27+
}
28+
2529
content := strings.Fields(section.Find("h2").Text())
2630
pkg, path := content[0], content[1]
2731
if !strings.Contains(pkg, name) && !strings.Contains(path, name) {

‎pkg/npm/search.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ import (
1111
)
1212

1313
// SearchByScrape searches for NPM packages by scraping www.npmjs.com.
14-
func SearchByScrape(name string) ([]model.NPMPackageResult, error) {
14+
func SearchByScrape(name string, max int) ([]model.NPMPackageResult, error) {
1515
client := &http.Client{Timeout: 5 * time.Second}
1616
pipeline := util.NewDocumentPipeline(client, listing(name))
1717
doc, err := pipeline.Execute()
@@ -22,6 +22,10 @@ func SearchByScrape(name string) ([]model.NPMPackageResult, error) {
2222
result := []model.NPMPackageResult{}
2323

2424
doc.Find("main section").Each(func(i int, section *goquery.Selection) {
25+
if len(result) >= max {
26+
return
27+
}
28+
2529
pkg := section.Find("h3").Text()
2630

2731
match := section.Find("span#pkg-list-exact-match").Text()

‎pkg/pypi/search.go‎

Lines changed: 26 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ import (
44
"encoding/json"
55
"net/http"
66
"strings"
7-
"sync"
87
"time"
98

109
"github.com/huangsam/namigo/internal/model"
@@ -15,7 +14,7 @@ import (
1514
const workerCount = 5
1615

1716
// SearchByAPI searches for PyPI packages by querying pypi.org.
18-
func SearchByAPI(name string) ([]model.PyPIPackageResult, error) {
17+
func SearchByAPI(name string, max int) ([]model.PyPIPackageResult, error) {
1918
client := &http.Client{Timeout: 5 * time.Second}
2019

2120
b, err := util.RESTAPIQuery(client, listing())
@@ -28,55 +27,35 @@ func SearchByAPI(name string) ([]model.PyPIPackageResult, error) {
2827
return []model.PyPIPackageResult{}, err
2928
}
3029

31-
taskChan := make(chan string)
32-
33-
go func() {
34-
count := 0
35-
for _, project := range listingRes.Projects {
36-
if strings.HasPrefix(project.Name, name) {
37-
taskChan <- project.Name
38-
count++
39-
}
40-
if count >= 50 { // That's probably enough!
41-
break
42-
}
43-
}
44-
close(taskChan)
45-
}()
46-
4730
result := []model.PyPIPackageResult{}
4831

49-
worker := func() {
50-
for item := range taskChan {
51-
bd, err := util.RESTAPIQuery(client, detail(item))
52-
if err != nil {
53-
continue
54-
}
55-
var detailRes PypiDetailResponse
56-
if err := json.Unmarshal(bd, &detailRes); err != nil {
57-
continue
58-
}
59-
description := detailRes.Info.Summary
60-
if len(description) == 0 {
61-
description = model.NoDescription
62-
}
63-
author := detailRes.Info.Author
64-
if len(author) == 0 {
65-
author = model.NoAuthor
66-
}
67-
result = append(result, model.PyPIPackageResult{Name: item, Description: description, Author: author})
32+
count := 0
33+
for _, project := range listingRes.Projects {
34+
if !strings.HasPrefix(project.Name, name) {
35+
continue
36+
}
37+
bd, err := util.RESTAPIQuery(client, detail(project.Name))
38+
if err != nil {
39+
continue
40+
}
41+
var detailRes PypiDetailResponse
42+
if err := json.Unmarshal(bd, &detailRes); err != nil {
43+
continue
44+
}
45+
description := detailRes.Info.Summary
46+
if len(description) == 0 {
47+
description = model.NoDescription
48+
}
49+
author := detailRes.Info.Author
50+
if len(author) == 0 {
51+
author = model.NoAuthor
52+
}
53+
result = append(result, model.PyPIPackageResult{Name: project.Name, Description: description, Author: author})
54+
count++
55+
if count >= max {
56+
break
6857
}
6958
}
7059

71-
var wg sync.WaitGroup
72-
wg.Add(workerCount)
73-
for i := 0; i < workerCount; i++ {
74-
go func() {
75-
defer wg.Done()
76-
worker()
77-
}()
78-
}
79-
wg.Wait()
80-
8160
return result, nil
8261
}

0 commit comments

Comments
 (0)