Skip to content

Commit 96f744f

Browse files
YangRuchengYangRucheng
committed
feat(github_releases): concurrently request the GitHub API
1 parent 2bdc5be commit 96f744f

File tree

4 files changed

+125
-81
lines changed

4 files changed

+125
-81
lines changed

‎drivers/github_releases/driver.go‎

Lines changed: 116 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"fmt"
66
"net/http"
77
"strings"
8+
"sync"
89

910
"github.com/alist-org/alist/v3/internal/driver"
1011
"github.com/alist-org/alist/v3/internal/errs"
@@ -36,88 +37,130 @@ func (d *GithubReleases) Drop(ctx context.Context) error {
3637
return nil
3738
}
3839

39-
func (d *GithubReleases) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) {
40-
files := make([]File, 0)
41-
path := fmt.Sprintf("/%s", strings.Trim(dir.GetPath(), "/"))
40+
// processPoint 处理单个挂载点的文件列表
41+
func (d *GithubReleases) processPoint(point *MountPoint, path string, args model.ListArgs) []File {
42+
var pointFiles []File
4243

43-
for i := range d.points {
44-
point := &d.points[i]
44+
if !d.Addition.ShowAllVersion { // latest
45+
point.RequestLatestRelease(d.GetRequest, args.Refresh)
46+
pointFiles = d.processLatestVersion(point, path)
47+
} else { // all version
48+
point.RequestReleases(d.GetRequest, args.Refresh)
49+
pointFiles = d.processAllVersions(point, path)
50+
}
4551

46-
if !d.Addition.ShowAllVersion { // latest
47-
point.RequestRelease(d.GetRequest, args.Refresh)
52+
return pointFiles
53+
}
4854

49-
if point.Point == path { // 与仓库路径相同
50-
files = append(files, point.GetLatestRelease()...)
51-
if d.Addition.ShowReadme {
52-
files = append(files, point.GetOtherFile(d.GetRequest, args.Refresh)...)
53-
}
54-
} else if strings.HasPrefix(point.Point, path) { // 仓库目录的父目录
55-
nextDir := GetNextDir(point.Point, path)
56-
if nextDir == "" {
57-
continue
58-
}
55+
// processLatestVersion 处理最新版本的逻辑
56+
func (d *GithubReleases) processLatestVersion(point *MountPoint, path string) []File {
57+
var pointFiles []File
5958

60-
hasSameDir := false
61-
for index := range files {
62-
if files[index].GetName() == nextDir {
63-
hasSameDir = true
64-
files[index].Size += point.GetLatestSize()
65-
break
66-
}
67-
}
68-
if !hasSameDir {
69-
files = append(files, File{
70-
Path: path + "/" + nextDir,
71-
FileName: nextDir,
72-
Size: point.GetLatestSize(),
73-
UpdateAt: point.Release.PublishedAt,
74-
CreateAt: point.Release.CreatedAt,
75-
Type: "dir",
76-
Url: "",
77-
})
78-
}
59+
if point.Point == path { // 与仓库路径相同
60+
pointFiles = append(pointFiles, point.GetLatestRelease()...)
61+
if d.Addition.ShowReadme {
62+
files := point.GetOtherFile(d.GetRequest, false)
63+
pointFiles = append(pointFiles, files...)
64+
}
65+
} else if strings.HasPrefix(point.Point, path) { // 仓库目录的父目录
66+
nextDir := GetNextDir(point.Point, path)
67+
if nextDir != "" {
68+
dirFile := File{
69+
Path: path + "/" + nextDir,
70+
FileName: nextDir,
71+
Size: point.GetLatestSize(),
72+
UpdateAt: point.Release.PublishedAt,
73+
CreateAt: point.Release.CreatedAt,
74+
Type: "dir",
75+
Url: "",
7976
}
80-
} else { // all version
81-
point.RequestReleases(d.GetRequest, args.Refresh)
77+
pointFiles = append(pointFiles, dirFile)
78+
}
79+
}
8280

83-
if point.Point == path { // 与仓库路径相同
84-
files = append(files, point.GetAllVersion()...)
85-
if d.Addition.ShowReadme {
86-
files = append(files, point.GetOtherFile(d.GetRequest, args.Refresh)...)
87-
}
88-
} else if strings.HasPrefix(point.Point, path) { // 仓库目录的父目录
89-
nextDir := GetNextDir(point.Point, path)
90-
if nextDir == "" {
91-
continue
92-
}
81+
return pointFiles
82+
}
9383

94-
hasSameDir := false
95-
for index := range files {
96-
if files[index].GetName() == nextDir {
97-
hasSameDir = true
98-
files[index].Size += point.GetAllVersionSize()
99-
break
100-
}
101-
}
102-
if !hasSameDir {
103-
files = append(files, File{
104-
FileName: nextDir,
105-
Path: path + "/" + nextDir,
106-
Size: point.GetAllVersionSize(),
107-
UpdateAt: (*point.Releases)[0].PublishedAt,
108-
CreateAt: (*point.Releases)[0].CreatedAt,
109-
Type: "dir",
110-
Url: "",
111-
})
112-
}
113-
} else if strings.HasPrefix(path, point.Point) { // 仓库目录的子目录
114-
tagName := GetNextDir(path, point.Point)
115-
if tagName == "" {
116-
continue
117-
}
84+
// processAllVersions 处理所有版本的逻辑
85+
func (d *GithubReleases) processAllVersions(point *MountPoint, path string) []File {
86+
var pointFiles []File
11887

119-
files = append(files, point.GetReleaseByTagName(tagName)...)
88+
if point.Point == path { // 与仓库路径相同
89+
pointFiles = append(pointFiles, point.GetAllVersion()...)
90+
if d.Addition.ShowReadme {
91+
files := point.GetOtherFile(d.GetRequest, false)
92+
pointFiles = append(pointFiles, files...)
93+
}
94+
} else if strings.HasPrefix(point.Point, path) { // 仓库目录的父目录
95+
nextDir := GetNextDir(point.Point, path)
96+
if nextDir != "" {
97+
dirFile := File{
98+
FileName: nextDir,
99+
Path: path + "/" + nextDir,
100+
Size: point.GetAllVersionSize(),
101+
UpdateAt: (*point.Releases)[0].PublishedAt,
102+
CreateAt: (*point.Releases)[0].CreatedAt,
103+
Type: "dir",
104+
Url: "",
120105
}
106+
pointFiles = append(pointFiles, dirFile)
107+
}
108+
} else if strings.HasPrefix(path, point.Point) { // 仓库目录的子目录
109+
tagName := GetNextDir(path, point.Point)
110+
if tagName != "" {
111+
pointFiles = append(pointFiles, point.GetReleaseByTagName(tagName)...)
112+
}
113+
}
114+
115+
return pointFiles
116+
}
117+
118+
// mergeFiles 合并文件列表，处理重复目录
119+
func (d *GithubReleases) mergeFiles(files *[]File, newFiles []File) {
120+
for _, newFile := range newFiles {
121+
if newFile.Type == "dir" {
122+
hasSameDir := false
123+
for index := range *files {
124+
if (*files)[index].GetName() == newFile.GetName() && (*files)[index].Type == "dir" {
125+
hasSameDir = true
126+
(*files)[index].Size += newFile.Size
127+
break
128+
}
129+
}
130+
if !hasSameDir {
131+
*files = append(*files, newFile)
132+
}
133+
} else {
134+
*files = append(*files, newFile)
135+
}
136+
}
137+
}
138+
139+
func (d *GithubReleases) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) {
140+
files := make([]File, 0)
141+
path := fmt.Sprintf("/%s", strings.Trim(dir.GetPath(), "/"))
142+
143+
if d.Addition.ConcurrentRequests && d.Addition.Token != "" { // 并发处理
144+
var mu sync.Mutex
145+
var wg sync.WaitGroup
146+
147+
for i := range d.points {
148+
wg.Add(1)
149+
go func(point *MountPoint) {
150+
defer wg.Done()
151+
pointFiles := d.processPoint(point, path, args)
152+
153+
mu.Lock()
154+
d.mergeFiles(&files, pointFiles)
155+
mu.Unlock()
156+
}(&d.points[i])
157+
}
158+
wg.Wait()
159+
} else { // 串行处理
160+
for i := range d.points {
161+
point := &d.points[i]
162+
pointFiles := d.processPoint(point, path, args)
163+
d.mergeFiles(&files, pointFiles)
121164
}
122165
}
123166

‎drivers/github_releases/meta.go‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,12 @@ import (
77

88
type Addition struct {
99
driver.RootID
10-
RepoStructure string `json:"repo_structure" type:"text" required:"true" default:"alistGo/alist" help:"structure:[path:]org/repo"`
11-
ShowReadme bool `json:"show_readme" type:"bool" default:"true" help:"show README、LICENSE file"`
12-
Token string `json:"token" type:"string" required:"false" help:"GitHub token, if you want to access private repositories or increase the rate limit"`
13-
ShowAllVersion bool `json:"show_all_version" type:"bool" default:"false" help:"show all versions"`
14-
GitHubProxy string `json:"gh_proxy" type:"string" default:"" help:"GitHub proxy, e.g. https://ghproxy.net/github.com or https://gh-proxy.com/github.com "`
10+
RepoStructure string `json:"repo_structure" type:"text" required:"true" default:"alistGo/alist" help:"structure:[path:]org/repo"`
11+
ShowReadme bool `json:"show_readme" type:"bool" default:"true" help:"show README、LICENSE file"`
12+
Token string `json:"token" type:"string" required:"false" help:"GitHub token, if you want to access private repositories or increase the rate limit"`
13+
ShowAllVersion bool `json:"show_all_version" type:"bool" default:"false" help:"show all versions"`
14+
ConcurrentRequests bool `json:"concurrent_requests" type:"bool" default:"false" help:"To concurrently request the GitHub API, you must enter a GitHub token"`
15+
GitHubProxy string `json:"gh_proxy" type:"string" default:"" help:"GitHub proxy, e.g. https://ghproxy.net/github.com or https://gh-proxy.com/github.com "`
1516
}
1617

1718
var config = driver.Config{

‎drivers/github_releases/types.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ type MountPoint struct {
1818
}
1919

2020
// 请求最新版本
21-
func (m *MountPoint) RequestRelease(get func(url string) (*resty.Response, error), refresh bool) {
21+
func (m *MountPoint) RequestLatestRelease(get func(url string) (*resty.Response, error), refresh bool) {
2222
if m.Repo == "" {
2323
return
2424
}

‎drivers/github_releases/util.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,8 @@ import (
66
"strings"
77

88
"github.com/alist-org/alist/v3/drivers/base"
9+
"github.com/alist-org/alist/v3/pkg/utils"
910
"github.com/go-resty/resty/v2"
10-
log "github.com/sirupsen/logrus"
1111
)
1212

1313
// 发送 GET 请求
@@ -23,7 +23,7 @@ func (d *GithubReleases) GetRequest(url string) (*resty.Response, error) {
2323
return nil, err
2424
}
2525
if res.StatusCode() != 200 {
26-
log.Warn("failed to get request: ", res.StatusCode(), res.String())
26+
utils.Log.Warnf("failed to get request: %s %d %s", url, res.StatusCode(), res.String())
2727
}
2828
return res, nil
2929
}

0 commit comments

Comments
 (0)