Skip to content

Commit 027091e

Browse files
keackeac
committed
feat(ENScanGo): 新增 miit 和 kuaicha 插件支持
- 新增 miit 插件，提供 ICP备案、APP、小程序等信息查询 - 新增 kuaicha 数据源，提供企业信息、微信公众号、招聘信息等查询 - 重构部分代码以支持插件扩展 - 优化数据处理和导出逻辑 - 修复AQC查询BUG
1 parent c72cb5e commit 027091e

File tree

10 files changed

+456
-22
lines changed

10 files changed

+456
-22
lines changed

‎common/config.go

Lines changed: 11 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ type ENOptions struct {
4949
OutPutType string // 导出文件类型
5050
IsApiMode bool
5151
ENConfig *ENConfig
52-
BranchFilter string
52+
BranchFilter string
5353
}
5454

5555
// EnsGo EnScan 接口请求通用格式接口
@@ -67,6 +67,7 @@ type EnsGo struct {
6767
GsData string // get请求需要加的特殊参数 TYC
6868
Rf string // 返回数据关键词 TYC
6969
DataModuleId int // 企点获取数据ID点
70+
AppParams [2]string // 插件需要获取的参数
7071
}
7172

7273
// ENsD 通用返回内容格式
@@ -117,48 +118,46 @@ var DefaultAllInfos = []string{"icp", "weibo", "wechat", "app", "weibo", "job",
117118
var DefaultInfos = []string{"icp", "weibo", "wechat", "app", "wx_app"}
118119
var CanSearchAllInfos = []string{"enterprise_info", "icp", "weibo", "wechat", "app", "job", "wx_app", "copyright", "supplier", "invest", "branch", "holds", "partner"}
119120
var DeepSearch = []string{"invest", "branch", "holds", "supplier"}
120-
var ENSTypes = []string{"aqc", "tyc"}
121+
var ENSTypes = []string{"aqc", "tyc", "kc", "miit"}
121122
var ScanTypeKeys = map[string]string{
122123
"aqc": "爱企查",
123124
"qcc": "企查查",
124125
"tyc": "天眼查",
125126
"xlb": "小蓝本",
127+
"kc": "快查",
126128
"all": "全部查询",
127129
"aldzs": "阿拉丁",
128130
"coolapk": "酷安市场",
129131
"qimai": "七麦数据",
130132
"chinaz": "站长之家",
133+
"miit": "miitICP",
131134
}
132135

133136
// ENConfig YML配置文件，更改时注意变更 cfgYV 版本
134137
type ENConfig struct {
135138
Version float64 `yaml:"version"`
136139
Cookies struct {
137140
Aldzs string `yaml:"aldzs"`
138-
Xlb string `yaml:"xlb"`
139141
Aiqicha string `yaml:"aiqicha"`
140142
Qidian string `yaml:"qidian"`
143+
KuaiCha string `yaml:"kuaicha"`
141144
Tianyancha string `yaml:"tianyancha"`
142145
Tycid string `yaml:"tycid"`
143-
Qcc string `yaml:"qcc"`
144-
QccTid string `yaml:"qcctid"`
145146
QiMai string `yaml:"qimai"`
146-
ChinaZ string `yaml:"chinaz"`
147+
}
148+
App struct {
149+
MiitApi string `yaml:"miit_api"`
147150
}
148151
}
149152

150153
var cfgYName = filepath.Join(utils.GetConfigPath(), "config.yaml")
151154
var cfgYV = 0.4
152155
var configYaml = `version: 0.4
156+
app:
157+
miit_api: ''
153158
cookies:
154159
aiqicha: '' # 爱企查 Cookie
155160
tianyancha: '' # 天眼查 Cookie
156161
tycid: '' # 天眼查 CApi ID(capi.tianyancha.com)
157-
qcc: '' # 企查查 Cookie
158-
qcctid: '' # 企查查 TID console.log(window.tid)
159-
aldzs: '' # 阿拉丁 Cookie
160-
xlb: '' # 小蓝本 Token
161162
qimai: '' # 七麦数据 Cookie
162-
chinaz: '' # 站长之家 Cookie
163-
veryvp: '' # veryvp Cookie
164163
`

‎common/parse.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -148,9 +148,10 @@ func Parse(options *ENOptions) {
148148
options.GetField = utils.SetStr(options.GetField)
149149

150150
if options.IsNoMerge {
151+
options.IsMergeOut = !options.IsNoMerge
151152
gologger.Info().Msgf("批量查询文件将单独导出！\n")
152153
}
153-
options.IsMergeOut = !options.IsNoMerge
154+
154155
options.GetField = utils.SetStr(options.GetField)
155156
options.ENConfig = conf
156157
}

‎interface/enscan.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,3 +12,8 @@ type ENScan interface {
1212
GetCompanyBaseInfoById(pid string) (gjson.Result, map[string]*common.EnsGo)
1313
GetEnInfoList(pid string, enMap *common.EnsGo) ([]gjson.Result, error)
1414
}
15+
16+
type App interface {
17+
GetInfoList(keyword string, types string) []gjson.Result
18+
GetENMap() map[string]*common.EnsGo
19+
}

‎internal/aiqicha/aiqicha.go

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ func (h *AQC) AdvanceFilter() ([]gjson.Result, error) {
2929
rq, _ := pageParseJson(content)
3030
enList := rq.Get("resultList").Array()
3131
//enList := gjson.Get(content, "data.resultList").Array()
32+
3233
if len(enList) == 0 {
3334
gologger.Debug().Str("查询请求", name).Msg(content)
3435
return enList, fmt.Errorf("【AQC】没有查询到关键词 ⌈%s⌋", name)
@@ -127,7 +128,12 @@ func getInfoList(pid string, types string, options *common.ENOptions) []gjson.Re
127128
for _, y := range listData {
128129
for _, o := range y.Get("domain").Array() {
129130
valueTmp, _ := sjson.Set(y.Raw, "domain", o.String())
130-
valueTmp, _ = sjson.Set(valueTmp, "homeSite", y.Get("homeSite").Array()[0].String())
131+
tmpHomeSite := y.Get("homeSite").Array()
132+
tmpStr := ""
133+
if len(tmpHomeSite) > 0 {
134+
tmpStr = tmpHomeSite[0].String()
135+
}
136+
valueTmp, _ = sjson.Set(valueTmp, "homeSite", tmpStr)
131137
tmp = append(tmp, gjson.Parse(valueTmp))
132138
}
133139
}

‎internal/app/miit/bean.go

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
package miit
2+
3+
import (
4+
"crypto/tls"
5+
"github.com/imroc/req/v3"
6+
"github.com/wgpsec/ENScan/common"
7+
"github.com/wgpsec/ENScan/common/gologger"
8+
"time"
9+
)
10+
11+
func getENMap() map[string]*common.EnsGo {
12+
ensInfoMap := make(map[string]*common.EnsGo)
13+
ensInfoMap = map[string]*common.EnsGo{
14+
"icp": {
15+
Name: "ICP备案",
16+
Api: "web",
17+
Field: []string{"", "domain", "domain", "serviceLicence", "unitName"},
18+
KeyWord: []string{"网站名称", "网址", "域名", "网站备案/许可证号", "公司名称"},
19+
},
20+
"app": {
21+
Name: "APP",
22+
Api: "app",
23+
Field: []string{"serviceName", "serviceType", "version", "updateRecordTime", "", "", "", "", ""},
24+
KeyWord: []string{"名称", "分类", "当前版本", "更新时间", "简介", "logo", "Bundle ID", "链接", "market"},
25+
},
26+
"wx_app": {
27+
Name: "小程序",
28+
Api: "miniapp",
29+
Field: []string{"serviceName", "serviceType", "", "", ""},
30+
KeyWord: []string{"名称", "分类", "头像", "二维码", "阅读量"},
31+
},
32+
"fastapp": {
33+
Name: "快应用",
34+
Api: "fastapp",
35+
Field: []string{"serviceName", "serviceType", "", "", ""},
36+
KeyWord: []string{"名称", "分类", "头像", "二维码", "阅读量"},
37+
},
38+
}
39+
for k := range ensInfoMap {
40+
// 获取插件需要的信息
41+
ensInfoMap[k].AppParams = [2]string{"enterprise_info", "name"}
42+
ensInfoMap[k].KeyWord = append(ensInfoMap[k].KeyWord, "数据关联 ")
43+
ensInfoMap[k].Field = append(ensInfoMap[k].Field, "inFrom")
44+
}
45+
return ensInfoMap
46+
}
47+
48+
func getReq(url string, data string, options *common.ENOptions) string {
49+
client := req.C()
50+
client.SetTLSFingerprintChrome()
51+
client.SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true})
52+
client.SetTimeout(time.Duration(options.TimeOut) * time.Minute)
53+
if options.Proxy != "" {
54+
client.SetProxyURL(options.Proxy)
55+
}
56+
client.SetCommonHeaders(map[string]string{
57+
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.80 Safari/537.36 Edg/98.0.1108.43",
58+
"Accept": "text/html, application/xhtml+xml, image/jxr, */*",
59+
"Content-Type": "application/json;charset=UTF-8",
60+
})
61+
//加入随机延时
62+
time.Sleep(time.Duration(options.GetDelayRTime()) * time.Second)
63+
clientR := client.R()
64+
65+
method := "GET"
66+
if data == "" {
67+
method = "GET"
68+
} else {
69+
method = "POST"
70+
clientR.SetBody(data)
71+
}
72+
73+
resp, err := clientR.Send(method, url)
74+
75+
if err != nil {
76+
if options.Proxy != "" {
77+
client.SetProxy(nil)
78+
}
79+
gologger.Error().Msgf("【miit】请求发生错误， %s 10秒后重试\n%s\n", url, err)
80+
time.Sleep(10 * time.Second)
81+
return getReq(url, data, options)
82+
}
83+
if resp.IsSuccessState() {
84+
return resp.String()
85+
} else {
86+
gologger.Error().Msgf("【miit】未知错误 %s\n", resp.StatusCode)
87+
}
88+
return ""
89+
}

‎internal/app/miit/miit.go

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
package miit
2+
3+
import (
4+
"github.com/tidwall/gjson"
5+
"github.com/wgpsec/ENScan/common"
6+
"github.com/wgpsec/ENScan/common/gologger"
7+
urlTool "net/url"
8+
"strconv"
9+
"time"
10+
)
11+
12+
type Miit struct {
13+
Options *common.ENOptions
14+
}
15+
16+
func (h *Miit) GetInfoList(keyword string, types string) []gjson.Result {
17+
enMap := getENMap()
18+
return getInfoList(keyword, enMap[types].Api, h.Options)
19+
}
20+
21+
func (h *Miit) GetENMap() map[string]*common.EnsGo {
22+
return getENMap()
23+
}
24+
25+
func getInfoList(keyword string, types string, options *common.ENOptions) []gjson.Result {
26+
url := options.ENConfig.App.MiitApi + "/query/" + types + "?page=1&search=" + urlTool.QueryEscape(keyword)
27+
content := getReq(url+"&page=1", "", options)
28+
var listData []gjson.Result
29+
data := gjson.Get(content, "data")
30+
listData = data.Get("list").Array()
31+
if data.Get("hasNextPage").Bool() {
32+
for i := 2; int(data.Get("pages").Int()) >= i; i++ {
33+
gologger.Info().Msgf("当前：%s,%d\n", types, i)
34+
reqUrls := url + "&page=" + strconv.Itoa(i)
35+
// 强制延时！！特别容易被封
36+
time.Sleep(5 * time.Second)
37+
content = getReq(reqUrls, "", options)
38+
listData = append(listData, gjson.Get(content, "data.list").Array()...)
39+
}
40+
}
41+
return listData
42+
}

‎internal/kuaicha/kuaicha.go

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
package kuaicha
2+
3+
import (
4+
"fmt"
5+
"github.com/tidwall/gjson"
6+
"github.com/wgpsec/ENScan/common"
7+
"github.com/wgpsec/ENScan/common/gologger"
8+
"math"
9+
"strconv"
10+
"strings"
11+
)
12+
13+
type KC struct {
14+
Options *common.ENOptions
15+
}
16+
17+
func (h *KC) AdvanceFilter() ([]gjson.Result, error) {
18+
name := h.Options.KeyWord
19+
url := "https://www.kuaicha365.com/enterprise_info_app/V1/search/company_search_pc"
20+
searchStr := `{"search_conditions":[],"keyword":"` + name + `","page":1,"page_size":20}"}`
21+
content := GetReq(url, searchStr, h.Options)
22+
content = strings.ReplaceAll(content, "<em>", "⌈")
23+
content = strings.ReplaceAll(content, "</em>", "⌋")
24+
enList := gjson.Get(content, "data.list").Array()
25+
if len(enList) == 0 {
26+
gologger.Debug().Str("查询请求", name).Msg(content)
27+
return enList, fmt.Errorf("【KC】没有查询到关键词 ⌈%s⌋ \n", name)
28+
}
29+
return enList, nil
30+
}
31+
32+
func (h *KC) GetENMap() map[string]*common.EnsGo {
33+
return getENMap()
34+
}
35+
36+
func (h *KC) GetEnsD() common.ENsD {
37+
ensD := common.ENsD{Name: h.Options.KeyWord, Pid: h.Options.CompanyID, Op: h.Options}
38+
return ensD
39+
}
40+
41+
func (h *KC) GetCompanyBaseInfoById(pid string) (gjson.Result, map[string]*common.EnsGo) {
42+
ensInfoMap := getENMap()
43+
detailRess := GetReq("https://www.kuaicha365.com/open/app/v1/pc_enterprise/basic/info?org_id="+pid, "", h.Options)
44+
detailRes := gjson.Get(detailRess, "data")
45+
// 匹配统计数据,需要从页面中提取数据来判断
46+
for k, _ := range ensInfoMap {
47+
ensInfoMap[k].Total = 1
48+
}
49+
return detailRes, ensInfoMap
50+
}
51+
52+
func (h *KC) GetEnInfoList(pid string, enMap *common.EnsGo) ([]gjson.Result, error) {
53+
return getInfoList(pid+enMap.Fids, enMap.Api, h.Options), nil
54+
}
55+
56+
func getInfoList(pid string, types string, options *common.ENOptions) []gjson.Result {
57+
urls := "https://www.kuaicha365.com/" + types
58+
if strings.Contains(types, "open/app/v1") {
59+
urls += "?page_size=10" + "&org_id=" + pid
60+
} else {
61+
urls += "?pageSize=10&&orgid=" + pid
62+
}
63+
content := GetReq(urls+"&page=1", "", options)
64+
var listData []gjson.Result
65+
if gjson.Get(content, "status_code").String() == "0" {
66+
data := gjson.Get(content, "data")
67+
68+
//判断是否多页，遍历获取所有数据
69+
pageCount := data.Get("total").Int()
70+
if data.Get("next_page").Bool() {
71+
for i := 1; int(math.Ceil(float64(pageCount)/float64(10))) >= i; i++ {
72+
gologger.Info().Msgf("当前：%s,%d\n", types, i)
73+
reqUrls := urls + "&page=" + strconv.Itoa(i)
74+
content = GetReq(reqUrls, "", options)
75+
listData = append(listData, gjson.Get(content, "data.list").Array()...)
76+
}
77+
} else {
78+
listData = data.Get("list").Array()
79+
}
80+
}
81+
return listData
82+
}

0 commit comments

Comments
 (0)