Skip to content

Commit fedf73a

Browse files
coder-hxlcoder-hxl
committed
Feat: Terminal prompt adjustment
1 parent d840e70 commit fedf73a

File tree

5 files changed

+50
-46
lines changed

5 files changed

+50
-46
lines changed

‎src/api.ts

Lines changed: 23 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ import {
1212
isUndefined,
1313
log,
1414
logError,
15+
logStart,
16+
logStatistics,
1517
logSuccess,
1618
logWarn,
1719
mkdirDirSync,
@@ -40,35 +42,29 @@ import { fingerprints } from './default'
4042

4143
// Extra config
4244
export interface ExtraCommonConfig {
43-
intervalTime: IntervalTime | undefined
44-
}
45+
type: 'page' | 'data' | 'file'
4546

46-
export interface ExtraDataAndFileCommonConfig {
47-
type: 'data' | 'file'
47+
intervalTime: IntervalTime | undefined
4848
}
4949

5050
interface ExtraPageConfig extends ExtraCommonConfig {
5151
browser: Browser
5252
onCrawlItemComplete:
53-
| ((crawlPageSingleRes: CrawlPageSingleResult) => void)
53+
| ((crawlPageSingleResult: CrawlPageSingleResult) => void)
5454
| undefined
5555
}
5656

57-
interface ExtraDataConfig<T>
58-
extends ExtraCommonConfig,
59-
ExtraDataAndFileCommonConfig {
57+
interface ExtraDataConfig<T> extends ExtraCommonConfig {
6058
onCrawlItemComplete:
61-
| ((crawlDataSingleRes: CrawlDataSingleResult<T>) => void)
59+
| ((crawlDataSingleResult: CrawlDataSingleResult<T>) => void)
6260
| undefined
6361
}
6462

65-
interface ExtraFileConfig
66-
extends ExtraCommonConfig,
67-
ExtraDataAndFileCommonConfig {
63+
interface ExtraFileConfig extends ExtraCommonConfig {
6864
saveFileErrorArr: { message: string; valueOf: () => number }[]
6965
saveFilePendingQueue: Promise<any>[]
7066
onCrawlItemComplete:
71-
| ((crawlFileSingleRes: CrawlFileSingleResult) => void)
67+
| ((crawlFileSingleResult: CrawlFileSingleResult) => void)
7268
| undefined
7369
onBeforeSaveItemFile:
7470
| ((info: {
@@ -135,7 +131,7 @@ interface CrawlPageConfig {
135131
selectFingerprintIndexs: number[]
136132

137133
onCrawlItemComplete:
138-
| ((crawlPageSingleRes: CrawlPageSingleResult) => void)
134+
| ((crawlPageSingleResult: CrawlPageSingleResult) => void)
139135
| undefined
140136
}
141137

@@ -146,7 +142,7 @@ interface CrawlDataConfig {
146142
selectFingerprintIndexs: number[]
147143

148144
onCrawlItemComplete:
149-
| ((crawlDataSingleRes: CrawlDataSingleResult<any>) => void)
145+
| ((crawlDataSingleResult: CrawlDataSingleResult<any>) => void)
150146
| undefined
151147
}
152148

@@ -165,7 +161,7 @@ interface CrawlFileConfig {
165161
}) => Promise<Buffer>)
166162
| undefined
167163
onCrawlItemComplete:
168-
| ((crawlDataSingleRes: CrawlDataSingleResult<any>) => void)
164+
| ((crawlDataSingleResult: CrawlDataSingleResult<any>) => void)
169165
| undefined
170166
}
171167

@@ -983,13 +979,14 @@ export function createCrawlPage(xCrawlConfig: LoaderXCrawlConfig) {
983979
createCrawlPageConfig(xCrawlConfig, config)
984980

985981
const extraConfig: ExtraPageConfig = {
982+
type: 'page',
983+
986984
browser: browser!,
987985
intervalTime,
988986
onCrawlItemComplete
989987
}
990988

991989
const crawlResultArr = (await controller(
992-
'page',
993990
xCrawlConfig.mode,
994991
detailTargets,
995992
extraConfig,
@@ -1046,7 +1043,6 @@ export function createCrawlData(xCrawlConfig: LoaderXCrawlConfig) {
10461043
}
10471044

10481045
const crawlResultArr = (await controller(
1049-
'data',
10501046
xCrawlConfig.mode,
10511047
detailTargets,
10521048
extraConfig,
@@ -1107,7 +1103,6 @@ export function createCrawlFile(xCrawlConfig: LoaderXCrawlConfig) {
11071103
}
11081104

11091105
const crawlResultArr = (await controller(
1110-
'file',
11111106
xCrawlConfig.mode,
11121107
detailTargets,
11131108
extraConfig,
@@ -1132,19 +1127,19 @@ export function createCrawlFile(xCrawlConfig: LoaderXCrawlConfig) {
11321127
errorIds.push(item.id)
11331128
}
11341129
})
1135-
log('Save statistics for the targets:')
1130+
log(logStatistics('Save files finish:'))
11361131
log(
11371132
logSuccess(
1138-
` Success - target total: ${
1139-
succssIds.length
1140-
}, targets id: [ ${succssIds.join(', ')} ]`
1133+
` Success - total: ${succssIds.length}, targets id: [ ${succssIds.join(
1134+
', '
1135+
)} ]`
11411136
)
11421137
)
11431138
log(
11441139
logError(
1145-
` Error - target total: ${
1146-
errorIds.length
1147-
}, targets id: [ ${errorIds.join(', ')} ]`
1140+
` Error - total: ${errorIds.length}, targets id: [ ${errorIds.join(
1141+
', '
1142+
)} ]`
11481143
)
11491144
)
11501145

@@ -1180,13 +1175,13 @@ export function startPolling(
11801175
const intervalId = setInterval(startCallback, total)
11811176

11821177
function startCallback() {
1183-
console.log(logSuccess(`Start the ${logWarn.bold(++count)} polling`))
1178+
console.log(logStart(`Start polling - count: ${++count}`))
11841179

11851180
callback(count, stopPolling)
11861181
}
11871182

11881183
function stopPolling() {
11891184
clearInterval(intervalId)
1190-
console.log(logSuccess(`Stop the polling`))
1185+
console.log(logWarn(`Stop the polling`))
11911186
}
11921187
}

‎src/batchCrawl.ts

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,13 @@ async function useSleepByBatch(
1414
? intervalTime
1515
: random(intervalTime.max, intervalTime.min)
1616

17-
log(`Target id: ${logNumber(id)} - sleep: ${logNumber(timeout + 'ms')}`)
17+
log(
18+
`Target id: ${logNumber(id)} - Sleep time: ${logNumber(timeout + 'ms')}`
19+
)
1820

1921
await sleep(timeout)
2022
} else {
21-
log(`Target id: ${logNumber(id)} - sleep: ${logNumber('0ms')}`)
23+
log(`Target id: ${logNumber(id)} - Sleep time: ${logNumber('0ms')}`)
2224
}
2325
}
2426

‎src/controller.ts

Lines changed: 17 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ import {
1515
log,
1616
logError,
1717
logNumber,
18+
logStart,
19+
logStatistics,
1820
logSuccess,
1921
logWarn
2022
} from './utils'
@@ -81,12 +83,13 @@ export async function controller<
8183
E extends ExtraCommonConfig,
8284
R
8385
>(
84-
name: 'page' | 'data' | 'file',
8586
mode: 'async' | 'sync',
8687
detailTargets: T[],
8788
extraConfig: E,
8889
singleCrawlHandle: (device: Device<T, R>, extraConfig: E) => Promise<void>
8990
) {
91+
const { type } = extraConfig
92+
9093
// 是否使用优先爬取
9194
const isPriorityCrawl = !detailTargets.every(
9295
(item) => item.priority === detailTargets[0].priority
@@ -136,9 +139,9 @@ export async function controller<
136139
)
137140

138141
log(
139-
`${logSuccess(`Start crawling`)} - name: ${logWarn(name)}, mode: ${logWarn(
140-
mode
141-
)}, target total: ${logNumber(devices.length)} `
142+
logStart(
143+
`Start crawling - type: ${type}, mode: ${mode}, total: ${devices.length}`
144+
)
142145
)
143146

144147
// 选择爬取模式
@@ -206,7 +209,9 @@ export async function controller<
206209

207210
log(
208211
logWarn(
209-
`Start retrying: ${++i} - Targets id: [ ${retriedIds.join(' - ')} ]`
212+
`Start retrying - count: ${++i}, targets id: [ ${retriedIds.join(
213+
', '
214+
)} ]`
210215
)
211216
)
212217
}
@@ -223,19 +228,19 @@ export async function controller<
223228
}
224229
})
225230

226-
log('Crawl statistics of the targets:')
231+
log(logStatistics(`Crawl ${type}s finish:`))
227232
log(
228233
logSuccess(
229-
` Success - target total: ${
230-
succssIds.length
231-
}, targets id: [ ${succssIds.join(', ')} ]`
234+
` Success - total: ${succssIds.length}, targets id: [ ${succssIds.join(
235+
', '
236+
)} ]`
232237
)
233238
)
234239
log(
235240
logError(
236-
` Error - target total: ${
237-
errorIds.length
238-
}, targets id: [ ${errorIds.join(', ')} ]`
241+
` Error - total: ${errorIds.length}, targets id: [ ${errorIds.join(
242+
', '
243+
)} ]`
239244
)
240245
)
241246

‎src/types/api.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@ export interface CrawlPageAdvancedConfig extends CrawlCommonConfig {
130130
cookies?: PageCookies
131131
viewport?: Viewport
132132

133-
onCrawlItemComplete?: (crawlPageSingleRes: CrawlPageSingleResult) => void
133+
onCrawlItemComplete?: (crawlPageSingleResult: CrawlPageSingleResult) => void
134134
}
135135

136136
export interface CrawlDataAdvancedConfig<T> extends CrawlCommonConfig {
@@ -140,7 +140,7 @@ export interface CrawlDataAdvancedConfig<T> extends CrawlCommonConfig {
140140

141141
headers?: AnyObject
142142

143-
onCrawlItemComplete?: (crawlDataSingleRes: CrawlDataSingleResult<T>) => void
143+
onCrawlItemComplete?: (crawlDataSingleResult: CrawlDataSingleResult<T>) => void
144144
}
145145

146146
export interface CrawlFileAdvancedConfig extends CrawlCommonConfig {
@@ -152,7 +152,7 @@ export interface CrawlFileAdvancedConfig extends CrawlCommonConfig {
152152
storeDir?: string
153153
extension?: string
154154

155-
onCrawlItemComplete?: (crawlFileSingleRes: CrawlFileSingleResult) => void
155+
onCrawlItemComplete?: (crawlFileSingleResult: CrawlFileSingleResult) => void
156156
onBeforeSaveItemFile?: (info: {
157157
id: number
158158
fileName: string

‎src/utils.ts

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,10 +31,12 @@ export function mkdirDirSync(dir: string) {
3131
}
3232

3333
export const log = console.log
34-
export const logNumber = chalk.hex('#a57fff')
34+
export const logStart = chalk.blueBright
35+
export const logStatistics = chalk.whiteBright
3536
export const logSuccess = chalk.green
3637
export const logError = chalk.red
3738
export const logWarn = chalk.yellow
39+
export const logNumber = chalk.hex('#a57fff')
3840

3941
export function isUndefined(value: any): value is undefined {
4042
return typeof value === 'undefined'

0 commit comments

Comments
 (0)