1
1
import process from 'node:process'
2
2
import path from 'node:path'
3
3
import { expect , test , jest } from '@jest/globals'
4
+ import chalk from 'chalk'
4
5
5
6
import IXCrawl from '../../src'
6
7
@@ -16,126 +17,100 @@ if (currentModel === 'dev') {
16
17
17
18
jest . setTimeout ( 20000 )
18
19
19
- // Protocol
20
+ // API
21
+ async function crawlPageAPI ( ) {
22
+ const myXCrawl = xCrawl ( { proxy : 'http://localhost:14892' } )
20
23
21
- function httpProtocol ( ) {
22
- return new Promise ( ( resolve ) => {
23
- const httpXCrawl = xCrawl ( )
24
+ const res = await myXCrawl . crawlPage ( 'https://docs.github.com/zh' )
25
+ const { browser } = res
26
+ browser . close ( )
24
27
25
- httpXCrawl
26
- . crawlData ( {
27
- requestConfig : 'http://localhost:9001/api/home/goodprice'
28
- } )
29
- . then ( ( ) => resolve ( true ) )
30
- } )
28
+ return true
31
29
}
32
30
33
- function httpsProtocol ( ) {
34
- return new Promise ( ( resolve ) => {
35
- const httpsXCrawl = xCrawl ( {
36
- timeout : 10000 ,
37
- proxy : 'http://localhost:14892'
38
- } )
39
-
40
- httpsXCrawl . crawlPage ( 'https://docs.github.com/zh' ) . then ( async ( res ) => {
41
- const { browser } = res
42
- await browser . close ( )
43
-
44
- resolve ( true )
45
- } )
31
+ async function crawlDataAPI ( ) {
32
+ const myXCrawl = xCrawl ( )
33
+
34
+ const res = await myXCrawl . crawlData ( {
35
+ requestConfig : {
36
+ url : 'http://localhost:9001/api/area/阳江市' ,
37
+ method : 'POST' ,
38
+ data : {
39
+ type : 'goodPrice' ,
40
+ offset : 0 ,
41
+ size : 20
42
+ }
43
+ }
46
44
} )
47
- }
48
45
49
- test ( 'http protocol' , async ( ) => {
50
- await expect ( httpProtocol ( ) ) . resolves . toBe ( true )
51
- } )
52
-
53
- test ( 'https protocol' , async ( ) => {
54
- await expect ( httpsProtocol ( ) ) . resolves . toBe ( true )
55
- } )
46
+ if ( res [ 0 ] . statusCode === 200 ) {
47
+ return true
48
+ } else {
49
+ return false
50
+ }
51
+ }
56
52
57
- // API
58
- function crawlPageAPI ( ) {
59
- return new Promise ( ( resolve ) => {
60
- const myXCrawl = xCrawl ( { proxy : 'http://localhost:14892' } )
53
+ async function crawlFileAPI ( ) {
54
+ const myXCrawl = xCrawl ( {
55
+ timeout : 10000 ,
56
+ intervalTime : { max : 2000 , min : 1000 } ,
57
+ proxy : 'http://localhost:14892'
58
+ } )
61
59
62
- myXCrawl . crawlPage ( 'https://docs.github.com/zh' ) . then ( async ( res ) => {
63
- const { browser } = res
64
- await browser . close ( )
60
+ const requestConfig : string [ ] = [
61
+ 'https://raw.githubusercontent.com/coder-hxl/airbnb-upload/master/area/4401.jpg' ,
62
+ 'https://raw.githubusercontent.com/coder-hxl/airbnb-upload/master/area/4403.jpg'
63
+ ]
65
64
66
- resolve ( true )
67
- } )
65
+ await myXCrawl . crawlFile ( {
66
+ requestConfig,
67
+ fileConfig : {
68
+ storeDir : path . resolve ( __dirname , './upload' )
69
+ }
68
70
} )
69
- }
70
71
71
- function crawlDataAPI ( ) {
72
- return new Promise ( ( resolve ) => {
73
- const myXCrawl = xCrawl ( )
74
-
75
- myXCrawl
76
- . crawlData ( {
77
- requestConfig : {
78
- url : 'http://localhost:9001/api/area/阳江市' ,
79
- method : 'POST' ,
80
- data : {
81
- type : 'goodPrice' ,
82
- offset : 0 ,
83
- size : 20
84
- }
85
- }
86
- } )
87
- . then ( ( ) => resolve ( true ) )
88
- } )
72
+ return true
89
73
}
90
74
91
- function crawlFileAPI ( ) {
75
+ function startPollingAPI ( ) {
76
+ const myXCrawl = xCrawl ( )
77
+
92
78
return new Promise ( ( resolve ) => {
93
- const myXCrawl = xCrawl ( {
94
- timeout : 10000 ,
95
- intervalTime : {
96
- max : 2000 ,
97
- min : 1000
79
+ myXCrawl . startPolling ( { m : 0.001 } , ( count , stopPolling ) => {
80
+ if ( count === 2 ) {
81
+ stopPolling ( )
82
+ resolve ( true )
98
83
}
99
84
} )
100
-
101
- myXCrawl
102
- . crawlData ( {
103
- requestConfig : {
104
- url : 'http://localhost:9001/api/area/阳江市' ,
105
- method : 'POST' ,
106
- data : {
107
- type : 'goodPrice' ,
108
- offset : 0 ,
109
- size : 20
110
- }
111
- }
112
- } )
113
- . then ( ( res ) => {
114
- const roomList = res [ 0 ] . data . data . list
115
- const requestConfig : string [ ] = roomList . map (
116
- ( item : any ) => item . coverUrl
117
- )
118
-
119
- myXCrawl
120
- . crawlFile ( {
121
- requestConfig,
122
- fileConfig : {
123
- storeDir : path . resolve ( __dirname , './upload' )
124
- }
125
- } )
126
- . then ( ( ) => resolve ( true ) )
127
- } )
128
85
} )
129
86
}
130
87
131
88
test ( 'crawlPage API' , async ( ) => {
89
+ console . log (
90
+ chalk . bgGreen ( '================ Start crawlPage API test ================' )
91
+ )
132
92
await expect ( crawlPageAPI ( ) ) . resolves . toBe ( true )
133
93
} )
134
94
135
95
test ( 'crawlData API' , async ( ) => {
96
+ console . log (
97
+ chalk . bgGreen ( '================ Start crawlData API test ================' )
98
+ )
136
99
await expect ( crawlDataAPI ( ) ) . resolves . toBe ( true )
137
100
} )
138
101
139
102
test ( 'crawlFile API' , async ( ) => {
103
+ console . log (
104
+ chalk . bgGreen ( '================ Start crawlFile API test ================' )
105
+ )
140
106
await expect ( crawlFileAPI ( ) ) . resolves . toBe ( true )
141
107
} )
108
+
109
+ test ( 'startPolling API' , async ( ) => {
110
+ console . log (
111
+ chalk . bgGreen (
112
+ '================ Start startPolling API test ================'
113
+ )
114
+ )
115
+ await expect ( startPollingAPI ( ) ) . resolves . toBe ( true )
116
+ } )
0 commit comments