Skip to content

Commit a7cc1a9

Browse files
coder-hxlcoder-hxl
committed
chore: test url to local
1 parent 1804653 commit a7cc1a9

File tree

11 files changed

+61
-45
lines changed

11 files changed

+61
-45
lines changed

‎test/environment/api/crawlData.test.ts

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,10 @@ async function testCrawlData() {
2020
const testXCrawl = xCrawl()
2121

2222
const res = await testXCrawl.crawlData({
23-
targets: ['http://localhost:8888', { url: 'http://localhost:8888' }]
23+
targets: [
24+
'http://localhost:8888/data',
25+
{ url: 'http://localhost:8888/data' }
26+
]
2427
})
2528

2629
return res.reduce((prev, item) => prev && item.isSuccess, true)

‎test/environment/api/crawlPage.test.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,8 @@ async function testCrawlPage() {
2121

2222
const res = await testXCrawl.crawlPage({
2323
targets: [
24-
'https://github.com/coder-hxl/x-crawl',
25-
{ url: 'https://github.com/coder-hxl/x-crawl' }
24+
'http://localhost:8888/html',
25+
{ url: 'http://localhost:8888/html' }
2626
]
2727
})
2828

‎test/environment/arguments/fingerprint.test.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,10 @@ async function fingerprint() {
2121

2222
const res = await testXCrawl.crawlPage({
2323
targets: [
24-
'http://localhost:8888',
25-
{ url: 'http://localhost:8888', fingerprint: null },
24+
'http://localhost:8888/html',
25+
{ url: 'http://localhost:8888/html', fingerprint: null },
2626
{
27-
url: 'http://localhost:8888',
27+
url: 'http://localhost:8888/html',
2828
fingerprint: {
2929
maxWidth: 1024,
3030
maxHeight: 800,

‎test/environment/arguments/mode.test.ts

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,8 @@ async function async() {
2020
const testXCrawl = xCrawl()
2121

2222
const res = await testXCrawl.crawlData([
23-
'http://localhost:8888',
24-
'http://localhost:8888'
23+
'http://localhost:8888/data',
24+
'http://localhost:8888/data'
2525
])
2626

2727
return res.reduce((prev, item) => prev && item.isSuccess, true)
@@ -31,8 +31,8 @@ async function sync() {
3131
const testXCrawl = xCrawl({ mode: 'sync' })
3232

3333
const res = await testXCrawl.crawlData([
34-
'http://localhost:8888',
35-
'http://localhost:8888'
34+
'http://localhost:8888/data',
35+
'http://localhost:8888/data'
3636
])
3737

3838
return res.reduce((prev, item) => prev && item.isSuccess, true)

‎test/environment/arguments/proxy.test.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ async function proxy() {
2020
const testXCrawl = xCrawl()
2121

2222
const res = await testXCrawl.crawlPage({
23-
targets: ['https://', 'http://localhost:8888'],
23+
targets: ['https://', 'http://localhost:8888/html'],
2424
maxRetry: 3,
2525
proxy: {
2626
urls: ['http://localhost:129032', 'http://localhost:14892'],

‎test/environment/written/crawlData.test.ts

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ jest.setTimeout(60000)
2121
async function writtenString() {
2222
const testXCrawl = xCrawl()
2323

24-
const res = await testXCrawl.crawlData('http://localhost:8888')
24+
const res = await testXCrawl.crawlData('http://localhost:8888/data')
2525

2626
return res.isSuccess
2727
}
@@ -31,7 +31,7 @@ async function writtenCrawlDataDetailConfig() {
3131
const testXCrawl = xCrawl()
3232

3333
const res = await testXCrawl.crawlData({
34-
url: 'http://localhost:8888'
34+
url: 'http://localhost:8888/data'
3535
})
3636

3737
return res.isSuccess
@@ -42,8 +42,8 @@ async function writtenStringAndCrawlDataDetailConfigArr() {
4242
const testXCrawl = xCrawl()
4343

4444
const res = await testXCrawl.crawlData([
45-
'http://localhost:8888',
46-
{ url: 'http://localhost:8888' }
45+
'http://localhost:8888/data',
46+
{ url: 'http://localhost:8888/data' }
4747
])
4848

4949
return res.reduce((prev, item) => prev && item.isSuccess, true)
@@ -54,7 +54,10 @@ async function writtenCrawlDataAdvancedConfig() {
5454
const testXCrawl = xCrawl()
5555

5656
const res = await testXCrawl.crawlData({
57-
targets: ['http://localhost:8888', { url: 'http://localhost:8888' }]
57+
targets: [
58+
'http://localhost:8888/data',
59+
{ url: 'http://localhost:8888/data' }
60+
]
5861
})
5962

6063
return res.reduce((prev, item) => prev && item.isSuccess, true)
@@ -71,7 +74,7 @@ async function loaderBaseConfig() {
7174
maxRetry: 0
7275
})
7376

74-
const res = await testXCrawl.crawlData(['/', '/'])
77+
const res = await testXCrawl.crawlData(['/data', '/data'])
7578

7679
return res.reduce((prev, item) => prev && item.isSuccess, true)
7780
}
@@ -83,7 +86,7 @@ async function loaderAdvancedConfig() {
8386
})
8487

8588
const res = await testXCrawl.crawlData({
86-
targets: ['/', '/'],
89+
targets: ['/data', '/data'],
8790
proxy: { urls: ['http://localhost:14892'] },
8891
timeout: 10000,
8992
intervalTime: { max: 1000 },

‎test/environment/written/crawlPage.test.ts

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ jest.setTimeout(60000)
2121
async function writtenString() {
2222
const testXCrawl = xCrawl()
2323

24-
const res = await testXCrawl.crawlPage('https://gitee.com/coderhxl')
24+
const res = await testXCrawl.crawlPage('http://localhost:8888/html')
2525

2626
await res.data.browser.close()
2727

@@ -33,7 +33,7 @@ async function writtenCrawlPageDetailConfig() {
3333
const testXCrawl = xCrawl({ proxy: { urls: ['http://localhost:14892'] } })
3434

3535
const res = await testXCrawl.crawlPage({
36-
url: 'https://github.com/coder-hxl/x-crawl'
36+
url: 'http://localhost:8888/html'
3737
})
3838

3939
await res.data.browser.close()
@@ -46,8 +46,8 @@ async function writtenStringAndCrawlPageDetailConfigArr() {
4646
const testXCrawl = xCrawl({ proxy: { urls: ['http://localhost:14892'] } })
4747

4848
const res = await testXCrawl.crawlPage([
49-
'https://github.com/coder-hxl/x-crawl',
50-
{ url: 'https://github.com/coder-hxl/x-crawl' }
49+
'http://localhost:8888/html',
50+
{ url: 'http://localhost:8888/html' }
5151
])
5252

5353
await res[0].data.browser.close()
@@ -61,8 +61,8 @@ async function writtenCrawlPageAdvancedConfig() {
6161

6262
const res = await testXCrawl.crawlPage({
6363
targets: [
64-
'https://github.com/coder-hxl/x-crawl',
65-
{ url: 'https://github.com/coder-hxl/x-crawl' }
64+
'http://localhost:8888/html',
65+
{ url: 'http://localhost:8888/html' }
6666
]
6767
})
6868

@@ -75,14 +75,14 @@ async function writtenCrawlPageAdvancedConfig() {
7575
// 2.1.Loader Base Config
7676
async function loaderBaseConfig() {
7777
const testXCrawl = xCrawl({
78-
baseUrl: 'https://github.com',
78+
baseUrl: 'http://localhost:8888',
7979
proxy: { urls: ['http://localhost:14892'] },
8080
timeout: 10000,
8181
intervalTime: { max: 1000 },
8282
maxRetry: 0
8383
})
8484

85-
const res = await testXCrawl.crawlPage(['/coder-hxl', '/coder-hxl/x-crawl'])
85+
const res = await testXCrawl.crawlPage(['/html', '/html'])
8686

8787
await res[0].data.browser.close()
8888

@@ -91,10 +91,10 @@ async function loaderBaseConfig() {
9191

9292
// 2.2.Loader Advanced Config
9393
async function loaderAdvancedConfig() {
94-
const testXCrawl = xCrawl({ baseUrl: 'https://github.com' })
94+
const testXCrawl = xCrawl({ baseUrl: 'http://localhost:8888' })
9595

9696
const res = await testXCrawl.crawlPage({
97-
targets: ['/coder-hxl', '/coder-hxl/x-crawl'],
97+
targets: ['/html', '/html'],
9898
proxy: { urls: ['http://localhost:14892'] },
9999
timeout: 10000,
100100
intervalTime: { max: 1000 },

‎test/server/index.js

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎test/server/index.ts

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,30 @@
11
import http from 'node:http'
22

3+
const html = `
4+
<body>
5+
<h1>Hi</h1>
6+
<script>document.body.appendChild(document.createElement("hr"))</script>
7+
</body>
8+
`
9+
310
http
411
.createServer((req, res) => {
5-
console.log(req.headers)
12+
const { url, method } = req
13+
console.log(method, url)
14+
15+
let contentType = 'text/plain'
16+
let content: any = 'Please select /html or /data'
17+
18+
if (url === '/html') {
19+
contentType = 'text/html; charset=utf-8'
20+
content = html
21+
} else if (url === '/data') {
22+
contentType = 'application/json'
23+
content = { code: 200, message: 'Hi' }
24+
}
625

7-
res.setHeader('Content-Type', 'text/plain')
8-
res.end('success')
26+
res.setHeader('Content-Type', contentType)
27+
res.end(typeof content === 'string' ? content : JSON.stringify(content))
928
})
1029
.listen(8888, () => {
1130
console.log(`服务器在 8888 端口启动成功~`)

0 commit comments

Comments
 (0)