Feat: Adding a test server

Author: coder-hxl

package.json

@@ -22,6 +22,7 @@
     "build-dts": "tsc && prettier --write ./publish/src",
     "build-strict": "pnpm test-dev && pnpm build && pnpm test-pro",
     "start": "rollup --config script/start.mjs",
+    "start-server": "rollup --config script/server.mjs",
     "test-dev": "jest test/environment/test.ts dev",
     "test-pro": "jest test/environment/test.ts pro",
     "test-crawlPage": "jest test/environment/crawlPage.test.ts dev",

script/server.mjs

@@ -0,0 +1,12 @@
+import tsPlugin from 'rollup-plugin-typescript2'
+import terserPlugin from '@rollup/plugin-terser'
+import runPlugin from '@rollup/plugin-run'
+
+export default {
+  input: 'test/server/index.ts',
+  output: {
+    file: 'test/server/index.js',
+    format: 'cjs'
+  },
+  plugins: [tsPlugin(), terserPlugin(), runPlugin({ stdin: { clear: true } })]
+}

script/start.mjs

@@ -8,5 +8,5 @@ export default {
     file: 'test/start/index.js',
     format: 'cjs'
   },
-  plugins: [tsPlugin(), terserPlugin(), runPlugin({ stdin: { clear: true } })]
+  plugins: [tsPlugin(), runPlugin({ stdin: { clear: true } })]
 }

test/environment/crawlData.test.ts

@@ -21,9 +21,7 @@ jest.setTimeout(60000)
 async function writtenString() {
   const testXCrawl = xCrawl()
 
-  const res = await testXCrawl.crawlData(
-    'http://localhost:9001/api/room/193581217'
-  )
+  const res = await testXCrawl.crawlData('http://localhost:8888')
 
   return res.isSuccess
 }
@@ -33,7 +31,7 @@ async function writtenCrawlDataDetailConfig() {
   const testXCrawl = xCrawl()
 
   const res = await testXCrawl.crawlData({
-    url: 'http://localhost:9001/api/room/193581217'
+    url: 'http://localhost:8888'
   })
 
   return res.isSuccess
@@ -44,8 +42,8 @@ async function writtenStringAndCrawlDataDetailConfigArr() {
   const testXCrawl = xCrawl()
 
   const res = await testXCrawl.crawlData([
-    'http://localhost:9001/api/room/193581217',
-    { url: 'http://localhost:9001/api/room/193581217' }
+    'http://localhost:8888',
+    { url: 'http://localhost:8888' }
   ])
 
   return res.reduce((prev, item) => prev && item.isSuccess, true)
@@ -56,10 +54,7 @@ async function writtenCrawlDataAdvancedConfig() {
   const testXCrawl = xCrawl()
 
   const res = await testXCrawl.crawlData({
-    targets: [
-      'http://localhost:9001/api/room/193581217',
-      { url: 'http://localhost:9001/api/room/193581217' }
-    ]
+    targets: ['http://localhost:8888', { url: 'http://localhost:8888' }]
   })
 
   return res.reduce((prev, item) => prev && item.isSuccess, true)
@@ -69,27 +64,27 @@ async function writtenCrawlDataAdvancedConfig() {
 // 2.1.Loader Base Config
 async function loaderBaseConfig() {
   const testXCrawl = xCrawl({
-    baseUrl: 'http://localhost:9001/api',
-    proxy: 'http://localhost:14892',
+    baseUrl: 'http://localhost:8888',
+    proxy: { urls: ['http://localhost:14892'] },
     timeout: 10000,
     intervalTime: { max: 1000 },
     maxRetry: 0
   })
 
-  const res = await testXCrawl.crawlData(['/room/193581217', '/room/193581217'])
+  const res = await testXCrawl.crawlData(['/', '/'])
 
   return res.reduce((prev, item) => prev && item.isSuccess, true)
 }
 
 // 2.2.Loader Advanced Config
 async function loaderAdvancedConfig() {
   const testXCrawl = xCrawl({
-    baseUrl: 'http://localhost:9001/api'
+    baseUrl: 'http://localhost:8888'
   })
 
   const res = await testXCrawl.crawlData({
-    targets: ['/room/193581217', '/room/193581217'],
-    proxy: 'http://localhost:14892',
+    targets: ['/', '/'],
+    proxy: { urls: ['http://localhost:14892'] },
     timeout: 10000,
     intervalTime: { max: 1000 },
     maxRetry: 0

test/environment/crawlFile.test.ts

@@ -27,7 +27,7 @@ const storeDir = path.resolve(__dirname, './upload')
 /* 1.Written */
 // 1.1.written CrawlFileDetailConfig
 async function writtenCrawlFileDetailConfig() {
-  const testXCrawl = xCrawl({ proxy: 'http://localhost:14892' })
+  const testXCrawl = xCrawl({ proxy: { urls: ['http://localhost:14892'] } })
 
   const res = await testXCrawl.crawlFile({ url: urls[0], storeDir })
 
@@ -36,7 +36,7 @@ async function writtenCrawlFileDetailConfig() {
 
 // 1.2.written CrawlFileDetailConfig[]
 async function writtenCrawlFileDetailConfigArr() {
-  const testXCrawl = xCrawl({ proxy: 'http://localhost:14892' })
+  const testXCrawl = xCrawl({ proxy: { urls: ['http://localhost:14892'] } })
 
   const res = await testXCrawl.crawlFile(urls.map((url) => ({ url, storeDir })))
 
@@ -48,7 +48,7 @@ async function writtenCrawlFileDetailConfigArr() {
 
 // 1.3.written CrawlFileAdvancedConfig
 async function writtenCrawlFileAdvancedConfig() {
-  const testXCrawl = xCrawl({ proxy: 'http://localhost:14892' })
+  const testXCrawl = xCrawl({ proxy: { urls: ['http://localhost:14892'] } })
 
   const res = await testXCrawl.crawlFile({
     targets: urls,
@@ -67,7 +67,7 @@ async function loaderBaseConfig() {
   const testXCrawl = xCrawl({
     baseUrl:
       'https://raw.githubusercontent.com/coder-hxl/airbnb-upload/master/area',
-    proxy: 'http://localhost:14892',
+    proxy: { urls: ['http://localhost:14892'] },
     timeout: 10000,
     intervalTime: { max: 1000 },
     maxRetry: 0
@@ -90,7 +90,7 @@ async function loaderAdvancedConfig() {
 
   const res = await testXCrawl.crawlFile({
     targets: ['/4401.jpg', '/4403.jpg'],
-    proxy: 'http://localhost:14892',
+    proxy: { urls: ['http://localhost:14892'] },
     timeout: 10000,
     storeDir,
     intervalTime: { max: 1000 },
@@ -105,7 +105,7 @@ async function storeConfig() {
   const testXCrawl = xCrawl({
     baseUrl:
       'https://raw.githubusercontent.com/coder-hxl/airbnb-upload/master/area',
-    proxy: 'http://localhost:14892'
+    proxy: { urls: ['http://localhost:14892'] }
   })
 
   const record: string[] = []

test/environment/crawlPage.test.ts

@@ -30,7 +30,7 @@ async function writtenString() {
 
 // 1.2.written CrawlPageDetailConfig
 async function writtenCrawlPageDetailConfig() {
-  const testXCrawl = xCrawl({ proxy: 'http://localohst:14892' })
+  const testXCrawl = xCrawl({ proxy: { urls: ['http://localhost:14892'] } })
 
   const res = await testXCrawl.crawlPage({
     url: 'https://github.com/coder-hxl/x-crawl'
@@ -43,7 +43,7 @@ async function writtenCrawlPageDetailConfig() {
 
 // 1.3.written (string | CrawlPageDetailConfig)[]
 async function writtenStringAndCrawlPageDetailConfigArr() {
-  const testXCrawl = xCrawl({ proxy: 'http://localohst:14892' })
+  const testXCrawl = xCrawl({ proxy: { urls: ['http://localhost:14892'] } })
 
   const res = await testXCrawl.crawlPage([
     'https://github.com/coder-hxl/x-crawl',
@@ -57,7 +57,7 @@ async function writtenStringAndCrawlPageDetailConfigArr() {
 
 // 1.4.written CrawlPageAdvancedConfig
 async function writtenCrawlPageAdvancedConfig() {
-  const testXCrawl = xCrawl({ proxy: 'http://localohst:14892' })
+  const testXCrawl = xCrawl({ proxy: { urls: ['http://localhost:14892'] } })
 
   const res = await testXCrawl.crawlPage({
     targets: [
@@ -76,7 +76,7 @@ async function writtenCrawlPageAdvancedConfig() {
 async function loaderBaseConfig() {
   const testXCrawl = xCrawl({
     baseUrl: 'https://github.com',
-    proxy: 'http://localhost:14892',
+    proxy: { urls: ['http://localhost:14892'] },
     timeout: 10000,
     intervalTime: { max: 1000 },
     maxRetry: 0
@@ -95,7 +95,7 @@ async function loaderAdvancedConfig() {
 
   const res = await testXCrawl.crawlPage({
     targets: ['/coder-hxl', '/coder-hxl/x-crawl'],
-    proxy: 'http://localhost:14892',
+    proxy: { urls: ['http://localhost:14892'] },
     timeout: 10000,
     intervalTime: { max: 1000 },
     maxRetry: 0

test/server/index.js

@@ -0,0 +1 @@
+"use strict";require("node:http").createServer(((e,t)=>{t.setHeader("Content-type","text/plain"),t.end("success")})).listen(8888,(()=>{console.log("服务器在 8888 端口启动成功~")}));

test/server/index.ts

@@ -0,0 +1,10 @@
+import http from 'node:http'
+
+http
+  .createServer((req, res) => {
+    res.setHeader('Content-type', 'text/plain')
+    res.end('success')
+  })
+  .listen(8888, () => {
+    console.log(`服务器在 8888 端口启动成功~`)
+  })