minor adjustments

Author: coder-hxl

README.md

@@ -50,7 +50,6 @@ Create a crawler instance via new XCrawl. The request queue is maintained by the
 
 ```ts
 class XCrawl {
-  private readonly baseConfig
   constructor(baseConfig?: IXCrawlBaseConifg)
   fetchHTML(config: IFetchHTMLConfig): Promise<IFetchHTML>
   fetchData<T = any>(config: IFetchDataConfig): Promise<IFetchCommon<T>>

document/cn.md

@@ -50,7 +50,6 @@ docsXCrawl.fetchHTML('/zh/get-started').then((res) => {
 
 ```ts
 class XCrawl {
-  private readonly baseConfig
   constructor(baseConfig?: IXCrawlBaseConifg)
   fetchHTML(config: IFetchHTMLConfig): Promise<IFetchHTML>
   fetchData<T = any>(config: IFetchDataConfig): Promise<IFetchCommon<T>>

package.json

@@ -1,7 +1,7 @@
 {
   "private": true,
   "name": "x-crawl",
-  "version": "0.1.5",
+  "version": "0.1.6",
   "author": "CoderHxl",
   "description": "XCrawl is a Nodejs multifunctional crawler library. Crawl HTML, JSON, file resources, etc. through simple configuration.",
   "license": "MIT",

publish/package.json

@@ -1,6 +1,6 @@
 {
   "name": "x-crawl",
-  "version": "0.1.5",
+  "version": "0.1.6",
   "author": "CoderHxl",
   "description": "XCrawl is a Nodejs multifunctional crawler library. Crawl HTML, JSON, file resources, etc. through simple configuration.",
   "license": "MIT",

src/index.ts

@@ -27,45 +27,39 @@ import {
   IIntervalTime
 } from './types'
 
-function mergeConfig<T extends IFetchBaseConifg>(
-  baseConfig: IXCrawlBaseConifg,
-  config: T
-): IFetchBaseConifg & T {
-  const {
-    baseUrl,
-    timeout: baseTimeout,
-    intervalTime: baseIntervalTime
-  } = baseConfig
-  const { requestConifg, intervalTime } = config
-
-  const requestConifgArr = isArray(requestConifg)
-    ? requestConifg
-    : [requestConifg]
-
-  for (const requestItem of requestConifgArr) {
-    const { url, timeout } = requestItem
-
-    if (!isUndefined(baseUrl)) {
-      requestItem.url = baseUrl + url
-    }
+export default class XCrawl {
+  private readonly baseConfig: IXCrawlBaseConifg
 
-    if (isUndefined(timeout) && !isUndefined(baseTimeout)) {
-      requestItem.timeout = baseTimeout
-    }
+  constructor(baseConfig: IXCrawlBaseConifg = {}) {
+    this.baseConfig = baseConfig
   }
 
-  if (isUndefined(intervalTime) && !isUndefined(baseIntervalTime)) {
-    config.intervalTime = baseIntervalTime
-  }
+  private mergeConfig<T extends IFetchBaseConifg>(rawConfig: T): T {
+    const baseConfig = this.baseConfig
+    const newConfig: T = structuredClone(rawConfig)
 
-  return config
-}
+    // 1.处理 requestConifg
+    const requestConifgArr = isArray(newConfig.requestConifg)
+      ? newConfig.requestConifg
+      : [newConfig.requestConifg]
+    for (const requestItem of requestConifgArr) {
+      const { url, timeout } = requestItem
 
-export default class XCrawl {
-  private readonly baseConfig: IXCrawlBaseConifg
+      if (!isUndefined(baseConfig.baseUrl)) {
+        requestItem.url = baseConfig.baseUrl + url
+      }
 
-  constructor(baseConfig: IXCrawlBaseConifg = {}) {
-    this.baseConfig = baseConfig
+      if (isUndefined(timeout)) {
+        requestItem.timeout = baseConfig.timeout
+      }
+    }
+
+    // 2.处理 intervalTime
+    if (isUndefined(newConfig.intervalTime)) {
+      newConfig.intervalTime = baseConfig.intervalTime
+    }
+
+    return newConfig
   }
 
   private async useBatchRequestByMode(
@@ -87,12 +81,8 @@ export default class XCrawl {
   }
 
   async fetchHTML(config: IFetchHTMLConfig): Promise<IFetchHTML> {
-    const rawRequestConifg: IFetchHTMLConfig = isString(config)
-      ? { url: config }
-      : config
-
-    const { requestConifg } = mergeConfig(this.baseConfig, {
-      requestConifg: rawRequestConifg
+    const { requestConifg } = this.mergeConfig({
+      requestConifg: isString(config) ? { url: config } : config
     })
 
     const requestRes = await request(requestConifg)
@@ -110,7 +100,7 @@ export default class XCrawl {
   }
 
   async fetchData<T = any>(config: IFetchDataConfig): Promise<IFetchCommon<T>> {
-    const { requestConifg, intervalTime } = mergeConfig(this.baseConfig, config)
+    const { requestConifg, intervalTime } = this.mergeConfig(config)
 
     const requestRes = await this.useBatchRequestByMode(
       requestConifg,
@@ -134,10 +124,8 @@ export default class XCrawl {
   }
 
   async fetchFile(config: IFetchFileConfig): Promise<IFetchCommon<IFileInfo>> {
-    const { requestConifg, intervalTime, fileConfig } = mergeConfig(
-      this.baseConfig,
-      config
-    )
+    const { requestConifg, intervalTime, fileConfig } = this.mergeConfig(config)
+
     const requestRes = await this.useBatchRequestByMode(
       requestConifg,
       intervalTime
@@ -170,7 +158,7 @@ export default class XCrawl {
 
     const saveTotal = requestRes.length
     const success = container.length
-    const error = requestRes.length - container.length
+    const error = saveTotal - success
     log(
       `saveTotal: ${logNumber(saveTotal)}, success: ${logSuccess(
         success

src/utils.ts

@@ -7,8 +7,8 @@ export function sleep(timeout: number) {
 export function random(max: number, min = 0) {
   let res = Math.floor(Math.random() * max)
 
-  if (res < min) {
-    res = random(max, min)
+  while (res < min) {
+    res = Math.floor(Math.random() * max)
   }
 
   return res