Feat: Added Rotating Proxy Feature

coder-hxl · coder-hxl · commit 667d758aa08a · 2023-04-23T21:34:00.000+08:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,9 +1,10 @@
-# [v7.0.0](https://github.com/coder-hxl/x-crawl/compare/v6.0.1...v7.0.0) (2023-04-23)
+# [v7.0.0](https://github.com/coder-hxl/x-crawl/compare/v6.0.1...v7.0.0) (2023-04-24)
 
 ### 🚨 重大改变
 
 - 进阶写法的配置指纹改用数组写法，里面存放 DetailTargetFingerprintCommon 类型的对象，方便定制。内部会将里面的对象随机分配给目标。
 - crawlPage 的进阶写法和详细目标写法的指纹配置的最大宽高改为可选项。
+- 创建爬虫实例、进阶写法以及详细目标写法的 proxy 更改为对象写法, 拥有 urls、switchByHttpStatus 以及 switchByErrorCount 这三个属性，urls 可以设置多个代理 URL ，内部默认先采用第一个，switchByHttpStatus 设置遇到哪些不符合的响应状态码需要切换代理，switchByErrorCount 设置像超时等错误时到达多少次需要切换代理。该代理轮换功能需要配合错误重试才能使用。
 
 ### 🚀 特征
 
diff --git a/docs/cn.md b/docs/cn.md
@@ -14,6 +14,7 @@ x-crawl 是一个灵活的 Node.js 多功能爬虫库。用法灵活，并且内
 - **👀 设备指纹** - 零配置或自定义配置，避免指纹识别从不同位置识别并跟踪我们。
 - **⏱️ 间隔爬取** - 无间隔、固定间隔以及随机间隔，产生或避免高并发爬取。
 - **🔄 失败重试** - 避免因短暂的问题而造成爬取失败，无限制重试次数。
+- **➡️ 轮换代理** - 配合失败重试，根据自定义错误次数以及 HTTP 状态码自动轮换代理。
 - **🚀 优先队列** - 根据单个爬取目标的优先级可以优先于其他目标提前爬取。
 - **☁️ 爬取 SPA** - 爬取 SPA（单页应用程序）生成预渲染内容（即“SSR”（服务器端渲染））。
 - **⚒️ 控制页面** - 可以表单提交、键盘输入、事件操作、生成页面的屏幕截图等。
diff --git a/src/api.ts b/src/api.ts
@@ -84,20 +84,30 @@ interface PageSingleCrawlResult {
 
 // Create config
 // Loader
+export type ProxyDetails = { url: string; state: boolean }[]
+
+type LoaderCommonConfig = {
+  proxyUrl?: string
+  proxyDetails: ProxyDetails
+}
+
 type LoaderHasConfig = {
   timeout: number
   maxRetry: number
   priority: number
 }
 
-export type LoaderCrawlPageDetail = CrawlPageDetailTargetConfig &
-  LoaderHasConfig
+export type LoaderCrawlPageDetail = LoaderCommonConfig &
+  LoaderHasConfig &
+  CrawlPageDetailTargetConfig
 
-export type LoaderCrawlDataDetail = CrawlDataDetailTargetConfig &
-  LoaderHasConfig
+export type LoaderCrawlDataDetail = LoaderCommonConfig &
+  LoaderHasConfig &
+  CrawlDataDetailTargetConfig
 
-export type LoaderCrawlFileDetail = CrawlFileDetailTargetConfig &
-  LoaderHasConfig
+export type LoaderCrawlFileDetail = LoaderCommonConfig &
+  LoaderHasConfig &
+  CrawlFileDetailTargetConfig
 
 //  AdvancedDetailTargets
 interface CrawlPageAdvancedDetailTargetsConfig extends CrawlPageAdvancedConfig {
@@ -382,7 +392,16 @@ function loaderCommonConfigToCrawlConfig(
         }
       }
 
-      // 1.3.porxy
+      // 1.3.maxRetry
+      if (isUndefined(maxRetry)) {
+        if (!isUndefined(advancedDetailTargetsConfig.maxRetry)) {
+          detail.maxRetry = advancedDetailTargetsConfig.maxRetry
+        } else {
+          detail.maxRetry = xCrawlConfig.maxRetry
+        }
+      }
+
+      // 1.4.proxy
       if (isUndefined(proxy)) {
         if (!isUndefined(advancedDetailTargetsConfig.proxy)) {
           detail.proxy = advancedDetailTargetsConfig.proxy
@@ -391,26 +410,24 @@ function loaderCommonConfigToCrawlConfig(
         }
       }
 
-      // 1.4.maxRetry
-      if (isUndefined(maxRetry)) {
-        if (!isUndefined(advancedDetailTargetsConfig.maxRetry)) {
-          detail.maxRetry = advancedDetailTargetsConfig.maxRetry
-        } else {
-          detail.maxRetry = xCrawlConfig.maxRetry
-        }
+      // 1.5.proxyUrl & proxyDetail
+      if (!isUndefined(detail.proxy?.urls)) {
+        const urls = detail.proxy!.urls
+        detail.proxyUrl = urls[0]
+        detail.proxyDetails = urls.map((url) => ({ url, state: true }))
       }
 
-      // 1.5.priority
+      // 1.6.priority
       if (isUndefined(priority)) {
         detail.priority = 0
       }
 
-      // 1.6.header
+      // 1.7.header
       if (isUndefined(headers) && advancedDetailTargetsConfig.headers) {
         detail.headers = { ...advancedDetailTargetsConfig.headers }
       }
 
-      // 1.7.fingerprint(公共部分)
+      // 1.8.fingerprint(公共部分)
       if (fingerprint) {
         // detaileTarget
 
@@ -675,9 +692,9 @@ async function pageSingleCrawlHandle(
 
   let response: HTTPResponse | null = null
   try {
-    if (detailTarget.proxy) {
+    if (detailTarget.proxyUrl) {
       await browser.createIncognitoBrowserContext({
-        proxyServer: detailTarget.proxy
+        proxyServer: detailTarget.proxyUrl
       })
     } else {
       await browser.createIncognitoBrowserContext({
@@ -973,8 +990,6 @@ export function createCrawlData(xCrawlConfig: LoaderXCrawlConfig) {
       }
 
       const crawlDataSingleRes: AnyObject = detaileInfo
-      delete crawlDataSingleRes.detailTarget
-      delete crawlDataSingleRes.detailTargetRes
 
       if (onCrawlItemComplete) {
         onCrawlItemComplete(crawlDataSingleRes as CrawlDataSingleRes<T>)
diff --git a/src/batchCrawlHandle.ts b/src/batchCrawlHandle.ts
@@ -1,7 +1,7 @@
 import { isNumber, isUndefined, log, logNumber, random, sleep } from './utils'
 
 import type { ExtraCommonConfig } from './api'
-import type { DetailInfo, CrawlDetail } from './controller'
+import { DetailInfo, CrawlDetail, getCrawlStatus } from './controller'
 
 async function useSleepByBatch(
   isHaventervalTime: boolean,
@@ -45,8 +45,8 @@ export async function asyncBatchCrawl<
   const isNumberIntervalTime = isNumber(intervalTime)
 
   const crawlPendingQueue: Promise<any>[] = []
-  for (const detaileInfo of detailInfos) {
-    const { id } = detaileInfo
+  for (const detailInfo of detailInfos) {
+    const { id } = detailInfo
 
     await useSleepByBatch(
       isHaventervalTime,
@@ -55,24 +55,34 @@ export async function asyncBatchCrawl<
       id
     )
 
-    const crawlSinglePending = singleCrawlHandle(detaileInfo, extraConfig)
+    const crawlSinglePending = singleCrawlHandle(detailInfo, extraConfig)
       .catch((error) => {
-        detaileInfo.crawlErrorQueue.push(error)
+        detailInfo.crawlErrorQueue.push(error)
         return false
       })
       .then((detailTargetRes) => {
+        const notAllowRetry = detailInfo.retryCount === detailInfo.maxRetry
+
         if (typeof detailTargetRes === 'boolean') {
-          if (detaileInfo.retryCount === detaileInfo.maxRetry) {
-            singleResultHandle(detaileInfo, extraConfig)
+          if (notAllowRetry) {
+            singleResultHandle(detailInfo, extraConfig)
           }
 
           return
         }
 
-        detaileInfo.isSuccess = true
-        detaileInfo.detailTargetRes = detailTargetRes
+        detailInfo.isSuccess = true
+        detailInfo.detailTargetRes = detailTargetRes
 
-        singleResultHandle(detaileInfo, extraConfig)
+        // 根据 状态码/是否无法重试 决定处理结果
+        const { detailTarget } = detailInfo
+
+        const status = getCrawlStatus(detailTargetRes)
+        const switchByHttpStatus = detailTarget.proxy?.switchByHttpStatus ?? []
+        if ((status && !switchByHttpStatus.includes(status)) || notAllowRetry) {
+          singleResultHandle(detailInfo, extraConfig)
+          delete detailInfo._notHandle
+        }
       })
 
     crawlPendingQueue.push(crawlSinglePending)
@@ -120,8 +130,20 @@ export async function syncBatchCrawl<
       detailInfo.crawlErrorQueue.push(error)
     }
 
-    if (detailInfo.isSuccess || detailInfo.retryCount === detailInfo.maxRetry) {
+    // 根据 是否成功和状态码/是否无法重试 决定处理结果
+    const { detailTarget, detailTargetRes } = detailInfo
+
+    const status = getCrawlStatus(detailTargetRes)
+    const switchByHttpStatus = detailTarget.proxy?.switchByHttpStatus ?? []
+    const notAllowRetry = detailInfo.retryCount === detailInfo.maxRetry
+    if (
+      (detailInfo.isSuccess &&
+        status &&
+        !switchByHttpStatus.includes(status)) ||
+      notAllowRetry
+    ) {
       singleResultHandle(detailInfo, extraConfig)
+      delete detailInfo._notHandle
     }
   }
 }
diff --git a/src/controller.ts b/src/controller.ts
@@ -5,22 +5,36 @@ import {
   ExtraCommonConfig,
   LoaderCrawlDataDetail,
   LoaderCrawlFileDetail,
-  LoaderCrawlPageDetail
+  LoaderCrawlPageDetail,
+  ProxyDetails
 } from './api'
 
-import { log, logError, logNumber, logSuccess, logWarn } from './utils'
+import {
+  isObject,
+  isUndefined,
+  log,
+  logError,
+  logNumber,
+  logSuccess,
+  logWarn
+} from './utils'
+import { HTTPResponse } from 'puppeteer'
+import { Request } from './request'
 
 export type CrawlDetail =
   | LoaderCrawlPageDetail
   | LoaderCrawlDataDetail
   | LoaderCrawlFileDetail
 
 export interface DetailInfo<T extends CrawlDetail, R> {
+  _notHandle: any
+
   id: number
   isSuccess: boolean
   maxRetry: number
   retryCount: number
   crawlErrorQueue: Error[]
+  proxyDetailes: ProxyDetails
   data: any | null
 
   detailTarget: T
@@ -32,6 +46,25 @@ type TargetSingleRes = Omit<
   'detailTarget' | 'detailTargetRes'
 >
 
+export function getCrawlStatus(detailTargetRes: any) {
+  let status: number | null = null
+
+  if (
+    isObject(detailTargetRes) &&
+    Object.hasOwn(detailTargetRes, 'response') &&
+    (detailTargetRes as any).response
+  ) {
+    // crawlPage
+    const response: HTTPResponse = (detailTargetRes as any).response
+    status = response.status()
+  } else if (isObject(detailTargetRes)) {
+    // crawlData / crawlFie
+    status = (detailTargetRes as any as Request).statusCode ?? null
+  }
+
+  return status
+}
+
 export async function controller<
   T extends CrawlDetail,
   E extends ExtraCommonConfig,
@@ -63,11 +96,14 @@ export async function controller<
   // 通过映射生成新的配置数组
   const detailInfos: DetailInfo<T, R>[] = detailTargetConfigs.map(
     (detailTarget, index) => ({
+      _notHandle: true,
+
       id: index + 1,
       isSuccess: false,
       maxRetry: detailTarget.maxRetry,
       retryCount: 0,
       crawlErrorQueue: [],
+      proxyDetailes: detailTarget.proxyDetails,
       data: null,
 
       detailTarget,
@@ -94,12 +130,65 @@ export async function controller<
       singleResultHandle
     )
 
-    crawlQueue = crawlQueue.filter(
-      (config) =>
-        config.maxRetry &&
-        !config.isSuccess &&
-        config.retryCount < config.maxRetry
-    )
+    crawlQueue = crawlQueue.filter((detailInfo) => {
+      const {
+        isSuccess,
+        maxRetry,
+        retryCount,
+        proxyDetailes,
+        crawlErrorQueue,
+        detailTarget,
+        detailTargetRes
+      } = detailInfo
+
+      let isRetry = false
+      const haveRetryChance = maxRetry && retryCount < maxRetry
+
+      // 没有被处理/没成功/状态码不符合
+      if (Object.hasOwn(detailInfo, '_notHandle') && haveRetryChance) {
+        // 1.不成功
+        if (!isSuccess) {
+          isRetry = true
+        }
+
+        // 2.代理多, 轮换代理
+        if (proxyDetailes.length >= 2) {
+          // 获取状态码
+          const status = getCrawlStatus(detailTargetRes)
+
+          // 错误次数 / 检测状态码
+          const switchByErrorCount = detailTarget.proxy?.switchByErrorCount ?? 0
+          const switchByHttpStatus =
+            detailTarget.proxy?.switchByHttpStatus ?? []
+          if (
+            (status && switchByHttpStatus.includes(status)) ||
+            switchByErrorCount >= crawlErrorQueue.length
+          ) {
+            isRetry = true
+            proxyDetailes.find(
+              (detail) => detail.url === detailTarget.proxyUrl
+            )!.state = false
+
+            // 寻找新代理 URL
+            const newProxyUrl = proxyDetailes.find(
+              (detaile) => detaile.state
+            )?.url
+
+            // 无则不切换
+            if (!isUndefined(newProxyUrl)) {
+              detailTarget.proxyUrl = newProxyUrl
+            }
+          }
+        }
+      }
+
+      // 重置需要重试的 isSuccess
+      if (isRetry) {
+        detailInfo.isSuccess = false
+      }
+
+      return isRetry
+    })
 
     if (crawlQueue.length) {
       const retriedIds = crawlQueue.map((item) => {
diff --git a/src/request.ts b/src/request.ts
@@ -63,8 +63,8 @@ function handleRequestConfig(
   const isHttp = protocol === 'http:'
 
   const config: RequestOptions & MapTypeEmptyObject<URL> = {
-    agent: rawConfig.proxy
-      ? HttpsProxyAgent(rawConfig.proxy)
+    agent: rawConfig.proxyUrl
+      ? HttpsProxyAgent(rawConfig.proxyUrl)
       : isHttp
       ? new http.Agent()
       : new https.Agent(),
diff --git a/src/types/api.ts b/src/types/api.ts
diff --git a/test/start/index.js b/test/start/index.js
diff --git a/test/start/index.ts b/test/start/index.ts
