Update: Print capture log

Author: coder-hxl

src/api.ts

@@ -5,7 +5,15 @@ import puppeteer, { Browser, HTTPResponse, Page, Protocol } from 'puppeteer'
 
 import { ControllerConfig, controller } from './controller'
 import { request } from './request'
-import { isArray, isObject, isUndefined, logSuccess, logWarn } from './utils'
+import {
+  isArray,
+  isObject,
+  isUndefined,
+  log,
+  logError,
+  logSuccess,
+  logWarn
+} from './utils'
 
 import {
   DataRequestConfig,
@@ -30,6 +38,7 @@ import {
   CrawlDataConfigObject
 } from './types/api'
 import { LoaderXCrawlBaseConfig } from './types'
+import { quickSort } from './sort'
 
 async function crawlRequestSingle(
   controllerConfig: ControllerConfig<DataRequestConfig & FileRequestConfig, any>
@@ -539,6 +548,35 @@ export function createCrawlFile(baseConfig: LoaderXCrawlBaseConfig) {
     // 等待保存文件完成
     await Promise.all(saveFileQueue)
 
+    // 打印保存错误
+    quickSort(saveFileErrorArr).forEach((item) => log(logError(item.message)))
+
+    // 统计保存
+    const succssIds: number[] = []
+    const errorIds: number[] = []
+    crawlResArr.forEach((item) => {
+      if (item.data?.data.isSuccess) {
+        succssIds.push(item.id)
+      } else {
+        errorIds.push(item.id)
+      }
+    })
+    log('Save file final result:')
+    log(
+      logSuccess(
+        `  Success - total: ${succssIds.length}, ids: [ ${succssIds.join(
+          ' - '
+        )} ]`
+      )
+    )
+    log(
+      logError(
+        `    Error - total: ${errorIds.length}, ids: [ ${errorIds.join(
+          ' - '
+        )} ]`
+      )
+    )
+
     const crawlRes = isArray(config.requestConfig)
       ? crawlResArr
       : crawlResArr[0]

src/batchCrawlHandle.ts

@@ -22,7 +22,7 @@ async function useSleepByBatch(
 
     await sleep(timeout)
   } else {
-    log(`Crawl ${logNumber(id)} does not need to sleep, send immediately`)
+    log(`Id: ${logNumber(id)} - Crawl does not need to sleep, send immediately`)
   }
 }
 

src/controller.ts

@@ -1,5 +1,6 @@
 import { asyncBatchCrawl, syncBatchCrawl } from './batchCrawlHandle'
 import { IntervalTime } from './types/api'
+import { log, logError, logNumber, logSuccess, logWarn } from './utils'
 
 export interface ControllerConfig<T, V> {
   id: number
@@ -12,6 +13,7 @@ export interface ControllerConfig<T, V> {
 }
 
 export async function controller<T extends { maxRetry?: number }, V, C>(
+  name: 'page' | 'data' | 'file',
   mode: 'async' | 'sync',
   requestConfigs: T[],
   intervalTime: IntervalTime | undefined,
@@ -34,6 +36,12 @@ export async function controller<T extends { maxRetry?: number }, V, C>(
     })
   )
 
+  log(
+    `${logSuccess(`Start crawling`)} - name: ${logWarn(name)}, mode: ${logWarn(
+      mode
+    )}, total: ${logNumber(controllerConfigs.length)} `
+  )
+
   // 选择爬取模式
   const batchCrawl = mode === 'async' ? asyncBatchCrawl : syncBatchCrawl
 
@@ -52,7 +60,37 @@ export async function controller<T extends { maxRetry?: number }, V, C>(
         !config.isSuccess &&
         config.retryCount < config.maxRetry
     )
+
+    if (crawlQueue.length) {
+      const retriedIds = crawlQueue.map((item) => item.id)
+      log(logWarn(`Ids to retry: [ ${retriedIds.join(' - ')} ]`))
+    }
   }
 
+  // 统计结果
+  const succssIds: number[] = []
+  const errorIds: number[] = []
+  controllerConfigs.forEach((item) => {
+    if (item.isSuccess) {
+      succssIds.push(item.id)
+    } else {
+      errorIds.push(item.id)
+    }
+  })
+
+  log('Crawl the final result:')
+  log(
+    logSuccess(
+      `  Success - total: ${succssIds.length}, ids: [ ${succssIds.join(
+        ' - '
+      )} ]`
+    )
+  )
+  log(
+    logError(
+      `    Error - total: ${errorIds.length}, ids: [ ${errorIds.join(' - ')} ]`
+    )
+  )
+
   return controllerConfigs
 }

test/start/index.js

@@ -1,4 +1,5 @@
 import path from 'node:path'
+import { Browser } from 'puppeteer'
 import xCrawl from 'x-crawl'
 
 const testXCrawl = xCrawl({
@@ -7,19 +8,24 @@ const testXCrawl = xCrawl({
 })
 
 testXCrawl
-  .crawlData({
-    url: 'http://localhost:9001/api/area/阳江市',
-    method: 'POST',
-    data: {
-      type: 'goodPrice',
-      offset: 0,
-      size: 20
-    }
+  .crawlPage({
+    requestConfigs: [
+      'https://www.google.com/search?q=1',
+      'https://www.google.com/search?q=2',
+      'https://www.google.com/search?q=2'
+    ],
+    maxRetry: 2
   })
   .then((res) => {
-    if (res.data?.statusCode === 200) {
-      return true
-    } else {
-      return false
-    }
+    let browser: Browser | null = null
+
+    res.forEach((item) => {
+      if (!browser) browser = item.data.browser
+
+      console.log(item.isSuccess, item.retryCount)
+
+      console.log(item.errorQueue.map((item) => item.message))
+    })
+
+    browser!.close()
   })