Refactor Request implementation and remove Playwright and RawHttp fetchers

- Consolidate Request class with native HTTPS request handling
- Remove separate Playwright and RawHttp fetcher implementations
- Simplify request logic by integrating core functionality directly into Request class
- Remove unnecessary FetcherType and related type definitions
- Update parsers to use simplified Request constructor
- Remove playwright dependency from package.json

Author: soul

bun.lockb

@@ -23,7 +23,6 @@
     "cheerio": "^1.0.0",
     "commander": "^13.0.0",
     "crypto-js": "^4.2.0",
-    "https-proxy-agent": "^7.0.6",
-    "playwright": "^1.50.0"
+    "https-proxy-agent": "^7.0.6"
   }
 }

package.json

@@ -12,7 +12,7 @@ export class Crawler {
       this.parsers.push(
         new NtdmParser(options),
         new DdysParser(options),
-        new TelegraphParser(options)
+        new TelegraphParser(options),
       );
     }
 

src/core/crawler/index.ts

@@ -3,20 +3,17 @@ import { mkdir, stat, unlink } from 'fs/promises';
 import { join, dirname, extname } from 'path';
 import { Request, type RequestOptions } from '../request';
 import { runConcurrent } from '../../utils/concurrent';
-import type { Parser, ParseResult, FetcherType, MediaItem } from '../../types';
+import type { Parser, ParseResult, MediaItem } from '../../types';
 import { DownloadFormatter } from '../../utils/format';
 import { logger } from '../../utils/logger';
 
 export abstract class BaseParser implements Parser {
     protected request: Request;
-    protected downloadRequest: Request; // 专门用于下载的 request
     abstract name: string;
-    abstract fetcherType: FetcherType;
     protected formatter: DownloadFormatter;
 
     constructor(options: RequestOptions = {}) {
         this.request = new Request(options);
-        this.downloadRequest = new Request(options);
         this.formatter = new DownloadFormatter();
     }
 
@@ -44,7 +41,7 @@ export abstract class BaseParser implements Parser {
             // 检查是否存在已下载的文件
             try {
                 fileStats = await stat(destPath);
-                const remoteHeaders = await this.downloadRequest.fetchHeaders(item.url, headers);
+                const remoteHeaders = await this.request.fetchHeaders(item.url, headers);
                 const remoteSize = parseInt(remoteHeaders['content-length'] || '0', 10);
 
                 if (fileStats.size === remoteSize && remoteSize > 0) {
@@ -72,10 +69,11 @@ export abstract class BaseParser implements Parser {
             const writeMode = headers['Range'] ? 'a' : 'w';
             writeStream = createWriteStream(destPath, { flags: writeMode });
 
-            await this.downloadRequest.fetchBuffer(
+            await this.request.fetchBuffer(
                 item.url,
                 headers,
                 (downloaded, total) => {
+
                     const now = Date.now();
                     if (now - lastLogged >= 1000) {
                         const dl = downloaded + (fileStats?.size || 0);
@@ -216,9 +214,6 @@ export abstract class BaseParser implements Parser {
     async close(): Promise<void> {
         await Promise.all([
             this.request.close(),
-            this.downloadRequest.close()
         ]);
     }
-
-
 }

src/core/parsers/base.ts

@@ -1,5 +1,5 @@
 import * as cheerio from 'cheerio';
-import type { FetcherType, MediaItem, ParseResult } from "../../../types";
+import type { MediaItem, ParseResult } from "../../../types";
 import { DownloadFormatter } from '../../../utils/format';
 import { Request, type RequestOptions } from '../../request';
 import { BaseParser } from "../base";
@@ -8,13 +8,11 @@ import { BaseParser } from "../base";
 export class DdysParser extends BaseParser {
     protected request: Request;
     name = "ddys";
-    fetcherType: FetcherType = 'raw-http';
     protected formatter: DownloadFormatter;
 
     constructor(options: RequestOptions = {}) {
         super(options);
-        this.request = new Request(options, this.fetcherType);
-      this.downloadRequest = new Request(options, this.fetcherType);
+        this.request = new Request(options);
         this.formatter = new DownloadFormatter();
     }
 

src/core/parsers/ddys/index.ts

@@ -1,3 +1,3 @@
 export { NtdmParser } from './ntdm';
 export { DdysParser } from './ddys';
-export { TelegraphParser } from './telegraph'; 
+export { TelegraphParser } from './telegraph';

src/core/parsers/index.ts

@@ -1,18 +1,16 @@
 import * as cheerio from 'cheerio';
-import type { FetcherType, ParseResult } from '../../../types';
+import type { ParseResult } from '../../../types';
 import { Request, type RequestOptions } from '../../request';
 import { BaseParser } from '../base';
 import { getVideoInfo } from './contexts/play';
 
-
 export class NtdmParser extends BaseParser {
     protected request: Request;
     name = 'ntdm';
-    fetcherType: FetcherType = 'raw-http';
-  
+
     constructor(options: RequestOptions = {}) {
       super();
-      this.request = new Request(options, this.fetcherType);
+      this.request = new Request(options);
     }
 
     async fetchHtml(url: string): Promise<string> {