proxy: support setting proxy via --proxyServer, PROXY_SERVER env var or PROXY_HOST + PROXY_PORT env vars

Dockerfile

@@ -6,11 +6,7 @@ FROM ${BROWSER_IMAGE_BASE}
 # needed to add args to main build stage
 ARG BROWSER_VERSION
 
-ENV PROXY_HOST=localhost \
-    PROXY_PORT=8080 \
-    PROXY_CA_URL=http://wsgiprox/download/pem \
-    PROXY_CA_FILE=/tmp/proxy-ca.pem \
-    DISPLAY=:99 \
+ENV DISPLAY=:99 \
     GEOMETRY=1360x1020x16 \
     BROWSER_VERSION=${BROWSER_VERSION} \
     BROWSER_BIN=google-chrome \

package.json

@@ -21,6 +21,7 @@
     "@webrecorder/wabac": "^2.16.12",
     "browsertrix-behaviors": "^0.6.0",
     "crc": "^4.3.2",
+    "fetch-socks": "^1.3.0",
     "get-folder-size": "^4.0.0",
     "husky": "^8.0.3",
     "ioredis": "^5.3.2",
@@ -34,6 +35,7 @@
     "sax": "^1.3.0",
     "sharp": "^0.32.6",
     "tsc": "^2.0.4",
+    "undici": "^6.18.2",
     "uuid": "8.3.2",
     "warcio": "^2.2.1",
     "ws": "^7.4.4",

src/crawler.ts

@@ -56,6 +56,7 @@ import { SitemapReader } from "./util/sitemapper.js";
 import { ScopedSeed } from "./util/seeds.js";
 import { WARCWriter, createWARCInfo, setWARCInfo } from "./util/warcwriter.js";
 import { isHTMLContentType } from "./util/reqresp.js";
+import { initProxy } from "./util/proxy.js";
 
 const behaviors = fs.readFileSync(
   new URL(
@@ -170,6 +171,8 @@ export class Crawler {
   maxHeapUsed = 0;
   maxHeapTotal = 0;
 
+  proxyServer?: string;
+
   driver!: (opts: {
     page: Page;
     data: PageState;
@@ -436,6 +439,8 @@ export class Crawler {
   async bootstrap() {
     const subprocesses: ChildProcess[] = [];
 
+    this.proxyServer = initProxy(this.params.proxyServer);
+
     subprocesses.push(this.launchRedis());
 
     await fsp.mkdir(this.logDir, { recursive: true });
@@ -1289,7 +1294,7 @@ self.__bx_behaviors.selectMainBehavior();
       emulateDevice: this.emulateDevice,
       swOpt: this.params.serviceWorker,
       chromeOptions: {
-        proxy: false,
+        proxy: this.proxyServer,
         userAgent: this.emulateDevice.userAgent,
         extraArgs: this.extraChromeArgs(),
       },

src/create-login-profile.ts

@@ -99,9 +99,10 @@ function cliOpts(): { [key: string]: Options } {
       default: getDefaultWindowSize(),
     },
 
-    proxy: {
-      type: "boolean",
-      default: false,
+    proxyServer: {
+      describe:
+        "if set, will use specified proxy server. Takes precedence over any env var proxy settings",
+      type: "string",
     },
 
     cookieDays: {
@@ -179,7 +180,7 @@ async function main() {
     headless: params.headless,
     signals: false,
     chromeOptions: {
-      proxy: false,
+      proxy: params.proxyServer,
       extraArgs: [
         "--window-position=0,0",
         `--window-size=${params.windowSize}`,

src/util/argParser.ts

@@ -545,6 +545,12 @@ class ArgParser {
         default: "disabled",
       },
 
+      proxyServer: {
+        describe:
+          "if set, will use specified proxy server. Takes precedence over any env var proxy settings",
+        type: "string",
+      },
+
       qaSource: {
         describe: "Required for QA mode. Source (WACZ or multi WACZ) for QA",
         type: "string",

src/util/blockrules.ts

@@ -4,6 +4,8 @@ import { logger, formatErr } from "./logger.js";
 import { HTTPRequest, Page } from "puppeteer-core";
 import { Browser } from "./browser.js";
 
+import { fetch } from "undici";
+
 const RULE_TYPES = ["block", "allowOnly"];
 
 const ALWAYS_ALLOW = ["https://pywb.proxy/", "http://pywb.proxy/"];

src/util/browser.ts

@@ -22,7 +22,7 @@ import { CDPSession, Target, Browser as PptrBrowser } from "puppeteer-core";
 import { Recorder } from "./recorder.js";
 
 type BtrixChromeOpts = {
-  proxy?: boolean;
+  proxy?: string;
   userAgent?: string | null;
   extraArgs?: string[];
 };
@@ -115,7 +115,6 @@ export class Browser {
         ? undefined
         : (target) => this.targetFilter(target),
     };
-
     await this._init(launchOpts, ondisconnect, recording);
   }
 
@@ -217,7 +216,7 @@ export class Browser {
   }
 
   chromeArgs({
-    proxy = true,
+    proxy = "",
     userAgent = null,
     extraArgs = [],
   }: BtrixChromeOpts) {
@@ -236,11 +235,13 @@ export class Browser {
       ...extraArgs,
     ];
 
+    if (proxy) {
+      logger.info("Using proxy", { proxy }, "browser");
+    }
+
     if (proxy) {
       args.push("--ignore-certificate-errors");
-      args.push(
-        `--proxy-server=http://${process.env.PROXY_HOST}:${process.env.PROXY_PORT}`,
-      );
+      args.push(`--proxy-server=${proxy}`);
     }
 
     return args;

src/util/proxy.ts

@@ -0,0 +1,60 @@
+import { Dispatcher, ProxyAgent, setGlobalDispatcher } from "undici";
+
+import { socksDispatcher } from "fetch-socks";
+import type { SocksProxyType } from "socks/typings/common/constants.js";
+
+export function getEnvProxyUrl() {
+  if (process.env.PROXY_SERVER) {
+    return process.env.PROXY_SERVER;
+  }
+
+  // for backwards compatibility with 0.x proxy settings
+  if (process.env.PROXY_HOST && process.env.PROXY_PORT) {
+    return `http://${process.env.PROXY_HOST}:${process.env.PROXY_PORT}`;
+  }
+
+  return "";
+}
+
+export function initProxy(proxy?: string): string {
+  if (!proxy) {
+    proxy = getEnvProxyUrl();
+  }
+  if (proxy) {
+    const dispatcher = createDispatcher(proxy);
+    if (dispatcher) {
+      setGlobalDispatcher(dispatcher);
+      return proxy;
+    }
+  }
+  return "";
+}
+
+export function createDispatcher(proxyUrl: string): Dispatcher | undefined {
+  if (proxyUrl.startsWith("http://") || proxyUrl.startsWith("https://")) {
+    // HTTP PROXY does not support auth, as it's not supported in the browser
+    // so must drop username/password for consistency
+    const url = new URL(proxyUrl);
+    url.username = "";
+    url.password = "";
+    return new ProxyAgent({ uri: url.href });
+  } else if (
+    proxyUrl.startsWith("socks://") ||
+    proxyUrl.startsWith("socks5://") ||
+    proxyUrl.startsWith("socks4://")
+  ) {
+    // support auth as SOCKS5 auth *is* supported in Brave (though not in Chromium)
+    const url = new URL(proxyUrl);
+    const type: SocksProxyType = url.protocol === "socks4:" ? 4 : 5;
+    const params = {
+      type,
+      host: url.hostname,
+      port: parseInt(url.port),
+      userId: url.username || undefined,
+      password: url.password || undefined,
+    };
+    return socksDispatcher(params);
+  } else {
+    return undefined;
+  }
+}

src/util/recorder.ts

@@ -8,6 +8,8 @@ import { logger, formatErr } from "./logger.js";
 import { sleep, timedRun, timestampNow } from "./timing.js";
 import { RequestResponseInfo, isHTMLContentType } from "./reqresp.js";
 
+import { fetch, Response } from "undici";
+
 // @ts-expect-error TODO fill in why error is expected
 import { baseRules as baseDSRules } from "@webrecorder/wabac/src/rewrite/index.js";
 import {

src/util/reqresp.ts

@@ -4,6 +4,7 @@ import { getStatusText } from "@webrecorder/wabac/src/utils.js";
 import { Protocol } from "puppeteer-core";
 import { postToGetUrl } from "warcio";
 import { HTML_TYPES } from "./constants.js";
+import { Response } from "undici";
 
 const CONTENT_LENGTH = "content-length";
 const CONTENT_TYPE = "content-type";

src/util/sitemapper.ts

@@ -9,6 +9,8 @@ import { logger, formatErr } from "./logger.js";
 import { DETECT_SITEMAP } from "./constants.js";
 import { sleep } from "./timing.js";
 
+import { fetch, Response } from "undici";
+
 const SITEMAP_CONCURRENCY = 5;
 
 const TEXT_CONTENT_TYPE = ["text/plain"];
@@ -237,7 +239,8 @@ export class SitemapReader extends EventEmitter {
       resp.headers.get("content-encoding") !== "gzip"
     ) {
       const ds = new DecompressionStream("gzip");
-      stream = body.pipeThrough(ds);
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      stream = body.pipeThrough(ds as any);
     } else {
       stream = body;
     }

tests/pdf-crawl.test.js

@@ -3,7 +3,7 @@ import fs from "fs";
 import path from "path";
 import { WARCParser } from "warcio";
 
-const PDF = "http://bibnum.bnf.fr/WARC/WARC_ISO_28500_version1_latestdraft.pdf";
+const PDF = "https://specs.webrecorder.net/wacz/1.1.1/wacz-2021.pdf";
 
 test("ensure pdf is crawled", async () => {
   child_process.execSync(

tests/proxy.test.js

@@ -0,0 +1,127 @@
+import { execSync, exec } from "child_process";
+
+const sleep = (ms) => new Promise((res) => setTimeout(res, ms));
+
+const PROXY_IMAGE = "tarampampam/3proxy:1.9.1";
+const SOCKS_PORT = "1080";
+const HTTP_PORT = "3128";
+const WRONG_PORT = "33130";
+
+const PDF = "https://specs.webrecorder.net/wacz/1.1.1/wacz-2021.pdf";
+const HTML = "https://webrecorder.net/";
+
+const extraArgs = "--limit 1 --failOnFailedSeed --timeout 10 --logging debug";
+
+let proxyAuthId;
+let proxyNoAuthId;
+
+beforeAll(() => {
+  execSync("docker network create proxy-test-net");
+
+  proxyAuthId = execSync(`docker run -e PROXY_LOGIN=user -e PROXY_PASSWORD=passw0rd -d --rm --network=proxy-test-net --name proxy-with-auth ${PROXY_IMAGE}`, {encoding: "utf-8"});
+
+  proxyNoAuthId = execSync(`docker run -d --rm --network=proxy-test-net --name proxy-no-auth ${PROXY_IMAGE}`, {encoding: "utf-8"});
+});
+
+afterAll(async () => {
+  execSync(`docker kill -s SIGINT ${proxyAuthId}`);
+  execSync(`docker kill -s SIGINT ${proxyNoAuthId}`);
+  await sleep(3000);
+  execSync("docker network rm proxy-test-net");
+});
+
+describe("socks5 + https proxy tests", () => {
+  for (const scheme of ["socks5", "http"]) {
+    const port = scheme === "socks5" ? SOCKS_PORT : HTTP_PORT;
+
+    for (const type of ["HTML page", "PDF"]) {
+
+      const url = type === "PDF" ? PDF : HTML;
+
+      test(`${scheme} proxy, ${type}, no auth`, () => {
+        let status = 0;
+
+        try {
+          execSync(`docker run -e PROXY_SERVER=${scheme}://proxy-no-auth:${port} --rm --network=proxy-test-net webrecorder/browsertrix-crawler crawl --url ${url} ${extraArgs}`, {encoding: "utf-8"});
+        } catch (e) {
+          status = e.status;
+        }
+        expect(status).toBe(0);
+      });
+
+      test(`${scheme} proxy, ${type}, with auth`, () => {
+        let status = 0;
+
+        try {
+          execSync(`docker run -e PROXY_SERVER=${scheme}://user:passw0rd@proxy-with-auth:${port} --rm --network=proxy-test-net webrecorder/browsertrix-crawler crawl --url ${url} ${extraArgs}`, {encoding: "utf-8"});
+        } catch (e) {
+          status = e.status;
+        }
+        // auth supported only for SOCKS5
+        expect(status).toBe(scheme === "socks5" ? 0 : 1);
+      });
+
+      test(`${scheme} proxy, ${type}, wrong auth`, () => {
+        let status = 0;
+
+        try {
+          execSync(`docker run -e PROXY_SERVER=${scheme}://user:passw1rd@proxy-with-auth:${port} --rm --network=proxy-test-net webrecorder/browsertrix-crawler crawl --url ${url} ${extraArgs}`, {encoding: "utf-8"});
+        } catch (e) {
+          status = e.status;
+        }
+        expect(status).toBe(1);
+      });
+
+      test(`${scheme} proxy, ${type}, wrong protocol`, () => {
+        let status = 0;
+
+        try {
+          execSync(`docker run -e PROXY_SERVER=${scheme}://user:passw1rd@proxy-with-auth:${scheme === "socks5" ? HTTP_PORT : SOCKS_PORT} --rm --network=proxy-test-net webrecorder/browsertrix-crawler crawl --url ${url} ${extraArgs}`, {encoding: "utf-8"});
+        } catch (e) {
+          status = e.status;
+        }
+        expect(status).toBe(1);
+      });
+    }
+
+    test(`${scheme} proxy, proxy missing error`, () => {
+      let status = 0;
+
+      try {
+        execSync(`docker run -e PROXY_SERVER=${scheme}://proxy-no-auth:${WRONG_PORT} --rm --network=proxy-test-net webrecorder/browsertrix-crawler crawl --url ${HTML} ${extraArgs}`, {encoding: "utf-8"});
+      } catch (e) {
+        status = e.status;
+      }
+      expect(status).toBe(1);
+    });
+  }
+});
+
+
+test("http proxy, PDF, separate env vars", () => {
+  execSync(`docker run -e PROXY_HOST=proxy-no-auth -e PROXY_PORT=${HTTP_PORT} --rm --network=proxy-test-net webrecorder/browsertrix-crawler crawl --url ${PDF} ${extraArgs}`, {encoding: "utf-8"});
+});
+
+test("http proxy set, but not running, separate env vars", () => {
+  let status = 0;
+
+  try {
+    execSync(`docker run -e PROXY_HOST=proxy-no-auth -e PROXY_PORT=${WRONG_PORT} --rm --network=proxy-test-net webrecorder/browsertrix-crawler crawl --url ${PDF} ${extraArgs}`, {encoding: "utf-8"});
+  } catch (e) {
+    status = e.status;
+  }
+  expect(status).toBe(1);
+});
+
+test("http proxy set, but not running, cli arg", () => {
+  let status = 0;
+
+  try {
+    execSync(`docker run --rm --network=proxy-test-net webrecorder/browsertrix-crawler crawl --proxyServer http://proxy-no-auth:${WRONG_PORT} --url ${PDF} ${extraArgs}`, {encoding: "utf-8"});
+  } catch (e) {
+    status = e.status;
+  }
+  expect(status).toBe(1);
+});
+
+