Support downloading seed file from URL

src/crawler.ts

@@ -62,7 +62,7 @@ import {
 } from "puppeteer-core";
 import { Recorder } from "./util/recorder.js";
 import { SitemapReader } from "./util/sitemapper.js";
-import { ScopedSeed } from "./util/seeds.js";
+import { ScopedSeed, parseSeeds } from "./util/seeds.js";
 import {
   WARCWriter,
   createWARCInfo,
@@ -134,7 +134,7 @@ export class Crawler {
 
   maxPageTime: number;
 
-  seeds: ScopedSeed[];
+  seeds: ScopedSeed[] = [];
   numOriginalSeeds = 0;
 
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -255,9 +255,6 @@ export class Crawler {
     this.saveStateFiles = [];
     this.lastSaveTime = 0;
 
-    this.seeds = this.params.scopedSeeds as ScopedSeed[];
-    this.numOriginalSeeds = this.seeds.length;
-
     // sum of page load + behavior timeouts + 2 x pageop timeouts (for cloudflare, link extraction) + extra page delay
     // if exceeded, will interrupt and move on to next page (likely behaviors or some other operation is stuck)
     this.maxPageTime =
@@ -514,6 +511,9 @@ export class Crawler {
 
     this.proxyServer = await initProxy(this.params, RUN_DETACHED);
 
+    this.seeds = await parseSeeds(this.params);
+    this.numOriginalSeeds = this.seeds.length;
+
     logger.info("Seeds", this.seeds);
 
     logger.info("Link Selectors", this.params.selectLinks);

src/replaycrawler.ts

@@ -96,8 +96,6 @@ export class ReplayCrawler extends Crawler {
     // skip text from first two frames, as they are RWP boilerplate
     this.skipTextDocs = SKIP_FRAMES;
 
-    this.params.scopedSeeds = [];
-
     this.params.screenshot = ["view"];
     this.params.text = ["to-warc"];
 

src/util/argParser.ts

@@ -20,7 +20,6 @@ import {
   BxFunctionBindings,
   DEFAULT_CRAWL_ID_TEMPLATE,
 } from "./constants.js";
-import { ScopedSeed } from "./seeds.js";
 import { interpolateFilename } from "./storage.js";
 import { screenshotTypes } from "./screenshots.js";
 import {
@@ -37,8 +36,6 @@ export type CrawlerArgs = ReturnType<typeof parseArgs> & {
   logExcludeContext: LogContext[];
   text: string[];
 
-  scopedSeeds: ScopedSeed[];
-
   customBehaviors: string[];
 
   selectLinks: ExtractSelector[];
@@ -770,22 +767,6 @@ class ArgParser {
       }
     }
 
-    if (argv.seedFile) {
-      const urlSeedFile = fs.readFileSync(argv.seedFile, "utf8");
-      const urlSeedFileList = urlSeedFile.split("\n");
-
-      if (typeof argv.seeds === "string") {
-        argv.seeds = [argv.seeds];
-      }
-
-      for (const seed of urlSeedFileList) {
-        if (seed) {
-          // eslint-disable-next-line @typescript-eslint/no-explicit-any
-          (argv.seeds as any).push(seed);
-        }
-      }
-    }
-
     let selectLinks: ExtractSelector[];
 
     if (argv.selectLinks) {
@@ -817,50 +798,10 @@ class ArgParser {
       //logger.debug(`Set netIdleWait to ${argv.netIdleWait} seconds`);
     }
 
-    const scopedSeeds: ScopedSeed[] = [];
-
-    if (!isQA) {
-      const scopeOpts = {
-        scopeType: argv.scopeType,
-        sitemap: argv.sitemap,
-        include: argv.include,
-        exclude: argv.exclude,
-        depth: argv.depth,
-        extraHops: argv.extraHops,
-      };
-
-      for (const seed of argv.seeds) {
-        const newSeed = typeof seed === "string" ? { url: seed } : seed;
-
-        try {
-          scopedSeeds.push(new ScopedSeed({ ...scopeOpts, ...newSeed }));
-          // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        } catch (e: any) {
-          logger.error("Failed to create seed", {
-            error: e.toString(),
-            ...scopeOpts,
-            ...newSeed,
-          });
-          if (argv.failOnFailedSeed) {
-            logger.fatal(
-              "Invalid seed specified, aborting crawl",
-              { url: newSeed.url },
-              "general",
-              1,
-            );
-          }
-        }
-      }
-
-      if (!scopedSeeds.length) {
-        logger.fatal("No valid seeds specified, aborting crawl");
-      }
-    } else if (!argv.qaSource) {
+    if (isQA && !argv.qaSource) {
       logger.fatal("--qaSource required for QA mode");
     }
 
-    argv.scopedSeeds = scopedSeeds;
-
     // Resolve statsFilename
     if (argv.statsFilename) {
       argv.statsFilename = path.resolve(argv.cwd, argv.statsFilename);

src/util/file_reader.ts

@@ -24,6 +24,35 @@ export type FileSource = {
 
 export type FileSources = FileSource[];
 
+async function getTempFile(
+  filename: string,
+  dirPrefix: string,
+): Promise<string> {
+  const tmpDir = `/tmp/${dirPrefix}-${crypto.randomBytes(4).toString("hex")}`;
+  await fsp.mkdir(tmpDir, { recursive: true });
+  return path.join(tmpDir, filename);
+}
+
+async function writeUrlContentsToFile(url: string, filepath: string) {
+  const res = await fetch(url, { dispatcher: getProxyDispatcher() });
+  const fileContents = await res.text();
+  await fsp.writeFile(filepath, fileContents);
+}
+
+export async function collectOnlineSeedFile(url: string): Promise<string> {
+  const filename = path.basename(new URL(url).pathname);
+  const filepath = await getTempFile(filename, "seeds-");
+
+  try {
+    await writeUrlContentsToFile(url, filepath);
+    logger.info("Seed file downloaded", { url, path: filepath });
+  } catch (e) {
+    logger.fatal("Error downloading seed file from URL", { url, error: e });
+  }
+
+  return filepath;
+}
+
 export async function collectCustomBehaviors(
   sources: string[],
 ): Promise<FileSources> {
@@ -88,17 +117,10 @@ async function collectGitBehaviors(gitUrl: string): Promise<FileSources> {
 
 async function collectOnlineBehavior(url: string): Promise<FileSources> {
   const filename = path.basename(new URL(url).pathname);
-  const tmpDir = path.join(
-    os.tmpdir(),
-    `behaviors-${crypto.randomBytes(4).toString("hex")}`,
-  );
-  await fsp.mkdir(tmpDir, { recursive: true });
-  const behaviorFilepath = path.join(tmpDir, filename);
+  const behaviorFilepath = await getTempFile(filename, "behaviors-");
 
   try {
-    const res = await fetch(url, { dispatcher: getProxyDispatcher() });
-    const fileContents = await res.text();
-    await fsp.writeFile(behaviorFilepath, fileContents);
+    await writeUrlContentsToFile(url, behaviorFilepath);
     logger.info(
       "Custom behavior file downloaded",
       { url, path: behaviorFilepath },

src/util/seeds.ts

@@ -1,5 +1,8 @@
-import { logger } from "./logger.js";
+import fs from "fs";
+
 import { MAX_DEPTH } from "./constants.js";
+import { collectOnlineSeedFile } from "./file_reader.js";
+import { logger } from "./logger.js";
 
 type ScopeType =
   | "prefix"
@@ -300,6 +303,71 @@ export class ScopedSeed {
   }
 }
 
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export async function parseSeeds(params: any): Promise<ScopedSeed[]> {
+  let seeds = params.seeds;
+  const scopedSeeds: ScopedSeed[] = [];
+
+  if (params.seedFile) {
+    let seedFilePath = params.seedFile;
+    if (params.seedFile.startsWith("http")) {
+      seedFilePath = await collectOnlineSeedFile(params.seedFile as string);
+    }
+
+    const urlSeedFile = fs.readFileSync(seedFilePath, "utf8");
+    const urlSeedFileList = urlSeedFile.split("\n");
+
+    if (typeof seeds === "string") {
+      seeds = [seeds];
+    }
+
+    for (const seed of urlSeedFileList) {
+      if (seed) {
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        (seeds as any).push(seed);
+      }
+    }
+  }
+
+  const scopeOpts = {
+    scopeType: params.scopeType,
+    sitemap: params.sitemap,
+    include: params.include,
+    exclude: params.exclude,
+    depth: params.depth,
+    extraHops: params.extraHops,
+  };
+
+  for (const seed of seeds) {
+    const newSeed = typeof seed === "string" ? { url: seed } : seed;
+
+    try {
+      scopedSeeds.push(new ScopedSeed({ ...scopeOpts, ...newSeed }));
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    } catch (e: any) {
+      logger.error("Failed to create seed", {
+        error: e.toString(),
+        ...scopeOpts,
+        ...newSeed,
+      });
+      if (params.failOnFailedSeed) {
+        logger.fatal(
+          "Invalid seed specified, aborting crawl",
+          { url: newSeed.url },
+          "general",
+          1,
+        );
+      }
+    }
+  }
+
+  if (!params.qaSource && !scopedSeeds.length) {
+    logger.fatal("No valid seeds specified, aborting crawl");
+  }
+
+  return scopedSeeds;
+}
+
 export function rxEscape(string: string) {
   return string.replace(/[-/\\^$*+?.()|[\]{}]/g, "\\$&");
 }

src/util/worker.ts

@@ -351,7 +351,7 @@ export class PageWorker {
     let loggedWaiting = false;
 
     while (await this.crawler.isCrawlRunning()) {
-      await crawlState.processMessage(this.crawler.params.scopedSeeds);
+      await crawlState.processMessage(this.crawler.seeds);
 
       const data = await crawlState.nextFromQueue();