From 52825e11f9340023081cce80d6ddafa1aa8bb244 Mon Sep 17 00:00:00 2001 From: Steve Worley Date: Thu, 13 Jun 2024 15:52:03 +1000 Subject: [PATCH 1/2] Feat: User-agent support for crawl. --- src/commands/crawl.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/commands/crawl.js b/src/commands/crawl.js index 0752755..cd9d405 100644 --- a/src/commands/crawl.js +++ b/src/commands/crawl.js @@ -98,6 +98,11 @@ command.builder = { type: 'boolean', default: false, }, + 'user-agent': { + describe: 'The user-agent to send with the request', + type: 'string', + default: 'Quant (+http://api.quantcdn.io)' + } }; /** @@ -151,6 +156,9 @@ command.handler = async function(argv) { crawl.maxConcurrency = argv.concurrency; crawl.respectRobotsTxt = argv.robots; crawl.acceptCookies = argv.cookies; + crawl.customHeaders = { + 'User-Agent': argv['user-agent'] + } const quant = client(config); From cfd2d652946d9b6a98fe0c9e334b85a667f39a97 Mon Sep 17 00:00:00 2001 From: Steve Worley Date: Thu, 13 Jun 2024 15:55:17 +1000 Subject: [PATCH 2/2] Update code style. --- src/commands/crawl.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/commands/crawl.js b/src/commands/crawl.js index cd9d405..9dfabc4 100644 --- a/src/commands/crawl.js +++ b/src/commands/crawl.js @@ -101,8 +101,8 @@ command.builder = { 'user-agent': { describe: 'The user-agent to send with the request', type: 'string', - default: 'Quant (+http://api.quantcdn.io)' - } + default: 'Quant (+http://api.quantcdn.io)', + }, }; /** @@ -157,8 +157,8 @@ command.handler = async function(argv) { crawl.respectRobotsTxt = argv.robots; crawl.acceptCookies = argv.cookies; crawl.customHeaders = { - 'User-Agent': argv['user-agent'] - } + 'User-Agent': argv['user-agent'], + }; const quant = client(config);