Merge pull request #146 from bluecadet/fix/json-media

fix json media url, and update get-urls

Author: claytercek

.changeset/shy-pears-sleep.md

@@ -0,0 +1,5 @@
+---
+"@bluecadet/launchpad-content": patch
+---
+
+Fix json content source media URL scraping

packages/content/lib/content-sources/json-source.js

@@ -54,7 +54,6 @@ class JsonSource extends ContentSource {
 	 */
 	async _downloadJsons() {
 		const result = new ContentResult();
-		
 		for (const [path, url] of Object.entries(this.config.files)) {
 			this.logger.debug(`Downloading json ${chalk.blue(url)}`);
 			const response = await got(url);
@@ -74,7 +73,7 @@ class JsonSource extends ContentSource {
 		for (const dataFile of result.dataFiles) {
 			this.logger.debug(`Scraping for media files in ${chalk.blue(dataFile.localPath)}...`);
 			const mediaUrls = JsonUtils.getUrls(dataFile.content, undefined, this.config.mediaPattern);
-			this.logger.debug(`Found ${chalk.blue(mediaUrls.size)} media files in ${chalk.blue(dataFile.localPath)}`);
+			this.logger.debug(`Found ${chalk.blue(mediaUrls.size.toString())} media files in ${chalk.blue(dataFile.localPath)}`);
 			result.addMediaDownloads([...mediaUrls].map(url => new MediaDownload({ url })));
 		}
 		return result;

packages/content/lib/utils/json-utils.js

@@ -22,6 +22,7 @@ class JsonUtils {
 		}
 
 		const possibleUrls = getUrls(JSON.stringify(json), config);
+
 		for (const url of possibleUrls) {
 			if (include && (include instanceof RegExp) && !url.match(include)) {
 				continue; // url doesn't match include
@@ -35,15 +36,17 @@ class JsonUtils {
 	}
 
 	/**
-	 * @param {any} [userSettings]
+	 * @param {import('get-urls').Options} [userSettings]
+	 * @returns {import('get-urls').Options}
 	 */
 	static getUrlOptions(userSettings) {
 		return {
 			...{
 				stripAuthentication: false,
 				stripWWW: false,
 				removeTrailingSlash: false,
-				sortQueryParameters: false
+				sortQueryParameters: false,
+				requireSchemeOrWww: true
 			},
 			...userSettings
 		};

packages/content/package.json

@@ -45,7 +45,7 @@
     "contentful": "^9.0.0",
     "filenamify": "^5.1.1",
     "fs-extra": "^10.0.0",
-    "get-urls": "^10.0.0",
+    "get-urls": "^12.1.0",
     "got": "^12.3.0",
     "jsonpath": "^1.1.1",
     "markdown-it": "^12.2.0",
@@ -60,6 +60,7 @@
   "devDependencies": {
     "@types/cli-progress": "^3.11.0",
     "@types/jsonpath": "^0.2.0",
+    "@types/markdown-it": "^12.2.0",
     "@types/progress-stream": "^2.0.2",
     "@types/qs": "^6.9.7",
     "@types/rimraf": "^4.0.5",