Merge pull request #146 from bluecadet/fix/json-media

fix json media url, and update get-urls
bluecadet · Jun 18, 2024 · 049642c · 049642c
2 parents fc10c76 + c203bda
commit 049642c
Show file tree

Hide file tree

Showing 4 changed files with 13 additions and 5 deletions.
diff --git a/.changeset/shy-pears-sleep.md b/.changeset/shy-pears-sleep.md
@@ -0,0 +1,5 @@
+---
+"@bluecadet/launchpad-content": patch
+---
+
+Fix json content source media URL scraping
diff --git a/packages/content/lib/content-sources/json-source.js b/packages/content/lib/content-sources/json-source.js
@@ -54,7 +54,6 @@ class JsonSource extends ContentSource {
 	 */
 	async _downloadJsons() {
 		const result = new ContentResult();
-
 		for (const [path, url] of Object.entries(this.config.files)) {
 			this.logger.debug(`Downloading json ${chalk.blue(url)}`);
 			const response = await got(url);
@@ -74,7 +73,7 @@ class JsonSource extends ContentSource {
 		for (const dataFile of result.dataFiles) {
 			this.logger.debug(`Scraping for media files in ${chalk.blue(dataFile.localPath)}...`);
 			const mediaUrls = JsonUtils.getUrls(dataFile.content, undefined, this.config.mediaPattern);
-			this.logger.debug(`Found ${chalk.blue(mediaUrls.size)} media files in ${chalk.blue(dataFile.localPath)}`);
+			this.logger.debug(`Found ${chalk.blue(mediaUrls.size.toString())} media files in ${chalk.blue(dataFile.localPath)}`);
 			result.addMediaDownloads([...mediaUrls].map(url => new MediaDownload({ url })));
 		}
 		return result;

diff --git a/packages/content/lib/utils/json-utils.js b/packages/content/lib/utils/json-utils.js
@@ -22,6 +22,7 @@ class JsonUtils {
 		}
 
 		const possibleUrls = getUrls(JSON.stringify(json), config);
+
 		for (const url of possibleUrls) {
 			if (include && (include instanceof RegExp) && !url.match(include)) {
 				continue; // url doesn't match include
@@ -35,15 +36,17 @@ class JsonUtils {
 	}
 
 	/**
-	 * @param {any} [userSettings]
+	 * @param {import('get-urls').Options} [userSettings]
+	 * @returns {import('get-urls').Options}
 	 */
 	static getUrlOptions(userSettings) {
 		return {
 			...{
 				stripAuthentication: false,
 				stripWWW: false,
 				removeTrailingSlash: false,
-				sortQueryParameters: false
+				sortQueryParameters: false,
+				requireSchemeOrWww: true
 			},
 			...userSettings
 		};

diff --git a/packages/content/package.json b/packages/content/package.json
@@ -45,7 +45,7 @@
     "contentful": "^9.0.0",
     "filenamify": "^5.1.1",
     "fs-extra": "^10.0.0",
-    "get-urls": "^10.0.0",
+    "get-urls": "^12.1.0",
     "got": "^12.3.0",
     "jsonpath": "^1.1.1",
     "markdown-it": "^12.2.0",
@@ -60,6 +60,7 @@
   "devDependencies": {
     "@types/cli-progress": "^3.11.0",
     "@types/jsonpath": "^0.2.0",
+    "@types/markdown-it": "^12.2.0",
     "@types/progress-stream": "^2.0.2",
     "@types/qs": "^6.9.7",
     "@types/rimraf": "^4.0.5",