From 5812428f1878b28003c587f6d246a5db69779b1e Mon Sep 17 00:00:00 2001 From: Ben Shi Date: Wed, 7 Nov 2018 23:33:28 +1100 Subject: [PATCH 1/4] keep original name of remote files along with hash --- .../src/__tests__/utils.js | 15 ++++++---- .../src/create-remote-file-node.js | 5 ++-- .../gatsby-source-filesystem/src/utils.js | 28 ++++++++++++++++++- 3 files changed, 39 insertions(+), 9 deletions(-) diff --git a/packages/gatsby-source-filesystem/src/__tests__/utils.js b/packages/gatsby-source-filesystem/src/__tests__/utils.js index 0501d92bfb567..9a8ac914f991a 100644 --- a/packages/gatsby-source-filesystem/src/__tests__/utils.js +++ b/packages/gatsby-source-filesystem/src/__tests__/utils.js @@ -1,20 +1,23 @@ -const { getRemoteFileExtension } = require(`../utils`) +const { getRemoteFileExtension, getRemoteFileName } = require(`../utils`) describe(`create remote file node`, () => { - it(`can correctly retrieve files extensions`, () => { + it(`can correctly retrieve file name and extensions`, () => { ;[ [ `https://scontent.xx.fbcdn.net/v/t51.2885-15/42078503_294439751160571_1602896118583132160_n.jpg?_nc_cat=101&oh=e30490a47409051c45dc3daacf616bc0&oe=5C5EA8EB`, + `42078503_294439751160571_1602896118583132160_n`, `.jpg`, ], [ `https://facebook.com/hello,_world_asdf12341234.jpeg?test=true&other_thing=also-true`, + `hello,_world_asdf12341234`, `.jpeg`, ], - [`https://test.com/asdf.png`, `.png`], - [`./path/to/relative/file.tiff`, `.tiff`], - [`/absolutely/this/will/work.bmp`, `.bmp`], - ].forEach(([url, ext]) => { + [`https://test.com/asdf.png`, `asdf`, `.png`], + [`./path/to/relative/file.tiff`, `file`, `.tiff`], + [`/absolutely/this/will/work.bmp`, `work`, `.bmp`], + ].forEach(([url, name, ext]) => { + expect(getRemoteFileName(url)).toBe(name) expect(getRemoteFileExtension(url)).toBe(ext) }) }) diff --git a/packages/gatsby-source-filesystem/src/create-remote-file-node.js b/packages/gatsby-source-filesystem/src/create-remote-file-node.js index b7eaa80b41351..24115ed1fc914 100644 --- a/packages/gatsby-source-filesystem/src/create-remote-file-node.js +++ b/packages/gatsby-source-filesystem/src/create-remote-file-node.js @@ -8,7 +8,7 @@ const readChunk = require(`read-chunk`) const fileType = require(`file-type`) const { createFileNode } = require(`./create-file-node`) -const { getRemoteFileExtension } = require(`./utils`) +const { getRemoteFileExtension, getRemoteFileName } = require(`./utils`) const cacheId = url => `create-remote-file-node-${url}` /******************** @@ -198,6 +198,7 @@ async function processRemoteNode({ // Create the temp and permanent file names for the url. const digest = createHash(url) + const name = getRemoteFileName(url) if (!ext) { ext = getRemoteFileExtension(url) } @@ -218,7 +219,7 @@ async function processRemoteNode({ ext = `.${filetype.ext}` } } - const filename = createFilePath(programDir, digest, ext) + const filename = createFilePath(programDir, `${digest}_${name}`, ext) // If the status code is 200, move the piped temp file to the real name. if (response.statusCode === 200) { await fs.move(tmpFilename, filename, { overwrite: true }) diff --git a/packages/gatsby-source-filesystem/src/utils.js b/packages/gatsby-source-filesystem/src/utils.js index a543af4fd96cc..7d2e4032c7be7 100644 --- a/packages/gatsby-source-filesystem/src/utils.js +++ b/packages/gatsby-source-filesystem/src/utils.js @@ -1,6 +1,19 @@ const path = require(`path`) const Url = require(`url`) +/** + * getParsedPath + * -- + * Parses remote url to a path object + * + * + * @param {String} url + * @return {Object} path + */ +function getParsedPath(url) { + return path.parse(Url.parse(url).pathname) +} + /** * getRemoteFileExtension * -- @@ -11,5 +24,18 @@ const Url = require(`url`) * @return {String} extension */ export function getRemoteFileExtension(url) { - return path.parse(Url.parse(url).pathname).ext + return getParsedPath(url).ext +} + +/** + * getRemoteFileName + * -- + * Parses remote url to retrieve remote file name + * + * + * @param {String} url + * @return {String} filename + */ +export function getRemoteFileName(url) { + return getParsedPath(url).name } From caad0479567f1a2885aa53e5bb77b97d8aab82aa Mon Sep 17 00:00:00 2001 From: Ben Shi Date: Fri, 23 Nov 2018 22:40:48 +1100 Subject: [PATCH 2/4] maintain original filename by creating hashed directories --- .../gatsby-source-filesystem/src/create-remote-file-node.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/gatsby-source-filesystem/src/create-remote-file-node.js b/packages/gatsby-source-filesystem/src/create-remote-file-node.js index 24115ed1fc914..b73b4f7af469a 100644 --- a/packages/gatsby-source-filesystem/src/create-remote-file-node.js +++ b/packages/gatsby-source-filesystem/src/create-remote-file-node.js @@ -219,7 +219,9 @@ async function processRemoteNode({ ext = `.${filetype.ext}` } } - const filename = createFilePath(programDir, `${digest}_${name}`, ext) + + const digestFolder = path.join(programDir, digest) + const filename = createFilePath(digestFolder, name, ext) // If the status code is 200, move the piped temp file to the real name. if (response.statusCode === 200) { await fs.move(tmpFilename, filename, { overwrite: true }) From 43009a4f82fb3b22312f52470f4ef5a533b8d396 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Mon, 10 Dec 2018 22:32:05 +0100 Subject: [PATCH 3/4] correct dir location --- .../src/create-remote-file-node.js | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/packages/gatsby-source-filesystem/src/create-remote-file-node.js b/packages/gatsby-source-filesystem/src/create-remote-file-node.js index 11a8641dc2f0c..c41f2253f2658 100644 --- a/packages/gatsby-source-filesystem/src/create-remote-file-node.js +++ b/packages/gatsby-source-filesystem/src/create-remote-file-node.js @@ -75,7 +75,7 @@ const FS_PLUGIN_DIR = `gatsby-source-filesystem` * @return {String} */ const createFilePath = (directory, filename, ext) => - path.join(directory, CACHE_DIR, FS_PLUGIN_DIR, `${filename}${ext}`) + path.join(directory, `${filename}${ext}`) /******************** * Queue Management * @@ -178,8 +178,12 @@ async function processRemoteNode({ ext, }) { // Ensure our cache directory exists. - const programDir = store.getState().program.directory - await fs.ensureDir(path.join(programDir, CACHE_DIR, FS_PLUGIN_DIR)) + const pluginCacheDir = path.join( + store.getState().program.directory, + CACHE_DIR, + FS_PLUGIN_DIR + ) + await fs.ensureDir(pluginCacheDir) // See if there's response headers for this url // from a previous request. @@ -203,7 +207,7 @@ async function processRemoteNode({ ext = getRemoteFileExtension(url) } - const tmpFilename = createFilePath(programDir, `tmp-${digest}`, ext) + const tmpFilename = createFilePath(pluginCacheDir, `tmp-${digest}`, ext) // Fetch the file. try { @@ -220,8 +224,11 @@ async function processRemoteNode({ } } - const digestFolder = path.join(programDir, digest) - const filename = createFilePath(digestFolder, name, ext) + const filename = createFilePath( + path.join(pluginCacheDir, digest), + name, + ext + ) // If the status code is 200, move the piped temp file to the real name. if (response.statusCode === 200) { await fs.move(tmpFilename, filename, { overwrite: true }) From 8fb4d7801410b36f28e27131c59069041efa3779 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Mon, 10 Dec 2018 23:02:14 +0100 Subject: [PATCH 4/4] yarn.lock --- yarn.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarn.lock b/yarn.lock index 1b4d7a1b49f66..14adac823fefc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9122,7 +9122,7 @@ graphql-type-json@^0.2.1: resolved "https://registry.yarnpkg.com/graphql-type-json/-/graphql-type-json-0.2.1.tgz#d2c177e2f1b17d87f81072cd05311c0754baa420" integrity sha1-0sF34vGxfYf4EHLNBTEcB1S6pCA= -graphql@0.13.2, graphql@^0.13.0, graphql@^0.13.2: +graphql@^0.13.0, graphql@^0.13.2: version "0.13.2" resolved "http://registry.npmjs.org/graphql/-/graphql-0.13.2.tgz#4c740ae3c222823e7004096f832e7b93b2108270" integrity sha512-QZ5BL8ZO/B20VA8APauGBg3GyEgZ19eduvpLWoq5x7gMmWnHoy8rlQWPLmWgFvo1yNgjSEFMesmS4R6pPr7xog==