perf(gatsby-source-contentful): prevent redundant fs/remote fetches f…

…or base64 (#28438) (#28515) * perf(gatsby-source-contentful): add inFlight check for base64 fetches * Add resolved map to return data sync when possible (cherry picked from commit 2755cfa) Co-authored-by: Peter van der Zee <209817+pvdz@users.noreply.github.com>
gatsbyjs · Dec 7, 2020 · e16f2b5 · e16f2b5
1 parent 6f984b6
commit e16f2b5
Showing 1 changed file with 38 additions and 3 deletions.
diff --git a/packages/gatsby-source-contentful/src/extend-node-type.js b/packages/gatsby-source-contentful/src/extend-node-type.js
@@ -38,6 +38,12 @@ const REMOTE_CACHE_FOLDER =
   path.join(process.cwd(), `.cache/remote_cache`)
 const CACHE_IMG_FOLDER = path.join(REMOTE_CACHE_FOLDER, `images`)
 
+// Promises that rejected should stay in this map. Otherwise remove promise and put their data in resolvedBase64Cache
+const inFlightBase64Cache = new Map()
+// This cache contains the resolved base64 fetches. This prevents async calls for promises that have resolved.
+// The images are based on urls with w=20 and should be relatively small (<2kb) but it does stick around in memory
+const resolvedBase64Cache = new Map()
+
 const {
   ImageFormatType,
   ImageResizingBehavior,
@@ -53,11 +59,26 @@ const isImage = image =>
     _.get(image, `file.contentType`)
   )
 
+// Note: this may return a Promise<body>, body (sync), or null
 const getBase64Image = imageProps => {
-  if (!imageProps) return null
+  if (!imageProps) {
+    return null
+  }
 
   const requestUrl = `https:${imageProps.baseUrl}?w=20`
 
+  // Prefer to return data sync if we already have it
+  const alreadyFetched = resolvedBase64Cache.get(requestUrl)
+  if (alreadyFetched) {
+    return alreadyFetched
+  }
+
+  // If already in flight for this url return the same promise as the first call
+  const inFlight = inFlightBase64Cache.get(requestUrl)
+  if (inFlight) {
+    return inFlight
+  }
+
   // Note: sha1 is unsafe for crypto but okay for this particular case
   const shasum = crypto.createHash(`sha1`)
   shasum.update(requestUrl)
@@ -70,10 +91,16 @@ const getBase64Image = imageProps => {
 
   if (fs.existsSync(cacheFile)) {
     // TODO: against dogma, confirm whether readFileSync is indeed slower
-    return fs.promises.readFile(cacheFile, `utf8`)
+    const promise = fs.promises.readFile(cacheFile, `utf8`)
+    inFlightBase64Cache.set(requestUrl, promise)
+    return promise.then(body => {
+      inFlightBase64Cache.delete(requestUrl)
+      resolvedBase64Cache.set(requestUrl, body)
+      return body
+    })
   }
 
-  return new Promise((resolve, reject) => {
+  const promise = new Promise((resolve, reject) => {
     base64Img.requestBase64(requestUrl, (a, b, body) => {
       // TODO: against dogma, confirm whether writeFileSync is indeed slower
       fs.promises
@@ -87,6 +114,14 @@ const getBase64Image = imageProps => {
         })
     })
   })
+
+  inFlightBase64Cache.set(requestUrl, promise)
+
+  return promise.then(body => {
+    inFlightBase64Cache.delete(requestUrl)
+    resolvedBase64Cache.set(requestUrl, body)
+    return body
+  })
 }
 
 const getBasicImageProps = (image, args) => {