[v2] Initial commit of gatsby-plugin-guess-js & gatsby-source-wikipedia

packages/gatsby-plugin-guess-js/.babelrc

@@ -0,0 +1,5 @@
+{
+  "presets": [
+    ["../../.babel-preset.js", { "browser": true }]
+  ]
+}

packages/gatsby-plugin-guess-js/.gitignore

@@ -0,0 +1,3 @@
+/*.js
+!index.js
+yarn.lock

packages/gatsby-plugin-guess-js/.npmignore

@@ -0,0 +1,34 @@
+# Logs
+logs
+*.log
+
+# Runtime data
+pids
+*.pid
+*.seed
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+
+# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (http://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directory
+# https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git
+node_modules
+*.un~
+yarn.lock
+src
+flow-typed
+coverage
+decls
+examples

packages/gatsby-plugin-guess-js/README.md

@@ -0,0 +1,48 @@
+# gatsby-plugin-guess-js
+
+Gatsby plugin for integrating [Guess.js](https://github.com/guess-js/guess) with Gatsby.
+
+Guess.js is a library for enabling data-driven user-experiences on the web.
+
+When this plugin is added to your site, it will automatically download Google Analytics
+data and use this to create a model to predict which page a user is most likely to visit
+from a given page.
+
+The plugin uses this information to do two things.
+
+* When generating HTML pages, it automatically adds `<link prefetch>` for resources on pages
+the user is likely to visit. This means that as soon as a person visits your site, their browser
+will immediately start downloading in the background code and data for links they'll likely click on which
+can dramatically improve the site performance.
+* Once the person has loaded your site and visits additional pages, the plugin will continue to predict
+which pages will be visited and prefetch their resources as well.
+
+## Demo
+
+https://guess-gatsby-wikipedia-demo.firebaseapp.com
+
+## Install
+
+`npm install --save gatsby-plugin-guess-js`
+
+## How to use
+
+```javascript
+// In your gatsby-config.js
+module.exports = {
+  plugins: [
+    {
+      resolve: "gatsby-source-wikipedia",
+      options: {
+        // Find the view id in the GA admin in a section labeled "views"
+        GAViewID: `VIEW_ID`,
+        minimumThreshold: 0.03,
+        // The "period" for fetching analytic data.
+        period: {
+          startDate: new Date("2018-1-1"),
+          endDate: new Date(),
+        },
+      },
+    },
+  ],
+}

packages/gatsby-plugin-guess-js/index.js

@@ -0,0 +1 @@
+// noop

packages/gatsby-plugin-guess-js/package.json

@@ -0,0 +1,32 @@
+{
+  "name": "gatsby-plugin-guess-js",
+  "version": "1.0.0-alpha.1",
+  "description": "Gatsby plugin providing drop-in integration with Guess.js to enabling using machine learning and analytics data to power prefetching",
+  "main": "index.js",
+  "scripts": {
+    "build": "babel src --out-dir . --ignore **/__tests__",
+    "watch": "babel -w src --out-dir . --ignore **/__tests__",
+    "prepublish": "cross-env NODE_ENV=production npm run build"
+  },
+  "keywords": [
+    "gatsby", "gatsby-plugin", "guess.js", "google analytics", "machine learning", "prefetch"
+  ],
+  "author": "Kyle Mathews <mathews.kyle@gmail.com>",
+  "bugs": {
+    "url": "https://github.com/gatsbyjs/gatsby/issues"
+  },
+  "homepage": "https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-plugin-guess-js#readme",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/gatsbyjs/gatsby.git"
+  },
+  "license": "MIT",
+  "dependencies": {
+    "@babel/runtime": "^7.0.0-beta.38"
+  },
+  "devDependencies": {
+    "@babel/cli": "^7.0.0-beta.38",
+    "@babel/core": "^7.0.0-beta.38",
+    "cross-env": "^5.0.5"
+  }
+}

packages/gatsby-plugin-guess-js/src/gatsby-browser.js

@@ -0,0 +1,85 @@
+const { guess } = require(`guess-webpack/api`)
+
+exports.disableCorePrefetching = () => true
+
+const currentPathname = () =>
+  window.location.pathname.slice(-1) === `/`
+    ? window.location.pathname.slice(0, -1)
+    : window.location.pathname
+
+let initialPath
+let notNavigated = true
+exports.onRouteUpdate = ({ location }) => {
+  if (initialPath !== location.pathname) {
+    notNavigated = false
+    return
+  }
+  initialPath = location.pathname
+}
+
+let chunksPromise
+const chunks = pathPrefix => {
+  if (!chunksPromise) {
+    chunksPromise = fetch(`${window.location.origin}/webpack.stats.json`).then(
+      res => res.json()
+    )
+  }
+
+  return chunksPromise
+}
+
+let hasPrefetched = {}
+const prefetch = url => {
+  if (hasPrefetched[url]) {
+    return
+  }
+  hasPrefetched[url] = true
+  const link = document.createElement(`link`)
+  link.setAttribute(`rel`, `prefetch`)
+  link.setAttribute(`href`, url)
+  const parentElement =
+    document.getElementsByTagName(`head`)[0] ||
+    document.getElementsByName(`script`)[0].parentNode
+  parentElement.appendChild(link)
+}
+
+exports.onPrefetchPathname = ({ pathname, pathPrefix }, pluginOptions) => {
+  if (process.env.NODE_ENV === `production`) {
+    const predictions = guess(currentPathname(), [pathname])
+    const matchedPaths = Object.keys(predictions).filter(
+      match =>
+        // If the prediction is below the minimum threshold for prefetching
+        // we skip.
+        pluginOptions.minimumThreshold &&
+        pluginOptions.minimumThreshold > predictions[match]
+          ? false
+          : true
+    )
+
+    // Don't prefetch from client for the initial path as we did that
+    // during SSR
+    if (notNavigated && initialPath === window.location.pathname) {
+      return
+    }
+
+    if (matchedPaths.length > 0) {
+      matchedPaths.forEach(p => {
+        chunks(pathPrefix).then(chunk => {
+          // eslint-disable-next-line
+          const page = ___loader.getPage(p)
+          if (!page) return
+          let resources = []
+          if (chunk.assetsByChunkName[page.componentChunkName]) {
+            resources = resources.concat(
+              chunk.assetsByChunkName[page.componentChunkName]
+            )
+          }
+          // eslint-disable-next-line
+          resources.push(`static/d/${___dataPaths[page.jsonName]}.json`)
+          // TODO add support for pathPrefix
+          resources.forEach(r => prefetch(`/${r}`))
+        })
+      })
+    }
+  }
+}

packages/gatsby-plugin-guess-js/src/gatsby-node.js

@@ -0,0 +1,38 @@
+const { GuessPlugin } = require(`guess-webpack`)
+
+let guessPlugin
+exports.onPreBootstrap = (_, pluginOptions) => {
+  const { period, GAViewID } = pluginOptions
+  period.startDate = new Date(period.startDate)
+  period.endDate = new Date(period.endDate)
+  guessPlugin = new GuessPlugin({
+    // GA view ID.
+    GA: GAViewID,
+
+    // Hints Guess to not perform prefetching and delegate this logic to
+    // its consumer.
+    runtime: {
+      delegate: true,
+    },
+
+    // Since Gatsby already has the required metadata for pre-fetching,
+    // Guess does not have to collect the routes and the corresponding
+    // bundle entry points.
+    routeProvider: false,
+
+    // Optional argument. It takes the data for the last year if not
+    // specified.
+    period: period
+      ? period
+      : {
+          startDate: new Date(`2018-1-1`),
+          endDate: new Date(),
+        },
+  })
+}
+
+exports.onCreateWebpackConfig = ({ actions, stage }, pluginOptions) => {
+  actions.setWebpackConfig({
+    plugins: [guessPlugin],
+  })
+}

packages/gatsby-plugin-guess-js/src/gatsby-ssr.js

@@ -0,0 +1,93 @@
+const _ = require(`lodash`)
+const nodePath = require(`path`)
+const fs = require(`fs`)
+const React = require(`react`)
+
+const { guess } = require(`guess-webpack/api`)
+
+// Google Analytics removes trailing slashes from pathnames
+const removeTrailingSlash = pathname =>
+  pathname.slice(-1) === `/` ? pathname.slice(0, -1) : pathname
+
+function urlJoin(...parts) {
+  return parts.reduce((r, next) => {
+    const segment = next == null ? `` : String(next).replace(/^\/+/, ``)
+    return segment ? `${r.replace(/\/$/, ``)}/${segment}` : r
+  }, ``)
+}
+
+let pd = []
+const readPageData = () => {
+  if (pd.length > 0) {
+    return pd
+  } else {
+    pd = JSON.parse(
+      fs.readFileSync(nodePath.join(process.cwd(), `.cache`, `data.json`))
+    )
+    return pd
+  }
+}
+
+let s
+const readStats = () => {
+  if (s) {
+    return s
+  } else {
+    s = JSON.parse(
+      fs.readFileSync(`${process.cwd()}/public/webpack.stats.json`, `utf-8`)
+    )
+    return s
+  }
+}
+
+exports.onRenderBody = (
+  { setHeadComponents, pathname, pathPrefix },
+  pluginOptions
+) => {
+  if (process.env.NODE_ENV === `production`) {
+    const pagesData = readPageData()
+    const stats = readStats()
+    const path = removeTrailingSlash(pathname)
+    const predictions = guess(path)
+    if (!_.isEmpty(predictions)) {
+      const matchedPaths = Object.keys(predictions).filter(
+        match =>
+          // If the prediction is below the minimum threshold for prefetching
+          // we skip.
+          pluginOptions.minimumThreshold &&
+          pluginOptions.minimumThreshold > predictions[match]
+            ? false
+            : true
+      )
+      const matchedPages = matchedPaths.map(match =>
+        _.find(
+          pagesData.pages,
+          page => removeTrailingSlash(page.path) === match
+        )
+      )
+      let componentUrls = []
+      matchedPages.forEach(p => {
+        if (p && p.componentChunkName) {
+          const fetchKey = `assetsByChunkName[${p.componentChunkName}]`
+          let chunks = _.get(stats, fetchKey)
+          componentUrls = [...componentUrls, ...chunks]
+        }
+      })
+      componentUrls = _.uniq(componentUrls)
+      const components = componentUrls.map(c =>
+        React.createElement(`Link`, {
+          rel: `prefetch`,
+          as: c.slice(-2) === `js` ? `script` : undefined,
+          rel:
+            c.slice(-2) === `js` ? `prefetch` : `prefetch alternate stylesheet`,
+          key: c,
+          href: urlJoin(pathPrefix, c),
+        })
+      )
+
+      setHeadComponents(components)
+    }
+
+    return true
+  }
+}

packages/gatsby-source-filesystem/src/create-remote-file-node.js

@@ -158,7 +158,14 @@ const requestRemoteNode = (url, headers, tmpFilename, filename) =>
  * @param {CreateRemoteFileNodePayload} options
  * @return {Promise<Object>} Resolves with the fileNode
  */
-async function processRemoteNode({ url, store, cache, createNode, auth = {}, createNodeId }) {
+async function processRemoteNode({
+  url,
+  store,
+  cache,
+  createNode,
+  auth = {},
+  createNodeId,
+}) {
   // Ensure our cache directory exists.
   const programDir = store.getState().program.directory
   await fs.ensureDir(path.join(programDir, CACHE_DIR, FS_PLUGIN_DIR))
@@ -260,7 +267,14 @@ const pushTask = task =>
  * @param {CreateRemoteFileNodePayload} options
  * @return {Promise<Object>}                  Returns the created node
  */
-module.exports = ({ url, store, cache, createNode, auth = {}, createNodeId }) => {
+module.exports = ({
+  url,
+  store,
+  cache,
+  createNode,
+  auth = {},
+  createNodeId,
+}) => {
   // Check if we already requested node for this remote file
   // and return stored promise if we did.
   if (processingCache[url]) {