diff --git a/packages/gatsby-core-utils/README.md b/packages/gatsby-core-utils/README.md index 1f6ff7bade6de..ea1a0bc57eef5 100644 --- a/packages/gatsby-core-utils/README.md +++ b/packages/gatsby-core-utils/README.md @@ -121,3 +121,17 @@ await fs.writeFile("pathToFile", "my custom content") await mutex.release() ``` + +### Hashing + +Parts of [`hash-wasm`](https://github.com/Daninet/hash-wasm) are re-exported from `gatsby-core-utils` or used in custom functions. When working on hashing where you'd normally use `crypto` from Node.js, you can use these functions instead. They especially show their advantage on large inputs so don't feel obliged to _always_ use them. Refer to `hash-wasm`'s documentation for more details on usage for the re-exported functions. + +```js +const { md5File, md5, createMD5, sha256, sha1 } = require("gatsby-core-utils") + +// For md5, createMD5, sha256, sha1 refer to hash-wasm +await md5(`some-string`) + +// md5File gives you the MD5 hex hash for a given filepath +await md5File(`package.json`) +``` diff --git a/packages/gatsby-core-utils/package.json b/packages/gatsby-core-utils/package.json index e1495ddabcf75..68d02a4595d3b 100644 --- a/packages/gatsby-core-utils/package.json +++ b/packages/gatsby-core-utils/package.json @@ -68,6 +68,7 @@ "file-type": "^16.5.3", "fs-extra": "^11.1.0", "got": "^11.8.5", + "hash-wasm": "^4.9.0", "import-from": "^4.0.0", "lmdb": "2.5.3", "lock": "^1.1.0", diff --git a/packages/gatsby-core-utils/src/hash.ts b/packages/gatsby-core-utils/src/hash.ts new file mode 100644 index 0000000000000..e753575de3ed1 --- /dev/null +++ b/packages/gatsby-core-utils/src/hash.ts @@ -0,0 +1 @@ +export { md5, createMD5, sha256, sha1 } from "hash-wasm" diff --git a/packages/gatsby-core-utils/src/index.ts b/packages/gatsby-core-utils/src/index.ts index 42fad8101cd39..27ee20a046826 100644 --- a/packages/gatsby-core-utils/src/index.ts +++ b/packages/gatsby-core-utils/src/index.ts @@ -21,5 +21,7 @@ export { createFilePath } from "./filename-utils" export { readConfigFile, getConfigPath } from "./utils" export { lock } from "./lock" export { murmurhash } from "./murmurhash" +export * from "./hash" +export { md5File } from "./md5-file" export type { IFetchRemoteFileOptions } from "./fetch-remote-file" diff --git a/packages/gatsby-core-utils/src/md5-file.ts b/packages/gatsby-core-utils/src/md5-file.ts new file mode 100644 index 0000000000000..1ecba2c23ce8e --- /dev/null +++ b/packages/gatsby-core-utils/src/md5-file.ts @@ -0,0 +1,29 @@ +import { createMD5 } from "hash-wasm" +import * as fs from "fs-extra" + +/** + * Create a MD5 hash from a given filePath + * @param filePath Absolute path to the file + * @returns MD5 hash in hex format + */ +export const md5File = async (filePath: string): Promise => { + const md5hasher = await createMD5() + + return new Promise((resolve, reject) => { + md5hasher.init() + + const fileInput = fs.createReadStream(filePath) + + fileInput.on(`error`, err => { + reject(err) + }) + + fileInput.on(`data`, data => { + md5hasher.update(data) + }) + + fileInput.on(`end`, () => { + resolve(md5hasher.digest(`hex`)) + }) + }) +} diff --git a/packages/gatsby/package.json b/packages/gatsby/package.json index b5079c2a16b4f..4d22faa119a8c 100644 --- a/packages/gatsby/package.json +++ b/packages/gatsby/package.json @@ -120,7 +120,6 @@ "latest-version": "^7.0.0", "lmdb": "2.5.3", "lodash": "^4.17.21", - "md5-file": "^5.0.0", "meant": "^1.0.3", "memoizee": "^0.4.15", "micromatch": "^4.0.5", diff --git a/packages/gatsby/src/bootstrap/load-plugins/utils/create-hash.ts b/packages/gatsby/src/bootstrap/load-plugins/utils/create-hash.ts index cc8130ccf00e1..fc1b99f48ed7d 100644 --- a/packages/gatsby/src/bootstrap/load-plugins/utils/create-hash.ts +++ b/packages/gatsby/src/bootstrap/load-plugins/utils/create-hash.ts @@ -6,6 +6,7 @@ export function createFileContentHash( root: string, globPattern: string ): string { + // TODO: Use hash-wasm const hash = crypto.createHash(`md5`) const files = glob.sync(`${root}/${globPattern}`, { nodir: true }) diff --git a/packages/gatsby/src/bootstrap/redirects-writer.ts b/packages/gatsby/src/bootstrap/redirects-writer.ts index e7e7ddbd4941c..d22cc1a26515f 100644 --- a/packages/gatsby/src/bootstrap/redirects-writer.ts +++ b/packages/gatsby/src/bootstrap/redirects-writer.ts @@ -1,10 +1,9 @@ import _ from "lodash" -import crypto from "crypto" import fs from "fs-extra" +import { joinPath, md5 } from "gatsby-core-utils" +import reporter from "gatsby-cli/lib/reporter" import { store, emitter } from "../redux" import { IRedirect } from "../redux/types" -import { joinPath } from "gatsby-core-utils" -import reporter from "gatsby-cli/lib/reporter" let lastHash: string | null = null let bootstrapFinished = false @@ -53,10 +52,7 @@ export const writeRedirects = async (): Promise => { ) } - const newHash = crypto - .createHash(`md5`) - .update(JSON.stringify(browserRedirects)) - .digest(`hex`) + const newHash = await md5(JSON.stringify(browserRedirects)) if (newHash === lastHash) { return diff --git a/packages/gatsby/src/bootstrap/requires-writer.ts b/packages/gatsby/src/bootstrap/requires-writer.ts index a8c89e52df4dc..032ed766d4d4d 100644 --- a/packages/gatsby/src/bootstrap/requires-writer.ts +++ b/packages/gatsby/src/bootstrap/requires-writer.ts @@ -1,11 +1,9 @@ import _ from "lodash" import path from "path" import fs from "fs-extra" -import crypto from "crypto" -import { slash } from "gatsby-core-utils" import reporter from "gatsby-cli/lib/reporter" import { match } from "@gatsbyjs/reach-router" -import { joinPath } from "gatsby-core-utils" +import { joinPath, md5, slash } from "gatsby-core-utils" import { store, emitter } from "../redux/" import { IGatsbyState, IGatsbyPage, IGatsbySlice } from "../redux/types" import { @@ -177,22 +175,6 @@ const getMatchPaths = ( }) } -const createHash = ( - matchPaths: Array, - components: Array, - cleanedSSRVisitedPageComponents: Array -): string => - crypto - .createHash(`md5`) - .update( - JSON.stringify({ - matchPaths, - components, - cleanedSSRVisitedPageComponents, - }) - ) - .digest(`hex`) - // Write out pages information. export const writeAll = async (state: IGatsbyState): Promise => { const { program, slices } = state @@ -212,10 +194,12 @@ export const writeAll = async (state: IGatsbyState): Promise => { ) } - const newHash = createHash( - matchPaths, - components, - cleanedSSRVisitedPageComponents + const newHash = await md5( + JSON.stringify({ + matchPaths, + components, + cleanedSSRVisitedPageComponents, + }) ) if (newHash === lastHash) { diff --git a/packages/gatsby/src/query/file-parser.js b/packages/gatsby/src/query/file-parser.js index 5a478cfb9dfd6..d404e7a355db1 100644 --- a/packages/gatsby/src/query/file-parser.js +++ b/packages/gatsby/src/query/file-parser.js @@ -23,7 +23,7 @@ const report = require(`gatsby-cli/lib/reporter`) import type { DocumentNode } from "graphql" import { babelParseToAst } from "../utils/babel-parse-to-ast" import { codeFrameColumns } from "@babel/code-frame" -import { getPathToLayoutComponent } from "gatsby-core-utils" +import { getPathToLayoutComponent, md5 } from "gatsby-core-utils" const apiRunnerNode = require(`../utils/api-runner-node`) const { actions } = require(`../redux/actions`) @@ -524,11 +524,7 @@ export default class FileParser { return null } - const hash = crypto - .createHash(`md5`) - .update(file) - .update(text) - .digest(`hex`) + const hash = await md5(file + text) try { if (!cache[hash]) { diff --git a/packages/gatsby/src/query/graphql-runner.ts b/packages/gatsby/src/query/graphql-runner.ts index 18f4c9f81e3fa..5a42e989c31eb 100644 --- a/packages/gatsby/src/query/graphql-runner.ts +++ b/packages/gatsby/src/query/graphql-runner.ts @@ -1,4 +1,3 @@ -import crypto from "crypto" import { Span } from "opentracing" import { parse, @@ -14,8 +13,9 @@ import { } from "graphql" import { debounce } from "lodash" import reporter from "gatsby-cli/lib/reporter" -import { createPageDependency } from "../redux/actions/add-page-dependency" +import { sha1 } from "gatsby-core-utils/hash" +import { createPageDependency } from "../redux/actions/add-page-dependency" import withResolverContext from "../schema/context" import { LocalNodeModel } from "../schema/node-model" import { Store } from "redux" @@ -210,9 +210,9 @@ export class GraphQLRunner { if (this.stats) { this.stats.totalQueries++ - this.stats.uniqueQueries.add( - crypto.createHash(`sha1`).update(queryText).digest(`hex`) - ) + const hash = await sha1(queryText) + + this.stats.uniqueQueries.add(hash) } const { errors, warnings, document } = this.validate( diff --git a/packages/gatsby/src/query/query-runner.ts b/packages/gatsby/src/query/query-runner.ts index f652947f5f930..5bd88552af66c 100644 --- a/packages/gatsby/src/query/query-runner.ts +++ b/packages/gatsby/src/query/query-runner.ts @@ -2,8 +2,8 @@ import { Span } from "opentracing" import _ from "lodash" import fs from "fs-extra" import report from "gatsby-cli/lib/reporter" -import crypto from "crypto" import { ExecutionResult, GraphQLError } from "graphql" +import { sha1 } from "gatsby-core-utils/hash" import path from "path" import { store } from "../redux" @@ -172,10 +172,7 @@ export async function queryRunner( } const resultJSON = JSON.stringify(result) - const resultHash = crypto - .createHash(`sha1`) - .update(resultJSON) - .digest(`base64`) + const resultHash = await sha1(resultJSON) const resultHashCache = getResultHashCache() diff --git a/packages/gatsby/src/services/initialize.ts b/packages/gatsby/src/services/initialize.ts index 2667dbad71cce..d1ee5d625b3b2 100644 --- a/packages/gatsby/src/services/initialize.ts +++ b/packages/gatsby/src/services/initialize.ts @@ -2,8 +2,7 @@ import _ from "lodash" import { slash, isCI } from "gatsby-core-utils" import * as fs from "fs-extra" import { releaseAllMutexes } from "gatsby-core-utils/mutex" -import md5File from "md5-file" -import crypto from "crypto" +import { md5, md5File } from "gatsby-core-utils" import path from "path" import telemetry from "gatsby-telemetry" import glob from "globby" @@ -338,10 +337,7 @@ export async function initialize({ ) ) - const pluginsHash = crypto - .createHash(`md5`) - .update(JSON.stringify(pluginVersions.concat(hashes))) - .digest(`hex`) + const pluginsHash = await md5(JSON.stringify(pluginVersions.concat(hashes))) const oldPluginsHash = state && state.status ? state.status.PLUGINS_HASH : `` diff --git a/packages/gatsby/src/utils/worker/__tests__/queries.ts b/packages/gatsby/src/utils/worker/__tests__/queries.ts index 5b2eb7adee543..12b9668d6580d 100644 --- a/packages/gatsby/src/utils/worker/__tests__/queries.ts +++ b/packages/gatsby/src/utils/worker/__tests__/queries.ts @@ -384,7 +384,7 @@ describe(`worker (queries)`, () => { queryType: `static`, path: `sq--q1`, queryHash: `q1-hash`, - resultHash: `Dr5hgCDB+R0S9oRBWeZYj3lB7VI=`, + resultHash: `0ebe618020c1f91d12f6844159e6588f7941ed52`, }, type: `PAGE_QUERY_RUN`, }, @@ -415,7 +415,7 @@ describe(`worker (queries)`, () => { componentPath: `/foo.js`, queryType: `page`, path: `/foo`, - resultHash: `8dW7PoqwZNk/0U8LO6kTj1qBCwU=`, + resultHash: `f1d5bb3e8ab064d93fd14f0b3ba9138f5a810b05`, }, type: `PAGE_QUERY_RUN`, }, @@ -430,7 +430,7 @@ describe(`worker (queries)`, () => { componentPath: `/bar.js`, queryType: `page`, path: `/bar`, - resultHash: `iKmhf9XgbsfK7qJw0tw95pmGwJM=`, + resultHash: `88a9a17fd5e06ec7caeea270d2dc3de69986c093`, }, type: `PAGE_QUERY_RUN`, }, diff --git a/yarn.lock b/yarn.lock index cc5ee43b3fb71..234ac658314f2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12219,6 +12219,11 @@ has@^1.0.0, has@^1.0.1, has@^1.0.3: dependencies: function-bind "^1.1.1" +hash-wasm@^4.9.0: + version "4.9.0" + resolved "https://registry.yarnpkg.com/hash-wasm/-/hash-wasm-4.9.0.tgz#7e9dcc9f7d6bd0cc802f2a58f24edce999744206" + integrity sha512-7SW7ejyfnRxuOc7ptQHSf4LDoZaWOivfzqw+5rpcQku0nHfmicPKE51ra9BiRLAmT8+gGLestr1XroUkqdjL6w== + hasha@^5.2.2: version "5.2.2" resolved "https://registry.yarnpkg.com/hasha/-/hasha-5.2.2.tgz#a48477989b3b327aea3c04f53096d816d97522a1"