Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(gatsby): use lmdb for resultHash cache so shared across workers #34925

Merged
merged 10 commits into from
Mar 11, 2022
1 change: 1 addition & 0 deletions packages/gatsby/src/query/__tests__/data-tracking.js
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ jest.mock(`../../utils/cache-lmdb`, () => {
get = jest.fn(() => Promise.resolve())
set = jest.fn(() => Promise.resolve())
},
__esModule: true,
}
})

Expand Down
19 changes: 15 additions & 4 deletions packages/gatsby/src/query/query-runner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,18 @@ import errorParser from "./error-parser"
import { GraphQLRunner } from "./graphql-runner"
import { IExecutionResult, PageContext } from "./types"
import { pageDataExists, savePageQueryResult } from "../utils/page-data"

const resultHashes = new Map()
import GatsbyCacheLmdb from "../utils/cache-lmdb"

let resultHashCache: GatsbyCacheLmdb | undefined
function getResultHashCache(): GatsbyCacheLmdb {
if (!resultHashCache) {
resultHashCache = new GatsbyCacheLmdb({
name: `query-result-hashes`,
encoding: `string`,
}).init()
}
return resultHashCache
}

export interface IQueryJob {
id: string
Expand Down Expand Up @@ -171,12 +181,13 @@ export async function queryRunner(
.update(resultJSON)
.digest(`base64`)

const resultHashCache = getResultHashCache()
if (
resultHash !== resultHashes.get(queryJob.id) ||
resultHash !== (await resultHashCache.get(queryJob.id)) ||
(queryJob.isPage &&
!pageDataExists(path.join(program.directory, `public`), queryJob.id))
) {
resultHashes.set(queryJob.id, resultHash)
await resultHashCache.set(queryJob.id, resultHash)

if (queryJob.isPage) {
// We need to save this temporarily in cache because
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ Map {

exports[`Add pages allows you to add pages 1`] = `
Object {
"componentModified": false,
"contextModified": false,
"payload": Object {
"component": "/whatever/index.js",
Expand Down Expand Up @@ -124,6 +125,7 @@ Map {

exports[`Add pages allows you to add pages with context 1`] = `
Object {
"componentModified": false,
"contextModified": false,
"payload": Object {
"component": "/whatever/index.js",
Expand Down Expand Up @@ -172,6 +174,7 @@ Map {

exports[`Add pages allows you to add pages with matchPath 1`] = `
Object {
"componentModified": false,
"contextModified": false,
"payload": Object {
"component": "/whatever/index.js",
Expand Down
3 changes: 3 additions & 0 deletions packages/gatsby/src/redux/actions/public.js
Original file line number Diff line number Diff line change
Expand Up @@ -429,6 +429,8 @@ ${reservedFields.map(f => ` * "${f}"`).join(`\n`)}
const oldPage: Page = store.getState().pages.get(internalPage.path)
const contextModified =
!!oldPage && !_.isEqual(oldPage.context, internalPage.context)
const componentModified =
!!oldPage && !_.isEqual(oldPage.component, internalPage.component)

const alternateSlashPath = page.path.endsWith(`/`)
? page.path.slice(0, -1)
Expand Down Expand Up @@ -496,6 +498,7 @@ ${reservedFields.map(f => ` * "${f}"`).join(`\n`)}
...actionOptions,
type: `CREATE_PAGE`,
contextModified,
componentModified,
plugin,
payload: internalPage,
},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,12 @@ export const pendingPageDataWritesReducer = (
action: ActionsUnion
): IGatsbyState["pendingPageDataWrites"] => {
switch (action.type) {
case `CREATE_PAGE`:
if (action.componentModified) {
state.pagePaths.add(action.payload.path)
}

return state
case `ADD_PENDING_PAGE_DATA_WRITE`:
state.pagePaths.add(action.payload.path)
return state
Expand All @@ -15,6 +21,7 @@ export const pendingPageDataWritesReducer = (
for (const page of action.payload.pages) {
state.pagePaths.add(page)
}

return state
}

Expand Down
1 change: 1 addition & 0 deletions packages/gatsby/src/redux/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -707,6 +707,7 @@ export interface ICreatePageAction {
payload: IGatsbyPage
plugin?: IGatsbyPlugin
contextModified?: boolean
componentModified?: boolean
}

export interface ICreateRedirectAction {
Expand Down