Skip to content

Commit

Permalink
fix(gatsby): don't block event loop during inference (#37780) (#37801)
Browse files Browse the repository at this point in the history
Don't block the event loop during inference

(cherry picked from commit c08048d)

Co-authored-by: Tyler Barnes <tylerdbarnes@gmail.com>
Co-authored-by: Michal Piechowiak <misiek.piechowiak@gmail.com>
  • Loading branch information
3 people authored Mar 29, 2023
1 parent 50e3f94 commit 5bdef4a
Show file tree
Hide file tree
Showing 3 changed files with 48 additions and 12 deletions.
9 changes: 7 additions & 2 deletions packages/gatsby/src/redux/reducers/inference-metadata.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,14 @@ const incrementalReducer = (

case `BUILD_TYPE_METADATA`: {
// Overwrites existing metadata
const { nodes, typeName } = action.payload
const { nodes, typeName, clearExistingMetadata } = action.payload
if (!state[typeName]?.ignored) {
state[typeName] = addNodes(initialTypeMetadata(), nodes)
const initialMetadata =
clearExistingMetadata || !state[typeName]
? initialTypeMetadata()
: state[typeName]

state[typeName] = addNodes(initialMetadata, nodes)
}
return state
}
Expand Down
1 change: 1 addition & 0 deletions packages/gatsby/src/redux/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -895,6 +895,7 @@ interface IBuildTypeMetadataAction {
type: `BUILD_TYPE_METADATA`
payload: {
nodes: Array<IGatsbyNode>
clearExistingMetadata: boolean
typeName: string
}
}
Expand Down
50 changes: 40 additions & 10 deletions packages/gatsby/src/schema/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -60,18 +60,48 @@ const buildInferenceMetadata = ({ types }) =>
// TODO: use async iterators when we switch to node>=10
// or better investigate if we can offload metadata building to worker/Jobs API
// and then feed the result into redux?
const processNextType = () => {
const processNextType = async () => {
const typeName = typeNames.pop()
store.dispatch({
type: `BUILD_TYPE_METADATA`,
payload: {
typeName,
nodes: getDataStore().iterateNodesByType(typeName),
},
})

let processingNodes = []
let dispatchCount = 0
function dispatchNodes() {
return new Promise(res => {
store.dispatch({
type: `BUILD_TYPE_METADATA`,
payload: {
typeName,
// only clear metadata on the first chunk for this type
clearExistingMetadata: dispatchCount++ === 0,
nodes: processingNodes,
},
})
setImmediate(() => {
// clear this array after BUILD_TYPE_METADATA reducer has synchronously run
processingNodes = []
// dont block the event loop. node may decide to free previous processingNodes array from memory if it needs to.
setImmediate(() => {
res(null)
})
})
})
}

for (const node of getDataStore().iterateNodesByType(typeName)) {
processingNodes.push(node)

if (processingNodes.length > 1000) {
await dispatchNodes()
}
}

if (processingNodes.length > 0) {
await dispatchNodes()
}

if (typeNames.length > 0) {
// Give event-loop a break
setTimeout(processNextType, 0)
// dont block the event loop
setImmediate(() => processNextType())
} else {
resolve()
}
Expand Down

0 comments on commit 5bdef4a

Please sign in to comment.