Skip to content

Commit

Permalink
fix(gatsby): assign correct parentSpans to PQR activities (#33568)
Browse files Browse the repository at this point in the history
  • Loading branch information
pieh authored Oct 18, 2021
1 parent 31d5a5e commit 8dbf550
Show file tree
Hide file tree
Showing 3 changed files with 27 additions and 10 deletions.
8 changes: 5 additions & 3 deletions packages/gatsby/src/commands/build.ts
Original file line number Diff line number Diff line change
Expand Up @@ -221,7 +221,7 @@ module.exports = async function build(program: IBuildArgs): Promise<void> {
}

const cacheActivity = report.activityTimer(`Caching Webpack compilations`, {
parentSpan: buildActivityTimer.span,
parentSpan: buildSpan,
})
try {
cacheActivity.start()
Expand Down Expand Up @@ -249,12 +249,14 @@ module.exports = async function build(program: IBuildArgs): Promise<void> {

let waitForWorkerPoolRestart = Promise.resolve()
if (process.env.GATSBY_EXPERIMENTAL_PARALLEL_QUERY_RUNNING) {
await runQueriesInWorkersQueue(workerPool, queryIds)
await runQueriesInWorkersQueue(workerPool, queryIds, {
parentSpan: buildSpan,
})
// Jobs still might be running even though query running finished
await waitUntilAllJobsComplete()
// Restart worker pool before merging state to lower memory pressure while merging state
waitForWorkerPoolRestart = workerPool.restart()
await mergeWorkerState(workerPool)
await mergeWorkerState(workerPool, buildSpan)
} else {
await runStaticQueries({
queryIds,
Expand Down
2 changes: 1 addition & 1 deletion packages/gatsby/src/utils/worker/__tests__/queries.ts
Original file line number Diff line number Diff line change
Expand Up @@ -298,7 +298,7 @@ describeWhenLMDB(`worker (queries)`, () => {
const spy = jest.spyOn(worker.single, `runQueries`)

// @ts-ignore - worker is defined
await runQueriesInWorkersQueue(worker, queryIdsBig, 10)
await runQueriesInWorkersQueue(worker, queryIdsBig, { chunkSize: 10 })
const stateFromWorker = await worker.single.getState()

// Called the complete ABC so we can test _a
Expand Down
27 changes: 21 additions & 6 deletions packages/gatsby/src/utils/worker/pool.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { WorkerPool } from "gatsby-worker"
import { chunk } from "lodash"
import reporter from "gatsby-cli/lib/reporter"
import { cpuCoreCount } from "gatsby-core-utils"
import { Span } from "opentracing"

import { IGroupedQueryIds } from "../../services"
import { initJobsMessagingInMainProcess } from "../jobs/worker-messaging"
Expand Down Expand Up @@ -46,16 +47,27 @@ function handleRunQueriesInWorkersQueueError(e: Error): never {
export async function runQueriesInWorkersQueue(
pool: GatsbyWorkerPool,
queryIds: IGroupedQueryIds,
chunkSize = queriesChunkSize
opts?: {
chunkSize?: number
parentSpan?: Span
}
): Promise<void> {
const activity = reporter.createProgress(
`run queries in workers`,
queryIds.staticQueryIds.length + queryIds.pageQueryIds.length
queryIds.staticQueryIds.length + queryIds.pageQueryIds.length,
0,
{ parentSpan: opts?.parentSpan }
)
activity.start()
try {
const staticQuerySegments = chunk(queryIds.staticQueryIds, chunkSize)
const pageQuerySegments = chunk(queryIds.pageQueryIds, chunkSize)
const staticQuerySegments = chunk(
queryIds.staticQueryIds,
opts?.chunkSize ?? queriesChunkSize
)
const pageQuerySegments = chunk(
queryIds.pageQueryIds,
opts?.chunkSize ?? queriesChunkSize
)

pool.all.setComponents()

Expand Down Expand Up @@ -90,8 +102,11 @@ export async function runQueriesInWorkersQueue(
}
}

export async function mergeWorkerState(pool: GatsbyWorkerPool): Promise<void> {
const activity = reporter.activityTimer(`Merge worker state`)
export async function mergeWorkerState(
pool: GatsbyWorkerPool,
parentSpan?: Span
): Promise<void> {
const activity = reporter.activityTimer(`Merge worker state`, { parentSpan })
activity.start()

for (const { workerId } of pool.getWorkerInfo()) {
Expand Down

0 comments on commit 8dbf550

Please sign in to comment.