From 0f145e04420d06a4534ab037f50ad7fec28ab619 Mon Sep 17 00:00:00 2001 From: "James A. Petts" Date: Wed, 9 Oct 2019 14:22:31 +0100 Subject: [PATCH 1/3] Update vtkjs and cornerstone-tools versions to consume vtkjs paintfilter's strokeBuffer to improve performance. --- examples/VTKCornerstonePaintingSyncExample.js | 37 ++++++++++++++----- package.json | 6 +-- src/Custom/VTKMPRViewport.js | 7 +++- src/VTKViewport/View2D.js | 7 +++- src/VTKViewport/View3D.js | 7 +++- yarn.lock | 16 ++++---- 6 files changed, 54 insertions(+), 26 deletions(-) diff --git a/examples/VTKCornerstonePaintingSyncExample.js b/examples/VTKCornerstonePaintingSyncExample.js index 619de2d6..76ac76f4 100644 --- a/examples/VTKCornerstonePaintingSyncExample.js +++ b/examples/VTKCornerstonePaintingSyncExample.js @@ -160,7 +160,7 @@ class VTKCornerstonePaintingSyncExample extends Component { ); } - onPaintEnd = () => { + onPaintEnd = strokeBuffer => { const element = this.cornerstoneElements[0]; const enabledElement = cornerstone.getEnabledElement(element); const { getters, setters } = cornerstoneTools.getModule('segmentation'); @@ -174,16 +174,35 @@ class VTKCornerstonePaintingSyncExample extends Component { const stackData = stackState.data[0]; const numberOfFrames = stackData.imageIds.length; + const segmentIndex = labelmap3D.activeSegmentIndex; - // TODO -> Can do more efficiently if we can grab the strokeBuffer from vtk-js. for (let i = 0; i < numberOfFrames; i++) { - const labelmap2D = getters.labelmap2DByImageIdIndex( - labelmap3D, - i, - rows, - columns - ); - setters.updateSegmentsOnLabelmap2D(labelmap2D); + let labelmap2D = labelmap3D.labelmaps2D[i]; + + if (labelmap2D && labelmap2D.segmentsOnLabelmap.includes(segmentIndex)) { + continue; + } + + const frameLength = rows * columns; + const byteOffset = frameLength * i; + const strokeArray = new Uint8Array(strokeBuffer, byteOffset, frameLength); + + const strokeOnFrame = strokeArray.some(element => element === 1); + + if (!strokeOnFrame) { + continue; + } + + if (labelmap2D) { + labelmap2D.segmentsOnLabelmap.push(segmentIndex); + } else { + labelmap2D = getters.labelmap2DByImageIdIndex( + labelmap3D, + i, + rows, + columns + ); + } } cornerstone.updateImage(element); diff --git a/package.json b/package.json index bc47475b..3012df24 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,7 @@ "peerDependencies": { "react": "^16.8.6", "react-dom": "^16.8.6", - "vtk.js": "^11.1.3" + "vtk.js": "^11.2.0" }, "dependencies": { "date-fns": "^2.2.1", @@ -46,7 +46,7 @@ "copy-webpack-plugin": "^5.0.4", "cornerstone-core": "^2.3.0", "cornerstone-math": "^0.1.8", - "cornerstone-tools": "^4.0.9", + "cornerstone-tools": "^4.5.2", "cornerstone-wado-image-loader": "^3.0.5", "cross-env": "^5.2.0", "css-loader": "^3.0.0", @@ -81,7 +81,7 @@ "style-loader": "^0.23.1", "stylelint": "^10.1.0", "stylelint-config-recommended": "^2.2.0", - "vtk.js": "^11.1.3", + "vtk.js": "^11.2.0", "webpack": "4.34.0", "webpack-cli": "^3.3.4", "webpack-dev-server": "^3.8.0", diff --git a/src/Custom/VTKMPRViewport.js b/src/Custom/VTKMPRViewport.js index 01de44cc..64d4a248 100644 --- a/src/Custom/VTKMPRViewport.js +++ b/src/Custom/VTKMPRViewport.js @@ -216,9 +216,12 @@ export default class VtkMpr extends Component { ); this.subs.paintEnd.sub( this.viewWidget.onEndInteractionEvent(() => { - this.paintFilter.endStroke(); + const strokeBufferPromise = this.paintFilter.endStroke(); + if (this.props.onPaintEnd) { - this.props.onPaintEnd(); + strokeBufferPromise.then(strokeBuffer => { + this.props.onPaintEnd(strokeBuffer); + }); } }) ); diff --git a/src/VTKViewport/View2D.js b/src/VTKViewport/View2D.js index 5210d523..bdba4173 100644 --- a/src/VTKViewport/View2D.js +++ b/src/VTKViewport/View2D.js @@ -302,9 +302,12 @@ export default class View2D extends Component { ); this.subs.paintEnd.sub( this.viewWidget.onEndInteractionEvent(() => { - this.paintFilter.endStroke(); + const strokeBufferPromise = this.paintFilter.endStroke(); + if (this.props.onPaintEnd) { - this.props.onPaintEnd(); + strokeBufferPromise.then(strokeBuffer => { + this.props.onPaintEnd(strokeBuffer); + }); } }) ); diff --git a/src/VTKViewport/View3D.js b/src/VTKViewport/View3D.js index 71db675f..2379662d 100644 --- a/src/VTKViewport/View3D.js +++ b/src/VTKViewport/View3D.js @@ -239,9 +239,12 @@ export default class View3D extends Component { ); this.subs.paintEnd.sub( this.viewWidget.onEndInteractionEvent(() => { - this.paintFilter.endStroke(); + const strokeBufferPromise = this.paintFilter.endStroke(); + if (this.props.onPaintEnd) { - this.props.onPaintEnd(); + strokeBufferPromise.then(strokeBuffer => { + this.props.onPaintEnd(strokeBuffer); + }); } }) ); diff --git a/yarn.lock b/yarn.lock index 8a7820b6..445445ed 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3425,10 +3425,10 @@ cornerstone-math@^0.1.8: resolved "https://registry.yarnpkg.com/cornerstone-math/-/cornerstone-math-0.1.8.tgz#68ab1f9e4fdcd7c5cb23a0d2eb4263f9f894f1c5" integrity sha512-x7NEQHBtVG7j1yeyj/aRoKTpXv1Vh2/H9zNLMyqYJDtJkNng8C4Q8M3CgZ1qer0Yr7eVq2x+Ynmj6kfOm5jXKw== -cornerstone-tools@^4.0.9: - version "4.1.1" - resolved "https://registry.yarnpkg.com/cornerstone-tools/-/cornerstone-tools-4.1.1.tgz#69139314e24a667093a41f74ce1310cb34e53544" - integrity sha512-/eyhVvL+OLtwn7jIJuF0RC8kR1rP3fjuYhRgfuJEYWcwmEetYHQhYTFzZVqGWlUynjMVVp4s742JW82P9jQZOw== +cornerstone-tools@^4.5.2: + version "4.5.2" + resolved "https://registry.yarnpkg.com/cornerstone-tools/-/cornerstone-tools-4.5.2.tgz#1eec8d8bb7e6674f16dcc054d477486b6b9477ee" + integrity sha512-FDRqt6JxcICCD0U34lpZ3uHYPaQr6jypAtyX0KL6GaMzC8X7cR0XE2Rm2bLT85oewyyQpHZC5tAFmr//noYxUQ== dependencies: "@babel/runtime" "7.1.2" cornerstone-math "0.1.7" @@ -13602,10 +13602,10 @@ vm-browserify@^1.0.1: resolved "https://registry.yarnpkg.com/vm-browserify/-/vm-browserify-1.1.0.tgz#bd76d6a23323e2ca8ffa12028dc04559c75f9019" integrity sha512-iq+S7vZJE60yejDYM0ek6zg308+UZsdtPExWP9VZoCFCz1zkJoXFnAX7aZfd/ZwrkidzdUZL0C/ryW+JwAiIGw== -vtk.js@^11.1.3: - version "11.2.0" - resolved "https://registry.yarnpkg.com/vtk.js/-/vtk.js-11.2.0.tgz#ce11863623f4f9ef64dfe6c5be1bb2fd2ed7ef6f" - integrity sha512-d++pvYLLc+/TGfNhGL/USZBwDoQGBV14UFOBLPZs4SjZGYdUEISFtG2FI4/MmoZOpzH/4vSoAlgbHAuCstYIQQ== +vtk.js@^11.2.0: + version "11.4.2" + resolved "https://registry.yarnpkg.com/vtk.js/-/vtk.js-11.4.2.tgz#6a967a4522a88704b6982a0d1356f24f77b09989" + integrity sha512-lr0GFZM5fGVUwM008KhxaSCB9dIeqKeD1ZZJQGcP+4rfcrPgC7Zp2L16kgQkj6rupPTseI+Nsk6UV5Dc8lJN4w== dependencies: blueimp-md5 "2.10.0" commander "2.11.0" From d7d0493bd4ec83507d6d12d4aa917a49db7a034f Mon Sep 17 00:00:00 2001 From: "James A. Petts" Date: Wed, 9 Oct 2019 14:33:01 +0100 Subject: [PATCH 2/3] =?UTF-8?q?perf:=20=E2=9A=A1=EF=B8=8F=20Ingest=20vtkjs?= =?UTF-8?q?=20strokeBuffer=20for=20performance=20enhancement?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- examples/VTKCornerstonePaintingSyncExample.js | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/examples/VTKCornerstonePaintingSyncExample.js b/examples/VTKCornerstonePaintingSyncExample.js index 76ac76f4..eda3cbe1 100644 --- a/examples/VTKCornerstonePaintingSyncExample.js +++ b/examples/VTKCornerstonePaintingSyncExample.js @@ -14,7 +14,7 @@ import vtkVolume from 'vtk.js/Sources/Rendering/Core/Volume'; const { EVENTS } = cornerstoneTools; window.cornerstoneTools = cornerstoneTools; -function setupSyncedBrush(imageDataObject, element) { +function setupSyncedBrush(imageDataObject) { // Create buffer the size of the 3D volume const dimensions = imageDataObject.dimensions; const width = dimensions[0]; @@ -128,10 +128,7 @@ class VTKCornerstonePaintingSyncExample extends Component { }; const imageDataObject = getImageData(imageIds, displaySetInstanceUid); - const labelMapInputData = setupSyncedBrush( - imageDataObject, - this.cornerstoneElements[0] - ); + const labelMapInputData = setupSyncedBrush(imageDataObject); this.onMeasurementsChanged = event => { if (event.type !== EVENTS.LABELMAP_MODIFIED) { From 0b225903360964b76a2c3942080cf2339f77fc0b Mon Sep 17 00:00:00 2001 From: "James A. Petts" Date: Thu, 10 Oct 2019 16:37:17 +0100 Subject: [PATCH 3/3] =?UTF-8?q?perf:=20=E2=9A=A1=EF=B8=8F=20Use=20Float32A?= =?UTF-8?q?rray=20and=20avoid=20duplicated=20slice=20insertion?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use a Float32Array and fill it using the cornerstone Int16 data to avoid recalculation. Also prevent multiple loadImageData calls on the same imageDataObject from overloading each other. --- examples/App.js | 9 ++ examples/VTKCornerstonePaintingSyncExample.js | 27 ++-- examples/VTKLoadImageDataExample.js | 140 ++++++++++++++++++ examples/VTKMPRRotateExample.js | 28 ---- src/lib/getImageData.js | 2 +- src/lib/loadImageData.js | 86 +++++++---- 6 files changed, 224 insertions(+), 68 deletions(-) create mode 100644 examples/VTKLoadImageDataExample.js diff --git a/examples/App.js b/examples/App.js index 064c18b3..22515abe 100644 --- a/examples/App.js +++ b/examples/App.js @@ -5,6 +5,7 @@ import VTKBasicExample from './VTKBasicExample.js'; import VTKFusionExample from './VTKFusionExample.js'; import VTKMPRPaintingExample from './VTKMPRPaintingExample.js'; import VTKCornerstonePaintingSyncExample from './VTKCornerstonePaintingSyncExample.js'; +import VTKLoadImageDataExample from './VTKLoadImageDataExample.js'; import VTKCrosshairsExample from './VTKCrosshairsExample.js'; import VTKMPRRotateExample from './VTKMPRRotateExample.js'; @@ -69,6 +70,12 @@ function Index() { url: '/rotate', text: 'Demonstrates how to set up the MPR Rotate interactor style', }, + { + title: 'LoadImageData Example', + url: '/cornerstone-load-image-data', + text: + 'Generating vtkjs imagedata from cornerstone images and displaying them in a VTK viewport.', + }, ]; const exampleComponents = examples.map(e => { @@ -125,6 +132,7 @@ function AppRouter() { const basic = () => Example({ children: }); const fusion = () => Example({ children: }); const painting = () => Example({ children: }); + const loadImage = () => Example({ children: }); const synced = () => Example({ children: }); const crosshairs = () => Example({ children: }); @@ -140,6 +148,7 @@ function AppRouter() { + diff --git a/examples/VTKCornerstonePaintingSyncExample.js b/examples/VTKCornerstonePaintingSyncExample.js index eda3cbe1..c80b2836 100644 --- a/examples/VTKCornerstonePaintingSyncExample.js +++ b/examples/VTKCornerstonePaintingSyncExample.js @@ -93,16 +93,6 @@ const imageIds = [ `dicomweb://${ROOT_URL}/PTCTStudy/1.3.6.1.4.1.25403.52237031786.3872.20100510032221.5.dcm`, ]; -// Pre-retrieve the images for demo purposes -// Note: In a real application you wouldn't need to do this -// since you would probably have the image metadata ahead of time. -// In this case, we preload the images so the WADO Image Loader can -// read and store all of their metadata and subsequently the 'getImageData' -// can run properly (it requires metadata). -const promises = imageIds.map(imageId => { - return cornerstone.loadAndCacheImage(imageId); -}); - class VTKCornerstonePaintingSyncExample extends Component { state = { volumes: null, @@ -116,6 +106,16 @@ class VTKCornerstonePaintingSyncExample extends Component { this.components = {}; this.cornerstoneElements = {}; + // Pre-retrieve the images for demo purposes + // Note: In a real application you wouldn't need to do this + // since you would probably have the image metadata ahead of time. + // In this case, we preload the images so the WADO Image Loader can + // read and store all of their metadata and subsequently the 'getImageData' + // can run properly (it requires metadata). + const promises = imageIds.map(imageId => { + return cornerstone.loadAndCacheImage(imageId); + }); + Promise.all(promises).then( () => { const displaySetInstanceUid = '12345'; @@ -257,10 +257,11 @@ class VTKCornerstonePaintingSyncExample extends Component { accessed in 2D.

- Both components are displaying the same labelmap UInt8Array. For + Both components are displaying the same labelmap UInt16Array. For VTK, it has been encapsulated in a vtkDataArray and then a - vtkImageData Object. For Cornerstone Tools, it is accessed by - reference and index for each of the 2D slices. + vtkImageData Object. For Cornerstone Tools, the Uint16Array is + accessed through helpers based on the actively displayed image stack + and the index of the currently displayed image

Note: The PaintWidget (circle on hover) is not diff --git a/examples/VTKLoadImageDataExample.js b/examples/VTKLoadImageDataExample.js new file mode 100644 index 00000000..61630518 --- /dev/null +++ b/examples/VTKLoadImageDataExample.js @@ -0,0 +1,140 @@ +import React from 'react'; +import { Component } from 'react'; + +import { View2D, getImageData, loadImageData } from '@vtk-viewport'; +import cornerstone from 'cornerstone-core'; +import cornerstoneTools from 'cornerstone-tools'; +import './initCornerstone.js'; +import vtkVolumeMapper from 'vtk.js/Sources/Rendering/Core/VolumeMapper'; +import vtkVolume from 'vtk.js/Sources/Rendering/Core/Volume'; + +window.cornerstoneTools = cornerstoneTools; + +function createActorMapper(imageData) { + const mapper = vtkVolumeMapper.newInstance(); + mapper.setInputData(imageData); + + const actor = vtkVolume.newInstance(); + actor.setMapper(mapper); + + return { + actor, + mapper, + }; +} + +const ROOT_URL = + window.location.hostname === 'localhost' + ? window.location.host + : window.location.hostname; + +const imageIds = [ + `dicomweb://${ROOT_URL}/PTCTStudy/1.3.6.1.4.1.25403.52237031786.3872.20100510032221.1.dcm`, + `dicomweb://${ROOT_URL}/PTCTStudy/1.3.6.1.4.1.25403.52237031786.3872.20100510032221.2.dcm`, + `dicomweb://${ROOT_URL}/PTCTStudy/1.3.6.1.4.1.25403.52237031786.3872.20100510032221.3.dcm`, + `dicomweb://${ROOT_URL}/PTCTStudy/1.3.6.1.4.1.25403.52237031786.3872.20100510032221.4.dcm`, + `dicomweb://${ROOT_URL}/PTCTStudy/1.3.6.1.4.1.25403.52237031786.3872.20100510032221.5.dcm`, +]; + +class VTKLoadImageDataExample extends Component { + state = { + volumes: null, + vtkImageData: null, + cornerstoneViewportData: null, + focusedWidgetId: null, + isSetup: false, + }; + + componentDidMount() { + this.components = {}; + this.cornerstoneElements = {}; + + // Pre-retrieve the images for demo purposes + // Note: In a real application you wouldn't need to do this + // since you would probably have the image metadata ahead of time. + // In this case, we preload the images so the WADO Image Loader can + // read and store all of their metadata and subsequently the 'getImageData' + // can run properly (it requires metadata). + const promises = imageIds.map(imageId => { + return cornerstone.loadAndCacheImage(imageId); + }); + + Promise.all(promises).then( + () => { + const displaySetInstanceUid = '12345'; + const cornerstoneViewportData = { + stack: { + imageIds, + currentImageIdIndex: 0, + }, + displaySetInstanceUid, + }; + + const imageDataObject = getImageData(imageIds, displaySetInstanceUid); + + loadImageData(imageDataObject).then(() => { + const { actor } = createActorMapper(imageDataObject.vtkImageData); + + this.setState({ + vtkImageData: imageDataObject.vtkImageData, + volumes: [actor], + cornerstoneViewportData, + }); + }); + }, + error => { + throw new Error(error); + } + ); + } + + saveCornerstoneElements = viewportIndex => { + return event => { + this.cornerstoneElements[viewportIndex] = event.detail.element; + }; + }; + + setWidget = event => { + const widgetId = event.target.value; + + if (widgetId === 'rotate') { + this.setState({ + focusedWidgetId: null, + }); + } + }; + + render() { + return ( +

+
+

Loading a cornerstone displayset into vtkjs

+

+ The example demonstrates loading cornerstone images already + available in the application into a vtkjs viewport. +

+
+
+
+
+ +
+
+ {this.state.volumes && } +
+
+
+ ); + } +} + +export default VTKLoadImageDataExample; diff --git a/examples/VTKMPRRotateExample.js b/examples/VTKMPRRotateExample.js index 20fade7f..1ccbda8f 100644 --- a/examples/VTKMPRRotateExample.js +++ b/examples/VTKMPRRotateExample.js @@ -434,34 +434,6 @@ class VTKMPRRotateExample extends Component { for (let index = 0; index < volumeData.length; index++) { columns.push(
- {/*
- { - this.handleChangeX(index, event); - }} - /> - {this.state.rotation[index].x} -
-
- { - this.handleChangeY(index, event); - }} - /> - {this.state.rotation[index].y} -
*/} { - const { imagePositionPatient } = metaDataMap.get(image.imageId); - const sliceIndex = getSliceIndex(zAxis, imagePositionPatient); +// TODO: If we attempt to load multiple imageDataObjects at once this will break. +export default function loadImageDataProgressively(imageDataObject) { + if (imageDataObject.loaded) { + // Returning instantly resolved promise as good to go. + return new Promise(resolve => { + resolve(); + }); + } else if (imageDataObject.isLoading) { + // Returning promise to be resolved by other process as loading. + return new Promise(resolve => { + resolveStack.push(resolve); + }); + } - insertSlice(imageData, sliceIndex, image); - }; + return new Promise((resolve, reject) => { + const { imageIds, vtkImageData, metaDataMap, zAxis } = imageDataObject; + const loadImagePromises = imageIds.map(cornerstone.loadAndCacheImage); - loadImagePromises.forEach(promise => { - promise.then(insertPixelData).catch(error => { - console.error(error); - //throw new Error(error); - }); - }); + imageDataObject.isLoading = true; + + let numberOfSlices = imageIds.length; + + let slicesInserted = 0; + + //let resolved = false; + + const insertPixelData = image => { + const { imagePositionPatient } = metaDataMap.get(image.imageId); + const sliceIndex = getSliceIndex(zAxis, imagePositionPatient); - // TODO: Investigate progressive loading. Right now the UI gets super slow because - // we are rendering and decoding simultaneously. We might want to use fewer web workers - // for the decoding tasks. + insertSlice(vtkImageData, sliceIndex, image); - //return loadImagePromises[0]; - return Promise.all(loadImagePromises); + slicesInserted++; + + //if (!resolved) { + if (slicesInserted === numberOfSlices) { + imageDataObject.isLoading = false; + imageDataObject.loaded = true; + console.log('LOADED'); + while (resolveStack.length) { + resolveStack.pop()(); + } + + //resolved = true; + + resolve(); + } + }; + + loadImagePromises.forEach(promise => { + promise.then(insertPixelData).catch(error => { + console.error(error); + reject(error); + }); + }); + + // TODO: Investigate progressive loading. Right now the UI gets super slow because + // we are rendering and decoding simultaneously. We might want to use fewer web workers + // for the decoding tasks. + }); } +/* export default function loadImageData(imageDataObject) { - return loadImageDataProgressively( - imageDataObject.imageIds, - imageDataObject.vtkImageData, - imageDataObject.metaDataMap, - imageDataObject.zAxis - ).then(() => { - imageDataObject.loaded = true; - }); + return loadImageDataProgressively(imageDataObject).then(); } +*/