Skip to content

Commit

Permalink
feat: read metadata file instead of mongodb metadata data
Browse files Browse the repository at this point in the history 
1. mongodb maximum BSON document size is 16 megabytes,
the metadata that may more than 16mb, so store file instead of storing data

feat: replace binary data after dicom to json

original->
replace binary before instert metadata
why->
When processing json, binary data may cause process slowly.
  • Loading branch information
@Chinlinlee
Chinlinlee committed Aug 20, 2021
1 parent 7faaedb commit e690008
Show file tree
Hide file tree
Showing 4 changed files with 201 additions and 72 deletions.
170 changes: 163 additions & 7 deletions api/dicom-web/controller/metadata.js
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
const mongodb = require('models/mongodb');
const _ =require('lodash');
const mongoFunc = require('../../../models/mongodb/func');

const fs = require('fs');
const path = require('path');

const returnProject = {
$project : {
Expand All @@ -14,7 +15,7 @@ const returnProject = {
};

module.exports = async function (req , res ) {
let getMetaFunc = [getStudyMetadata , getSeriesMetadata , getInstanceMetadata];
let getMetaFunc = [getStudyInstanceStorePath, getSeriesInstanceStorePath , getInstanceStorePath];
let errorMessage = {
message : "cannot found"
}
Expand All @@ -24,16 +25,23 @@ module.exports = async function (req , res ) {
errorMessage.message = `${errorMessage.message} ${keys}:${req.params[i]}`
}
if (item.length <= 0) return res.status(404).json(errorMessage);
if (item[0].metadata) {
if (item[0].metadata.length > 0) {
res.setHeader('Content-Type' , 'application/dicom+json');
return res.send(item[0].metadata);
if (item) {
let resMetadata = [];
for (let instance of item) {
let fullPath = path.join(process.env.DICOM_STORE_ROOTPATH, instance.path);
let metadataStorePath = path.join(path.dirname(fullPath), `${instance.instanceUID}.metadata.json`);
let metadataJsonStr = fs.readFileSync(metadataStorePath , {encoding: 'utf8'});
let metadataJson = JSON.parse(metadataJsonStr);
resMetadata.push(metadataJson);
}
res.setHeader('Content-Type' , 'application/dicom+json');
return res.send(resMetadata);
}
res.setHeader('Content-Type' , 'application/dicom+json');
return res.status(404).json(errorMessage);
}


async function getStudyMetadata (params) {
const metadataQuery = [
{
Expand Down Expand Up @@ -86,7 +94,6 @@ async function getSeriesMetadata (params) {
}
}
]

let agg = await mongoFunc.aggregate_Func("dicomMetadata" , metadataQuery)
return agg;
}
Expand Down Expand Up @@ -121,4 +128,153 @@ async function getInstanceMetadata (params) {
]
let agg = await mongoFunc.aggregate_Func("dicomMetadata" , metadataQuery)
return agg;
}



async function getStudyInstanceStorePath(params) {
let aggregate_Query =
[
{
$match: {
'dicomJson.0020000D.Value': params.studyID,
}
},
{
$unwind: '$series'
},
{
$unwind: '$series.instance'
},
{
$group:
{
_id: null,
instanceStorePathList:
{
$push:
{
path: "$series.instance.store_path",
instanceUID: "$series.instance.uid"
}
}
}
}
]
let instances = await find_Aggregate_Func('ImagingStudy', aggregate_Query);
if (instances[0].instanceStorePathList.length <= 0) return false;
try {
return (instances[0].instanceStorePathList);
} catch (e) {
console.log("getInstancePath error\r\n" + JSON.stringify(aggregate_Query, null, 4));
//console.log(aggregate_Query);
return false;
}

}

async function getSeriesInstanceStorePath(params) {
let aggregate_Query =
[
{
$match: {
'dicomJson.0020000D.Value': params.studyID,
}
},
{
$unwind: '$series'
},
{
$match:
{
'series.dicomJson.0020000E.Value': params.seriesID,
}
},
{
$unwind: '$series.instance'
},
{
$group:
{
_id: null,
instanceStorePathList:
{
$push:
{
path: "$series.instance.store_path",
instanceUID: "$series.instance.uid"
}
}
}
}
]
let instances = await find_Aggregate_Func('ImagingStudy', aggregate_Query);
if (instances[0].instanceStorePathList.length <= 0) return false;
try {
return (instances[0].instanceStorePathList);
} catch (e) {
console.log("getInstancePath error\r\n" + JSON.stringify(aggregate_Query, null, 4));
//console.log(aggregate_Query);
return false;
}
}

async function getInstanceStorePath(params) {
let aggregate_Query =
[
{
$match: {
'dicomJson.0020000D.Value': params.studyID,
}
},
{
$unwind: '$series'
},
{
$match:
{
'series.dicomJson.0020000E.Value': params.seriesID,
}
},
{
$match:
{
'series.instance.dicomJson.00080018.Value': params.instanceID
}
},
{
$unwind: '$series.instance'
},
{
$group:
{
_id: null,
instanceStorePathList:
{
$push:
{
path: "$series.instance.store_path",
instanceUID: "$series.instance.uid"
}
}
}
}
]
let instances = await find_Aggregate_Func('ImagingStudy', aggregate_Query);
if (instances[0].instanceStorePathList.length <= 0) return false;
try {
return (instances[0].instanceStorePathList);
} catch (e) {
console.log("getInstancePath error\r\n" + JSON.stringify(aggregate_Query, null, 4));
//console.log(aggregate_Query);
return false;
}
}

async function find_Aggregate_Func(collection_Name, i_Query) {
return new Promise(async (resolve, reject) => {
let agg = await mongodb[collection_Name].aggregate(
i_Query);
return resolve(agg);
});
}
97 changes: 32 additions & 65 deletions api/dicom-web/stow/controller/postSTOW.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ const { storeImagingStudy } = require('../../../FHIR/ImagingStudy/controller/pos
const { getDeepKeys } = require('../../../Api_function');
const mkdirp = require('mkdirp');
const notImageSOPClass = require('../../../../models/DICOMWeb/notImageSOPClass');
const flat = require('flat');

//browserify
//https://github.com/node-formidable/formidable/blob/6baefeec3df6f38e34c018c9e978329ae68b4c78/src/Formidable.js#L496
Expand Down Expand Up @@ -82,7 +83,6 @@ async function dicomPatient2MongoDB(data) {

async function generateJpeg(dicomJson, dicomFile, jpegFile) {
try {
console.time("generate jpeg");
let studyUID = _.get(dicomJson, '0020000D.Value.0');
let seriesUID = _.get(dicomJson, '0020000E.Value.0');
let instanceUID = _.get(dicomJson, '00080018.Value.0');
Expand Down Expand Up @@ -121,12 +121,10 @@ async function generateJpeg(dicomJson, dicomFile, jpegFile) {
execCmdList = new Array();
}
}
console.timeEnd("generate jpeg");
} else {
for (let i = 1; i <= frameNumber; i++) {
await getJpeg[process.env.ENV].getJpegByPydicom(dicomFile, i);
}
console.timeEnd("generate jpeg");
}
await insertDicomToJpegTask({
studyUID: studyUID,
Expand Down Expand Up @@ -167,15 +165,23 @@ async function generateJpeg(dicomJson, dicomFile, jpegFile) {
async function convertDICOMFileToJSONModule(filename) {
try {
let dicomJson = await dcm2jsonV8.exec(filename);
let perFrameFunctionalGroupSQ = _.get(dicomJson, "52009230");
let tempPerFrameFunctionalGroupSQ = "";
if (perFrameFunctionalGroupSQ) {
tempPerFrameFunctionalGroupSQ = _.cloneDeep(perFrameFunctionalGroupSQ);
dicomJson = _.omit(dicomJson, ["52009230"]);
perFrameFunctionalGroupSQ = undefined;
flat(dicomJson);
let bigValueTags = ["52009230", "00480200"];
let tempBigTagValue = {};
for (let bigValueTag of bigValueTags) {
let bigValue = _.get(dicomJson, bigValueTag);
if (bigValue) {
_.set(tempBigTagValue, `${bigValueTag}`, _.cloneDeep(bigValue));
} else {
_.set(tempBigTagValue, `${bigValueTag}`, undefined);
}
bigValue = undefined;
}
dicomJson = _.omit(dicomJson, bigValueTags);
dicomJson = await replaceBinaryData(dicomJson);
let started_date = "";
started_date = dcm2jsonV8.dcmString(dicomJson, '00080020') + dcm2jsonV8.dcmString(dicomJson, '00080030');
if (!started_date) started_date = Date.now();
started_date = moment(started_date, "YYYYMMDDhhmmss").toISOString();
let started_date_split = started_date.split('-');
let year = started_date_split[0];
Expand All @@ -184,15 +190,17 @@ async function convertDICOMFileToJSONModule(filename) {
let shortUID = sh.unique(uid);
let relativeStorePath = `files/${year}/${month}/${shortUID}/`;
let fullStorePath = path.join(process.env.DICOM_STORE_ROOTPATH, relativeStorePath);

let instanceUID = dcm2jsonV8.dcmString(dicomJson, '00080018')
let instanceUID = dcm2jsonV8.dcmString(dicomJson, '00080018');
let metadataFullStorePath = path.join(fullStorePath, `${instanceUID}.metadata.json`);
if (tempPerFrameFunctionalGroupSQ) {
_.set(dicomJson, "52009230", tempPerFrameFunctionalGroupSQ);

for (let keys in tempBigTagValue) {
if (tempBigTagValue[keys]) {
_.set(dicomJson, keys, tempBigTagValue[keys]);
}
}
mkdirp.sync(fullStorePath,0755);
mkdirp.sync(fullStorePath, 0755);
fs.writeFileSync(metadataFullStorePath, JSON.stringify(dicomJson, null, 4));
dicomJson = _.omit(dicomJson, ["52009230"]);
dicomJson = _.omit(dicomJson, bigValueTags);
return {
status: true,
storePath: relativeStorePath,
Expand Down Expand Up @@ -247,55 +255,14 @@ async function saveDICOMFile(tempFilename, filename, dest) {
}
}

async function saveUploadDicom(tempFilename, filename) {
return new Promise(async (resolve, reject) => {
try {
let maxSize = 500 * 1024 * 1024;
let fileSize = fs.statSync(tempFilename).size;
let fhirData = "";
let dcmJson = "";
dcmJson = await dcm2jsonV8.exec(tempFilename);
dcmJson = _.omit(dcmJson, ["52009230"]);
if (_.isUndefined(newStoredFilename)) {
return resolve(false);
}
if (fileSize > maxSize) {
if (_.isString(tempFilename)) {
fhirData = await FHIR_Imagingstudy_model.DCMJson2FHIR(dcmJson);
}
} else {
fhirData = await FHIR_Imagingstudy_model.DCM2FHIR(tempFilename).catch((err) => {
console.error(err);
fs.unlinkSync(tempFilename);
return resolve(false);
});
}
if (!fhirData) {
fs.unlinkSync(tempFilename);
return resolve(false);
}
let newStoredFilename = await saveDICOMFile(fhirData, tempFilename, filename);
if (_.isUndefined(newStoredFilename)) {
return resolve(false);
}
fhirData = await getFHIRIntegrateDICOMJson(dcmJson, newStoredFilename, fhirData);
return resolve(fhirData);
} catch(e) {
console.error(e)
resolve(false);
}
});
}

async function replaceBinaryData(data) {
try {
let keys = getDeepKeys(data);
let binaryKeys = [];
for (let key of keys) {
let flatDicomJson = flat(data);
for (let key in flatDicomJson) {
if (key.includes("7FE00010")) continue;
let keyData = _.get(data, key);
if (keyData == "OW" || keyData == "OB") {
binaryKeys.push(key.substring(0 , key.lastIndexOf(".vr")));
if (flatDicomJson[key] == "OW" || flatDicomJson[key] == "OB") {
binaryKeys.push(key.substring(0, key.lastIndexOf(".vr")));
}
}
let port = process.env.DICOMWEB_PORT || "";
Expand Down Expand Up @@ -342,11 +309,12 @@ async function replaceBinaryData(data) {
}, bulkData , {
upsert: true
});



}

data["7FE00010"] = {
"vr": "OW",
"BulkDataURI": `http://${process.env.DICOMWEB_HOST}${port}/${process.env.DICOMWEB_API}/studies/${data['0020000D'].Value[0]}/series/${data['0020000E'].Value[0]}/instances/${data['00080018'].Value[0]}`
}
return data;
} catch(e) {
console.error(e);
throw e;
Expand All @@ -356,7 +324,6 @@ async function replaceBinaryData(data) {
function insertMetadata(metadata) {
return new Promise(async (resolve) => {
try {
await replaceBinaryData(metadata);
await mongodb.dicomMetadata.updateOne({
'studyUID': metadata.studyUID,
'seriesUID': metadata.seriesUID,
Expand Down
5 changes: 5 additions & 0 deletions package-lock.json
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
"express-session": "^1.16.2",
"express-validator": "^5.3.0",
"fhir": "^4.9.0",
"flat": "^5.0.2",
"formidable": "^1.2.2",
"glob": "^7.1.6",
"iconv-lite": "^0.6.2",
Expand Down

0 comments on commit e690008

Please sign in to comment.