From 836122ef3fc0b1ce56e828ce4b45e6d2c6e95701 Mon Sep 17 00:00:00 2001 From: Colin Diesh Date: Thu, 20 Feb 2025 13:05:19 -0500 Subject: [PATCH] Add status callback while parsing .delta (MUMmer), .chain (liftover) and PIF files (#4856) --- .../src/ChainAdapter/util.ts | 11 ++- .../src/DeltaAdapter/util.ts | 11 ++- .../PairwiseIndexedPAFAdapter.ts | 82 ++++++++++--------- 3 files changed, 63 insertions(+), 41 deletions(-) diff --git a/plugins/comparative-adapters/src/ChainAdapter/util.ts b/plugins/comparative-adapters/src/ChainAdapter/util.ts index f717abe063..2dfb81e4db 100644 --- a/plugins/comparative-adapters/src/ChainAdapter/util.ts +++ b/plugins/comparative-adapters/src/ChainAdapter/util.ts @@ -23,6 +23,8 @@ * SOFTWARE. */ +import type { BaseOptions } from '@jbrowse/core/data_adapters/BaseAdapter' + function generate_record( qname: string, qstart: number, @@ -51,7 +53,8 @@ function generate_record( } } -export function paf_chain2paf(buffer: Uint8Array) { +export function paf_chain2paf(buffer: Uint8Array, opts?: BaseOptions) { + const { statusCallback = () => {} } = opts || {} let t_name = '' let t_start = 0 let t_end = 0 @@ -64,9 +67,15 @@ export function paf_chain2paf(buffer: Uint8Array) { let cigar = '' const records = [] + let i = 0 let blockStart = 0 const decoder = new TextDecoder('utf8') while (blockStart < buffer.length) { + if (i++ % 10_000 === 0) { + statusCallback( + `Loading ${Math.floor(blockStart / 1_000_000).toLocaleString('en-US')}/${Math.floor(buffer.length / 1_000_000).toLocaleString('en-US')} MB`, + ) + } const n = buffer.indexOf(10, blockStart) if (n === -1) { break diff --git a/plugins/comparative-adapters/src/DeltaAdapter/util.ts b/plugins/comparative-adapters/src/DeltaAdapter/util.ts index 9b802e54f0..f184c90982 100644 --- a/plugins/comparative-adapters/src/DeltaAdapter/util.ts +++ b/plugins/comparative-adapters/src/DeltaAdapter/util.ts @@ -1,3 +1,5 @@ +import type { BaseOptions } from '@jbrowse/core/data_adapters/BaseAdapter/BaseOptions' + /* paf2delta from paftools.js in the minimap2 repository, license reproduced below * * The MIT License @@ -26,7 +28,8 @@ * SOFTWARE. */ -export function paf_delta2paf(buffer: Uint8Array) { +export function paf_delta2paf(buffer: Uint8Array, opts?: BaseOptions) { + const { statusCallback = () => {} } = opts || {} let rname = '' let qname = '' let qs = 0 @@ -45,8 +48,14 @@ export function paf_delta2paf(buffer: Uint8Array) { let blockStart = 0 let i = 0 + let j = 0 const decoder = new TextDecoder('utf8') while (blockStart < buffer.length) { + if (j++ % 10_000 === 0) { + statusCallback( + `Loading ${Math.floor(blockStart / 1_000_000).toLocaleString('en-US')}/${Math.floor(buffer.length / 1_000_000).toLocaleString('en-US')} MB`, + ) + } const n = buffer.indexOf(10, blockStart) if (n === -1) { break diff --git a/plugins/comparative-adapters/src/PairwiseIndexedPAFAdapter/PairwiseIndexedPAFAdapter.ts b/plugins/comparative-adapters/src/PairwiseIndexedPAFAdapter/PairwiseIndexedPAFAdapter.ts index 12f0dd76e8..c9b33dae25 100644 --- a/plugins/comparative-adapters/src/PairwiseIndexedPAFAdapter/PairwiseIndexedPAFAdapter.ts +++ b/plugins/comparative-adapters/src/PairwiseIndexedPAFAdapter/PairwiseIndexedPAFAdapter.ts @@ -1,5 +1,6 @@ import { TabixIndexedFile } from '@gmod/tabix' import { BaseFeatureDataAdapter } from '@jbrowse/core/data_adapters/BaseAdapter' +import { updateStatus } from '@jbrowse/core/util' import { openLocation } from '@jbrowse/core/util/io' import { ObservableCreate } from '@jbrowse/core/util/rxjs' @@ -77,6 +78,7 @@ export default class PAFAdapter extends BaseFeatureDataAdapter { } getFeatures(query: Region, opts: PAFOptions = {}) { + const { statusCallback = () => {} } = opts return ObservableCreate(async observer => { const { assemblyName } = query @@ -85,45 +87,47 @@ export default class PAFAdapter extends BaseFeatureDataAdapter { const flip = index === 0 const letter = flip ? 'q' : 't' - await this.pif.getLines(letter + query.refName, query.start, query.end, { - lineCallback: (line, fileOffset) => { - const r = parsePAFLine(line) - const refName = r.qname.slice(1) - const start = r.qstart - const end = r.qend - const mateName = r.tname - const mateStart = r.tstart - const mateEnd = r.tend - - const { extra, strand } = r - const { numMatches = 0, blockLen = 1, cg, ...rest } = extra - - observer.next( - new SyntenyFeature({ - uniqueId: fileOffset + assemblyName, - assemblyName, - start, - end, - type: 'match', - refName, - strand, - ...rest, - CIGAR: extra.cg, - syntenyId: fileOffset, - identity: numMatches / blockLen, - numMatches, - blockLen, - mate: { - start: mateStart, - end: mateEnd, - refName: mateName, - assemblyName: assemblyNames[+flip], - }, - }), - ) - }, - stopToken: opts.stopToken, - }) + await updateStatus('Downloading features', statusCallback, () => + this.pif.getLines(letter + query.refName, query.start, query.end, { + lineCallback: (line, fileOffset) => { + const r = parsePAFLine(line) + const refName = r.qname.slice(1) + const start = r.qstart + const end = r.qend + const mateName = r.tname + const mateStart = r.tstart + const mateEnd = r.tend + + const { extra, strand } = r + const { numMatches = 0, blockLen = 1, cg, ...rest } = extra + + observer.next( + new SyntenyFeature({ + uniqueId: fileOffset + assemblyName, + assemblyName, + start, + end, + type: 'match', + refName, + strand, + ...rest, + CIGAR: extra.cg, + syntenyId: fileOffset, + identity: numMatches / blockLen, + numMatches, + blockLen, + mate: { + start: mateStart, + end: mateEnd, + refName: mateName, + assemblyName: assemblyNames[+flip], + }, + }), + ) + }, + stopToken: opts.stopToken, + }), + ) observer.complete() })