Add status callback for parsing PAF, BLAST tabular, MashMap files (#4854

)

package.json

@@ -100,7 +100,7 @@
     "dependency-graph": "^1.0.0",
     "dotenv": "^16.3.1",
     "dotenv-expand": "^11.0.3",
-    "electron": "34.0.1",
+    "electron": "34.2.0",
     "electron-builder": "^25.1.6",
     "electron-mock-ipc": "^0.3.8",
     "eslint": "^9.17.0",
@@ -109,11 +109,11 @@
     "eslint-plugin-react-compiler": "^19.0.0-beta-6fc168f-20241025",
     "eslint-plugin-react-hooks": "^5.0.0",
     "eslint-plugin-react-refresh": "^0.4.3",
-    "eslint-plugin-unicorn": "^56.0.0",
+    "eslint-plugin-unicorn": "^57.0.0",
     "express": "^4.0.0",
     "express-basic-auth": "^1.2.1",
     "find-yarn-workspace-root": "^2.0.0",
-    "globals": "^15.9.0",
+    "globals": "^16.0.0",
     "html-webpack-plugin": "^5.5.0",
     "husky": "^9.0.10",
     "jest": "^29.0.0",

packages/core/util/index.ts

@@ -1164,6 +1164,7 @@ export function useLocalStorage<T>(key: string, initialValue: T) {
   const setValue = (value: T | ((val: T) => T)) => {
     try {
       const valueToStore =
+        // eslint-disable-next-line unicorn/no-instanceof-builtins
         value instanceof Function ? value(storedValue) : value
       setStoredValue(valueToStore)
       if (typeof window !== 'undefined') {

plugins/comparative-adapters/src/BlastTabularAdapter/BlastTabularAdapter.ts

@@ -216,7 +216,7 @@ export default class BlastTabularAdapter extends BaseFeatureDataAdapter {
       opts,
     )
     const columns: string = readConfObject(this.config, 'columns')
-    return parseLineByLine(buf, createBlastLineParser(columns))
+    return parseLineByLine(buf, createBlastLineParser(columns), opts)
   }
 
   async hasDataForRefName() {

plugins/comparative-adapters/src/MashMapAdapter/MashMapAdapter.ts

@@ -10,7 +10,7 @@ export default class MashMapAdapter extends PAFAdapter {
   async setupPre(opts?: BaseOptions) {
     const outLoc = openLocation(this.getConf('outLocation'), this.pluginManager)
     const buf = await fetchAndMaybeUnzip(outLoc, opts)
-    return parseLineByLine(buf, parseMashMapLine)
+    return parseLineByLine(buf, parseMashMapLine, opts)
   }
 }
 

plugins/comparative-adapters/src/PAFAdapter/PAFAdapter.ts

@@ -46,7 +46,7 @@ export default class PAFAdapter extends BaseFeatureDataAdapter {
     const pm = this.pluginManager
     const pafLocation = openLocation(this.getConf('pafLocation'), pm)
     const buf = await fetchAndMaybeUnzip(pafLocation, opts)
-    return parseLineByLine(buf, parsePAFLine)
+    return parseLineByLine(buf, parsePAFLine, opts)
   }
 
   async hasDataForRefName() {

plugins/comparative-adapters/src/util.ts

@@ -37,10 +37,14 @@ export function zip(a: number[], b: number[]) {
 export function parseLineByLine<T>(
   buffer: Uint8Array,
   cb: (line: string) => T | undefined,
+  opts?: BaseOptions,
 ): T[] {
+  const { statusCallback = () => {} } = opts || {}
   let blockStart = 0
   const entries: T[] = []
   const decoder = new TextDecoder('utf8')
+
+  let i = 0
   while (blockStart < buffer.length) {
     const n = buffer.indexOf(10, blockStart)
     if (n === -1) {
@@ -54,52 +58,42 @@ export function parseLineByLine<T>(
         entries.push(entry)
       }
     }
-
+    if (i++ % 10_000 === 0) {
+      statusCallback(
+        `Loading ${Math.floor(blockStart / 1_000_000).toLocaleString('en-US')}/${Math.floor(buffer.length / 1_000_000).toLocaleString('en-US')} MB`,
+      )
+    }
     blockStart = n + 1
   }
   return entries
 }
 
 export function parsePAFLine(line: string) {
-  const [
-    qname,
-    ,
-    qstart,
-    qend,
-    strand,
-    tname,
-    ,
-    tstart,
-    tend,
-    numMatches,
-    blockLen,
-    mappingQual,
-    ...fields
-  ] = line.split('\t')
+  const parts = line.split('\t')
+  const extraFields = parts.slice(12)
+  const extra: Record<string, string | number> = {
+    numMatches: +parts[9]!,
+    blockLen: +parts[10]!,
+    mappingQual: +parts[11]!,
+  }
 
-  const rest = Object.fromEntries(
-    fields.map(field => {
-      const r = field.indexOf(':')
-      const fieldName = field.slice(0, r)
-      const fieldValue = field.slice(r + 3)
-      return [fieldName, fieldValue]
-    }),
-  )
+  // Process extra fields only if they exist
+  if (extraFields.length) {
+    for (const field of extraFields) {
+      const colonIndex = field.indexOf(':')
+      extra[field.slice(0, colonIndex)] = field.slice(colonIndex + 3)
+    }
+  }
 
   return {
-    tname,
-    tstart: +tstart!,
-    tend: +tend!,
-    qname,
-    qstart: +qstart!,
-    qend: +qend!,
-    strand: strand === '-' ? -1 : 1,
-    extra: {
-      numMatches: +numMatches!,
-      blockLen: +blockLen!,
-      mappingQual: +mappingQual!,
-      ...rest,
-    },
+    tname: parts[5],
+    tstart: +parts[7]!,
+    tend: +parts[8]!,
+    qname: parts[0],
+    qstart: +parts[2]!,
+    qend: +parts[3]!,
+    strand: parts[4] === '-' ? -1 : 1,
+    extra,
   } as PAFRecord
 }
 

plugins/legacy-jbrowse/src/JBrowse1Connection/util.ts

@@ -80,7 +80,7 @@ export function structuredClone(src: any): any {
   }
   if (src instanceof Date) {
     // Date
-    return new Date(src.getTime()) // Date
+    return new Date(src) // Date
   }
   if (src instanceof RegExp) {
     // RegExp

plugins/sequence/src/LinearReferenceSequenceDisplay/configSchema.ts

@@ -1,6 +1,6 @@
 import { ConfigurationSchema } from '@jbrowse/core/configuration'
 
-import { default as divSequenceRendererConfigSchema } from '../DivSequenceRenderer/configSchema'
+import divSequenceRendererConfigSchema from '../DivSequenceRenderer/configSchema'
 
 /**
  * #config LinearReferenceSequenceDisplay

products/jbrowse-desktop/package.json

@@ -101,13 +101,13 @@
     "use-query-params": "^2.0.0"
   },
   "devDependencies": {
-    "electron": "34.0.1"
+    "electron": "34.2.0"
   },
   "browserslist": [
     "last 1 chrome version"
   ],
   "build": {
-    "electronVersion": "34.0.1",
+    "electronVersion": "34.2.0",
     "extraMetadata": {
       "main": "build/electron.js"
     },