-
Notifications
You must be signed in to change notification settings - Fork 31
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
fix: Zip CSV uploads not working (#1457)
Fixes #1080. Fixes #1416 Updates jszip and uses the internal stream helper instead of nodestream which we would need a polyfill for. The progress on zip uploads is a bit odd since we use the unzip progress. It seems papaparse loads more from the stream while processing/before processing and can finish reading the zip before it's done processing chunks. Also, uploading the tables seems to be a blocking operation. Not sure if that's an easy fix, but after the 50% mark on uploading a large zip, there are some blocks on the main thread (marching ants freezes as an indicator). There is a ~2GB limit for JSZip it seems Stuk/jszip#777
- Loading branch information
1 parent
6ff27a6
commit 08d0296
Showing
7 changed files
with
103 additions
and
65 deletions.
There are no files selected for viewing
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,33 @@ | ||
import type { JSZipObject, OnUpdateCallback, JSZipStreamHelper } from 'jszip'; | ||
|
||
/** | ||
* This is used to help papaparse understand our stream. | ||
* It uses these fields for feature detection, but never actually calls read() | ||
* https://github.com/mholt/PapaParse/blob/master/papaparse.js#L244 | ||
*/ | ||
interface ZipStreamHelper extends JSZipStreamHelper<string> { | ||
readable: boolean; | ||
read(): void; | ||
removeListener(): void; | ||
} | ||
|
||
export default function makeZipStreamHelper( | ||
zipObj: JSZipObject, | ||
onUpdate: OnUpdateCallback | ||
) { | ||
const helper: ZipStreamHelper = ( | ||
zipObj as JSZipObject & { | ||
// The type could be anything except nodebuffer from https://stuk.github.io/jszip/documentation/api_zipobject/internal_stream.html | ||
// We only need it as a string though | ||
// JSZip types don't include this method for some reason | ||
internalStream(type: 'string'): JSZipStreamHelper<string>; | ||
} | ||
).internalStream('string') as ZipStreamHelper; | ||
|
||
helper.readable = true; | ||
helper.read = () => false; | ||
helper.removeListener = () => false; | ||
helper.on('data', (_, metadata) => onUpdate(metadata)); | ||
|
||
return helper; | ||
} |