Replaced glob with readdir-glob to be memory efficient, and ensure re…

…addir stream is paused when archiving is on-going (#433)

benchmark/simple/pack-zip.js

@@ -3,7 +3,7 @@ var fs = require('fs');
 var mkdir = require('mkdirp');
 var streamBench = require('stream-bench');
 
-var archiver = require('../../lib/archiver');
+var archiver = require('../../');
 var common = require('../common');
 
 var binaryBuffer = common.binaryBuffer;
@@ -55,4 +55,4 @@ archive.pipe(bench);
 
 archive
   .file(file, { name: 'large file' })
-  .finalize();
+  .finalize();

lib/core.js

@@ -6,7 +6,7 @@
  * @copyright (c) 2012-2014 Chris Talkington, contributors.
  */
 var fs = require('fs');
-var glob = require('glob');
+var glob = require('readdir-glob');
 var async = require('async');
 var path = require('path');
 var util = require('archiver-utils');
@@ -389,13 +389,20 @@ Archiver.prototype._onQueueDrain = function() {
  * @return void
  */
 Archiver.prototype._onQueueTask = function(task, callback) {
-  if (this._state.finalizing || this._state.finalized || this._state.aborted) {
+  var fullCallback = () => {
+    if(task.data.callback) {
+      task.data.callback();
+    }
     callback();
+  }
+
+  if (this._state.finalizing || this._state.finalized || this._state.aborted) {
+    fullCallback();
     return;
   }
 
   this._task = task;
-  this._moduleAppend(task.source, task.data, callback);
+  this._moduleAppend(task.source, task.data, fullCallback);
 };
 
 /**
@@ -623,9 +630,8 @@ Archiver.prototype.directory = function(dirpath, destpath, data) {
   }
 
   var globOptions = {
-    stat: false,
-    dot: true,
-    cwd: dirpath
+    stat: true,
+    dot: true
   };
 
   function onGlobEnd() {
@@ -638,11 +644,14 @@ Archiver.prototype.directory = function(dirpath, destpath, data) {
   }
 
   function onGlobMatch(match){
+    globber.pause();
+
     var ignoreMatch = false;
     var entryData = Object.assign({}, data);
-    entryData.name = match;
+    entryData.name = match.relative;
     entryData.prefix = destpath;
-    match = globber._makeAbs(match);
+    entryData.stats = match.stat;
+    entryData.callback = globber.resume.bind(globber);
 
     try {
       if (dataFunction) {
@@ -660,13 +669,14 @@ Archiver.prototype.directory = function(dirpath, destpath, data) {
     }
 
     if (ignoreMatch) {
+      globber.resume();
       return;
     }
 
-    this._append(match, entryData);
+    this._append(match.absolute, entryData);
   }
 
-  var globber = glob('**', globOptions);
+  var globber = glob(dirpath, globOptions);
   globber.on('error', onGlobError.bind(this));
   globber.on('match', onGlobMatch.bind(this));
   globber.on('end', onGlobEnd.bind(this));
@@ -706,8 +716,8 @@ Archiver.prototype.file = function(filepath, data) {
 /**
  * Appends multiple files that match a glob pattern.
  *
- * @param  {String} pattern The [glob pattern]{@link https://github.com/isaacs/node-glob#glob-primer} to match.
- * @param  {Object} options See [node-glob]{@link https://github.com/isaacs/node-glob#options}.
+ * @param  {String} pattern The [glob pattern]{@link https://github.com/isaacs/minimatch} to match.
+ * @param  {Object} options See [node-glob]{@link https://github.com/yqnn/node-readdir-glob#options}.
  * @param  {EntryData} data See also [ZipEntryData]{@link ZipEntryData} and
  * [TarEntryData]{@link TarEntryData}.
  * @return {this}
@@ -716,7 +726,8 @@ Archiver.prototype.glob = function(pattern, options, data) {
   this._pending++;
 
   options = util.defaults(options, {
-    stat: false
+    stat: true,
+    pattern: pattern
   });
 
   function onGlobEnd() {
@@ -729,17 +740,16 @@ Archiver.prototype.glob = function(pattern, options, data) {
   }
 
   function onGlobMatch(match){
+    globber.pause();
     var entryData = Object.assign({}, data);
+    entryData.callback = globber.resume.bind(globber);
+    entryData.stats = match.stat;
+    entryData.name = match.relative;
 
-    if (options.cwd) {
-      entryData.name = match;
-      match = globber._makeAbs(match);
-    }
-
-    this._append(match, entryData);
+    this._append(match.absolute, entryData);
   }
 
-  var globber = glob(pattern, options);
+  var globber = glob(options.cwd || '.', options);
   globber.on('error', onGlobError.bind(this));
   globber.on('match', onGlobMatch.bind(this));
   globber.on('end', onGlobEnd.bind(this));

package.json

@@ -32,8 +32,8 @@
     "archiver-utils": "^2.1.0",
     "async": "^3.2.0",
     "buffer-crc32": "^0.2.1",
-    "glob": "^7.1.6",
     "readable-stream": "^3.6.0",
+    "readdir-glob": "^1.0.0",
     "tar-stream": "^2.1.2",
     "zip-stream": "^4.0.0"
   },