Merge pull request #121 from Klowner/file-descriptor-exhaustion-issue

Fix file descriptor exhaustion when streaming many files

Author: phated

lib/src/getContents/streamFile.js

@@ -2,9 +2,14 @@
 
 var fs = require('graceful-fs');
 var stripBom = require('strip-bom-stream');
+var lazystream = require('lazystream');
 
 function streamFile(file, opt, cb) {
-  file.contents = fs.createReadStream(file.path);
+  var filePath = file.path;
+
+  file.contents = new lazystream.Readable(function() {
+    return fs.createReadStream(filePath);
+  });
 
   if (opt.stripBOM) {
     file.contents = file.contents.pipe(stripBom());

package.json

@@ -16,6 +16,7 @@
     "graceful-fs": "^4.0.0",
     "gulp-sourcemaps": "^1.5.2",
     "is-valid-glob": "^0.3.0",
+    "lazystream": "^1.0.0",
     "merge-stream": "^1.0.0",
     "mkdirp": "^0.5.0",
     "object-assign": "^4.0.0",

test/dest.js

@@ -19,6 +19,7 @@ var should = require('should');
 require('mocha');
 
 var wipeOut = function() {
+  this.timeout(20000);
   spies.setError('false');
   statSpy.reset();
   chmodSpy.reset();
@@ -1366,4 +1367,36 @@ describe('dest stream', function() {
       .once('finish', done);
   });
 
+  it('should not exhaust available file descriptors when streaming thousands of files', function(done) {
+    // This can be a very slow test on boxes with slow disk i/o
+    this.timeout(0);
+
+    // Make a ton of hard links
+    var numFiles = 6000;
+    var srcFile = path.join(__dirname, './fixtures/test.coffee');
+    fs.mkdirSync(path.join(__dirname, './out-fixtures'));
+    fs.mkdirSync(path.join(__dirname, './out-fixtures/in/'));
+
+    for (var idx = 0; idx < numFiles; idx++) {
+      fs.linkSync(srcFile, path.join(__dirname, './out-fixtures/in/test' + idx + '.coffee'));
+    }
+
+    var srcStream = vfs.src(path.join(__dirname, './out-fixtures/in/*.coffee'), { buffer: false });
+    var destStream = vfs.dest('./out-fixtures/out/', { cwd: __dirname });
+
+    var fileCount = 0;
+
+    srcStream
+      .pipe(through.obj(function(file, enc, cb) {
+        fileCount++;
+
+        cb(null, file);
+      }))
+      .pipe(destStream)
+      .once('finish', function() {
+        fileCount.should.equal(numFiles);
+        done();
+      });
+  });
+
 });