fix(assets): support exceptions to exclude patterns (#4473)

* chore: better test * fix: folder exclusion * additional test cases * refactor existing tests * chore: add new test files * fix: listFilesRecursively refactor (wip) * fix: finish refactoring listFilesRecursively * fix: implement mkdirpSync * fix: symlink discovery * fix: don't follow symlinks early * fix: create empty directories * chore: remove useless let * fix: symlink fingerprinting * fix: don't throw when fingerprinting directories * chore: remove unneeded 'exclude' cloning * chore: refactor mkdirp calls * chore: refactor AssetFile * chore: refactor recursion * chore: prevent unneeded stats call * chore: createFsStructureFromTree, remove empty files * chore: cleanup * fix: empty-directory test * feat: shouldExcludeDeep * chore: fromTree in @/assert, cleanup fn, test * feat: shouldExcludeDirectory * chore: refactor listFiles with new methods (missing symlinks) * feat: add symlink support to fromTree * fix: fromTree external directory symlink * fix: listFiles symlink support * chore: additional contridactory test * chore: fix docs * chore: ExcludeRules class refactor * chore: evaluateFile refactor * chore: further evaluateFile refactor * chore: evaluateDirectory refactor * chore: move FsUtils to assets * chore: move FsUtils to assets (unstaged files)
aws · Oct 30, 2019 · b7b4336 · b7b4336
1 parent 004dfd4
commit b7b4336
Show file tree

Hide file tree

Showing 14 changed files with 1,109 additions and 130 deletions.
diff --git a/packages/@aws-cdk/assets/lib/fs/copy.ts b/packages/@aws-cdk/assets/lib/fs/copy.ts
@@ -2,56 +2,38 @@ import fs = require('fs');
 import path = require('path');
 import { CopyOptions } from './copy-options';
 import { FollowMode } from './follow-mode';
-import { shouldExclude, shouldFollow } from './utils';
+import { mkdirpSync } from './mkdirpSync';
+import { listFilesRecursively, shouldFollow } from './utils';
 
 export function copyDirectory(srcDir: string, destDir: string, options: CopyOptions = { }, rootDir?: string) {
   const follow = options.follow !== undefined ? options.follow : FollowMode.EXTERNAL;
-  const exclude = options.exclude || [];
 
   rootDir = rootDir || srcDir;
 
   if (!fs.statSync(srcDir).isDirectory()) {
     throw new Error(`${srcDir} is not a directory`);
   }
 
-  const files = fs.readdirSync(srcDir);
-  for (const file of files) {
-    const sourceFilePath = path.join(srcDir, file);
+  for (const assetFile of listFilesRecursively(srcDir, {...options, follow}, rootDir)) {
+    const filePath = assetFile.relativePath;
+    const destFilePath = path.join(destDir, filePath);
 
-    if (shouldExclude(exclude, path.relative(rootDir, sourceFilePath))) {
-      continue;
-    }
-
-    const destFilePath = path.join(destDir, file);
-
-    let stat: fs.Stats | undefined = follow === FollowMode.ALWAYS
-      ? fs.statSync(sourceFilePath)
-      : fs.lstatSync(sourceFilePath);
-
-    if (stat && stat.isSymbolicLink()) {
-      const target = fs.readlinkSync(sourceFilePath);
+    if (follow !== FollowMode.ALWAYS) {
+      if (assetFile.isSymbolicLink) {
+        const targetPath = path.normalize(path.resolve(srcDir, assetFile.symlinkTarget));
+        if (!shouldFollow(follow, rootDir, targetPath)) {
+          fs.symlinkSync(assetFile.symlinkTarget, destFilePath);
 
-      // determine if this is an external link (i.e. the target's absolute path
-      // is outside of the root directory).
-      const targetPath = path.normalize(path.resolve(srcDir, target));
-
-      if (shouldFollow(follow, rootDir, targetPath)) {
-        stat = fs.statSync(sourceFilePath);
-      } else {
-        fs.symlinkSync(target, destFilePath);
-        stat = undefined;
+          continue;
+        }
       }
     }
 
-    if (stat && stat.isDirectory()) {
-      fs.mkdirSync(destFilePath);
-      copyDirectory(sourceFilePath, destFilePath, options, rootDir);
-      stat = undefined;
-    }
-
-    if (stat && stat.isFile()) {
-      fs.copyFileSync(sourceFilePath, destFilePath);
-      stat = undefined;
+    if (!assetFile.isDirectory) {
+      mkdirpSync(path.dirname(destFilePath));
+      fs.copyFileSync(assetFile.absolutePath, destFilePath);
+    } else {
+      mkdirpSync(destFilePath);
     }
   }
 }
diff --git a/packages/@aws-cdk/assets/lib/fs/fingerprint.ts b/packages/@aws-cdk/assets/lib/fs/fingerprint.ts
@@ -3,7 +3,7 @@ import fs = require('fs');
 import path = require('path');
 import { CopyOptions } from './copy-options';
 import { FollowMode } from './follow-mode';
-import { shouldExclude, shouldFollow } from './utils';
+import { listFilesRecursively, shouldFollow } from './utils';
 
 const BUFFER_SIZE = 8 * 1024;
 const CTRL_SOH = '\x01';
@@ -38,40 +38,28 @@ export function fingerprint(fileOrDirectory: string, options: FingerprintOptions
   const rootDirectory = fs.statSync(fileOrDirectory).isDirectory()
     ? fileOrDirectory
     : path.dirname(fileOrDirectory);
-  const exclude = options.exclude || [];
-  _processFileOrDirectory(fileOrDirectory);
 
-  return hash.digest('hex');
-
-  function _processFileOrDirectory(symbolicPath: string, realPath = symbolicPath) {
-    if (shouldExclude(exclude, symbolicPath)) {
-      return;
-    }
-
-    const stat = fs.lstatSync(realPath);
-    const relativePath = path.relative(fileOrDirectory, symbolicPath);
+  for (const assetFile of listFilesRecursively(fileOrDirectory, {...options, follow}, rootDirectory)) {
+    const relativePath = path.relative(fileOrDirectory, assetFile.absolutePath);
 
-    if (stat.isSymbolicLink()) {
-      const linkTarget = fs.readlinkSync(realPath);
-      const resolvedLinkTarget = path.resolve(path.dirname(realPath), linkTarget);
-      if (shouldFollow(follow, rootDirectory, resolvedLinkTarget)) {
-        _processFileOrDirectory(symbolicPath, resolvedLinkTarget);
+    if (assetFile.isSymbolicLink) {
+      const resolvedLinkTarget = path.resolve(path.dirname(assetFile.absolutePath), assetFile.symlinkTarget);
+      if (!shouldFollow(follow, rootDirectory, resolvedLinkTarget)) {
+        _hashField(hash, `link:${relativePath}`, assetFile.symlinkTarget);
       } else {
-        _hashField(hash, `link:${relativePath}`, linkTarget);
+        _hashField(hash, `file:${relativePath}`, _contentFingerprint(assetFile.absolutePath, assetFile.size));
       }
-    } else if (stat.isFile()) {
-      _hashField(hash, `file:${relativePath}`, _contentFingerprint(realPath, stat));
-    } else if (stat.isDirectory()) {
-      for (const item of fs.readdirSync(realPath).sort()) {
-        _processFileOrDirectory(path.join(symbolicPath, item), path.join(realPath, item));
-      }
-    } else {
-      throw new Error(`Unable to hash ${symbolicPath}: it is neither a file nor a directory`);
+    } else if (assetFile.isFile) {
+      _hashField(hash, `file:${relativePath}`, _contentFingerprint(assetFile.absolutePath, assetFile.size));
+    } else if (!assetFile.isDirectory) {
+      throw new Error(`Unable to hash ${assetFile.absolutePath}: it is neither a file nor a directory`);
     }
   }
+
+  return hash.digest('hex');
 }
 
-function _contentFingerprint(file: string, stat: fs.Stats): string {
+function _contentFingerprint(file: string, size: number): string {
   const hash = crypto.createHash('sha256');
   const buffer = Buffer.alloc(BUFFER_SIZE);
   // tslint:disable-next-line: no-bitwise
@@ -85,7 +73,7 @@ function _contentFingerprint(file: string, stat: fs.Stats): string {
   } finally {
     fs.closeSync(fd);
   }
-  return `${stat.size}:${hash.digest('hex')}`;
+  return `${size}:${hash.digest('hex')}`;
 }
 
 function _hashField(hash: crypto.Hash, header: string, value: string | Buffer | DataView) {

diff --git a/packages/@aws-cdk/assets/lib/fs/mkdirpSync.ts b/packages/@aws-cdk/assets/lib/fs/mkdirpSync.ts
@@ -0,0 +1,67 @@
+// Slightly refactored version of fs-extra mkdirpSync
+// https://github.com/jprichardson/node-fs-extra/blob/d1a01e735e81688e08688557d7a254fa8297d98e/lib/mkdirs/mkdirs.js
+
+import fs = require('fs');
+import path = require('path');
+
+const INVALID_PATH_CHARS = /[<>:"|?*]/;
+const o777 = parseInt('0777', 8);
+
+function getRootPath(p: string) {
+    const paths = path.normalize(path.resolve(p)).split(path.sep);
+    if (paths.length > 0) { return paths[0]; }
+    return null;
+}
+
+function invalidWin32Path(p: string) {
+    const rp = getRootPath(p);
+    p = p.replace(rp || '', '');
+    return INVALID_PATH_CHARS.test(p);
+}
+
+export function mkdirpSync(p: string, opts?: any, made?: any) {
+    if (!opts || typeof opts !== 'object') {
+        opts = { mode: opts };
+    }
+
+    let mode = opts.mode;
+    const xfs = opts.fs || fs;
+
+    if (process.platform === 'win32' && invalidWin32Path(p)) {
+        const errInval = new Error(p + ' contains invalid WIN32 path characters.');
+        // @ts-ignore
+        errInval.code = 'EINVAL';
+        throw errInval;
+    }
+
+    if (mode === undefined) {
+        // tslint:disable-next-line: no-bitwise
+        mode = o777 & (~process.umask());
+    }
+    if (!made) { made = null; }
+
+    p = path.resolve(p);
+
+    try {
+        xfs.mkdirSync(p, mode);
+        made = made || p;
+    } catch (err0) {
+        if (err0.code === 'ENOENT') {
+            if (path.dirname(p) === p) { throw err0; }
+            made = mkdirpSync(path.dirname(p), opts, made);
+            mkdirpSync(p, opts, made);
+        } else {
+            // In the case of any other error, just see if there's a dir there
+            // already. If so, then hooray!  If not, then something is borked.
+            let stat;
+            try {
+                stat = xfs.statSync(p);
+            } catch (err1) {
+                throw err0;
+            }
+            if (!stat.isDirectory()) { throw err0; }
+        }
+    }
+
+    return made;
+}