From 6eead811e194b601748f45fae5a783bccb8ce4d8 Mon Sep 17 00:00:00 2001 From: Qing Tomlinson Date: Thu, 28 Sep 2023 15:50:41 -0700 Subject: [PATCH 1/3] Exclude .git directory content for package file count --- .../process/abstractClearlyDefinedProcessor.js | 2 +- providers/process/abstractProcessor.js | 18 ++++++++++++------ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/providers/process/abstractClearlyDefinedProcessor.js b/providers/process/abstractClearlyDefinedProcessor.js index b1c5176e..6eabc9f1 100644 --- a/providers/process/abstractClearlyDefinedProcessor.js +++ b/providers/process/abstractClearlyDefinedProcessor.js @@ -75,7 +75,7 @@ class AbstractClearlyDefinedProcessor extends AbstractProcessor { let count = 0 const bytes = await du(location, { filter: file => { - if (path.basename(file) === '.git') return false + if (!this.isValidExcludingGit(file)) return false count++ return true } diff --git a/providers/process/abstractProcessor.js b/providers/process/abstractProcessor.js index e371561e..bc4c3c6f 100644 --- a/providers/process/abstractProcessor.js +++ b/providers/process/abstractProcessor.js @@ -150,15 +150,21 @@ class AbstractProcessor extends BaseHandler { */ async filterFiles(location) { const fullList = await this.getFiles(location) - const exclusions = ['.git'] - const filteredList = fullList.filter(file => { - if (!file) return false - const segments = file.split(/[\\/]/g) - return !intersection(segments, exclusions).length - }) + const filteredList = fullList.filter(file => this.isValidExcludingGit(file)) return trimAllParents(filteredList, location).filter(x => x) } + _isValid(file, exclusions) { + if (!file) return false + const segments = file.split(/[\\/]/g) + return !intersection(segments, exclusions).length + } + + isValidExcludingGit(file) { + const exclusions = ['.git'] + return this._isValid(file, exclusions) + } + shouldFetch() { return true } From ca3901a816f41eb26d90b571de53e7caaea9455f Mon Sep 17 00:00:00 2001 From: Qing Tomlinson Date: Wed, 10 Jan 2024 21:50:31 -0700 Subject: [PATCH 2/3] Refactor and add unit test --- lib/utils.js | 10 +++++++++- .../process/abstractClearlyDefinedProcessor.js | 4 ++-- providers/process/abstractProcessor.js | 17 +++-------------- test/unit/lib/utilsTests.js | 18 +++++++++++++++++- 4 files changed, 31 insertions(+), 18 deletions(-) diff --git a/lib/utils.js b/lib/utils.js index 722b6fae..b3c2b63b 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -2,6 +2,7 @@ // SPDX-License-Identifier: MIT const { DateTime } = require('luxon') const { spawn } = require('child_process') +const { intersection } = require('lodash') const dateTimeFormats = [ 'EEE MMM d HH:mm:ss \'GMT\'ZZ yyyy' //in pom properties @@ -31,6 +32,13 @@ function trimAllParents(paths, parents) { return paths.map(path => trimParents(path, parents)) } +function isValidExcludingGit(file) { + const exclusions = ['.git'] + if (!file) return false + const segments = file.split(/[\\/]/g) + return !intersection(segments, exclusions).length +} + function extractDate(dateAndTime, formats = dateTimeFormats) { if (!dateAndTime) return dateAndTime let luxonResult = DateTime.fromISO(dateAndTime) @@ -75,4 +83,4 @@ function spawnPromisified(command, args, options) { }) } -module.exports = { normalizePath, normalizePaths, trimParents, trimAllParents, extractDate, spawnPromisified } +module.exports = { normalizePath, normalizePaths, trimParents, trimAllParents, isValidExcludingGit, extractDate, spawnPromisified } diff --git a/providers/process/abstractClearlyDefinedProcessor.js b/providers/process/abstractClearlyDefinedProcessor.js index 6eabc9f1..078d8db2 100644 --- a/providers/process/abstractClearlyDefinedProcessor.js +++ b/providers/process/abstractClearlyDefinedProcessor.js @@ -6,7 +6,7 @@ const throat = require('throat') const path = require('path') const { pick, merge } = require('lodash') const du = require('du') -const { trimParents } = require('../../lib/utils') +const { trimParents, isValidExcludingGit } = require('../../lib/utils') class AbstractClearlyDefinedProcessor extends AbstractProcessor { get toolVersion() { @@ -75,7 +75,7 @@ class AbstractClearlyDefinedProcessor extends AbstractProcessor { let count = 0 const bytes = await du(location, { filter: file => { - if (!this.isValidExcludingGit(file)) return false + if (!isValidExcludingGit(file)) return false count++ return true } diff --git a/providers/process/abstractProcessor.js b/providers/process/abstractProcessor.js index bc4c3c6f..23007ff0 100644 --- a/providers/process/abstractProcessor.js +++ b/providers/process/abstractProcessor.js @@ -6,11 +6,11 @@ const EntitySpec = require('../../lib/entitySpec') const fs = require('fs') const path = require('path') const shajs = require('sha.js') -const { clone, flatten, intersection, pick, set } = require('lodash') +const { clone, flatten, pick, set } = require('lodash') const { promisify } = require('util') const readdir = promisify(fs.readdir) const lstat = promisify(fs.lstat) -const { trimAllParents } = require('../../lib/utils') +const { trimAllParents, isValidExcludingGit } = require('../../lib/utils') class AbstractProcessor extends BaseHandler { constructor(options) { @@ -150,21 +150,10 @@ class AbstractProcessor extends BaseHandler { */ async filterFiles(location) { const fullList = await this.getFiles(location) - const filteredList = fullList.filter(file => this.isValidExcludingGit(file)) + const filteredList = fullList.filter(file => isValidExcludingGit(file)) return trimAllParents(filteredList, location).filter(x => x) } - _isValid(file, exclusions) { - if (!file) return false - const segments = file.split(/[\\/]/g) - return !intersection(segments, exclusions).length - } - - isValidExcludingGit(file) { - const exclusions = ['.git'] - return this._isValid(file, exclusions) - } - shouldFetch() { return true } diff --git a/test/unit/lib/utilsTests.js b/test/unit/lib/utilsTests.js index 82bee191..3ddc9933 100644 --- a/test/unit/lib/utilsTests.js +++ b/test/unit/lib/utilsTests.js @@ -3,7 +3,7 @@ const chai = require('chai') const chaiAsPromised = require('chai-as-promised') -const { normalizePath, normalizePaths, trimParents, trimAllParents, extractDate, spawnPromisified } = require('../../../lib/utils') +const { normalizePath, normalizePaths, trimParents, trimAllParents, extractDate, spawnPromisified, isValidExcludingGit } = require('../../../lib/utils') const { promisify } = require('util') const execFile = promisify(require('child_process').execFile) chai.use(chaiAsPromised) @@ -56,6 +56,22 @@ describe('Utils path functions', () => { }) }) +describe('Util isValidExcludingGit', () => { + it('should exclude .git and its contents', () => { + const data = new Map([ + [null, false], + ['/', true], + ['/tmp/tempX/package/src', true], + ['.git', false], + ['/tmp/tempX/package/.git', false], + ['/tmp/tempX/package/.git/hooks/pre-merge-commit.sample', false] + ]) + data.forEach((expected, input) => { + expect(isValidExcludingGit(input)).to.eq(expected) + }) + }) +}) + describe('Util extractDate', () => { it('handle null', () => { expect(extractDate(null)).to.be.null From 740967cd9f460e8fe4d1c6dd8f9ad4b7f28e5f6d Mon Sep 17 00:00:00 2001 From: Qing Tomlinson Date: Fri, 15 Mar 2024 14:30:39 -0700 Subject: [PATCH 3/3] Address review comments --- lib/utils.js | 7 ++--- .../abstractClearlyDefinedProcessor.js | 4 +-- providers/process/abstractProcessor.js | 4 +-- test/unit/lib/utilsTests.js | 28 +++++++++---------- 4 files changed, 21 insertions(+), 22 deletions(-) diff --git a/lib/utils.js b/lib/utils.js index b3c2b63b..df2e395a 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -32,11 +32,10 @@ function trimAllParents(paths, parents) { return paths.map(path => trimParents(path, parents)) } -function isValidExcludingGit(file) { - const exclusions = ['.git'] +function isGitFile(file) { if (!file) return false const segments = file.split(/[\\/]/g) - return !intersection(segments, exclusions).length + return intersection(segments, ['.git']).length > 0 } function extractDate(dateAndTime, formats = dateTimeFormats) { @@ -83,4 +82,4 @@ function spawnPromisified(command, args, options) { }) } -module.exports = { normalizePath, normalizePaths, trimParents, trimAllParents, isValidExcludingGit, extractDate, spawnPromisified } +module.exports = { normalizePath, normalizePaths, trimParents, trimAllParents, isGitFile, extractDate, spawnPromisified } diff --git a/providers/process/abstractClearlyDefinedProcessor.js b/providers/process/abstractClearlyDefinedProcessor.js index 078d8db2..506d860c 100644 --- a/providers/process/abstractClearlyDefinedProcessor.js +++ b/providers/process/abstractClearlyDefinedProcessor.js @@ -6,7 +6,7 @@ const throat = require('throat') const path = require('path') const { pick, merge } = require('lodash') const du = require('du') -const { trimParents, isValidExcludingGit } = require('../../lib/utils') +const { trimParents, isGitFile } = require('../../lib/utils') class AbstractClearlyDefinedProcessor extends AbstractProcessor { get toolVersion() { @@ -75,7 +75,7 @@ class AbstractClearlyDefinedProcessor extends AbstractProcessor { let count = 0 const bytes = await du(location, { filter: file => { - if (!isValidExcludingGit(file)) return false + if (isGitFile(file)) return false count++ return true } diff --git a/providers/process/abstractProcessor.js b/providers/process/abstractProcessor.js index 23007ff0..4ccb5d32 100644 --- a/providers/process/abstractProcessor.js +++ b/providers/process/abstractProcessor.js @@ -10,7 +10,7 @@ const { clone, flatten, pick, set } = require('lodash') const { promisify } = require('util') const readdir = promisify(fs.readdir) const lstat = promisify(fs.lstat) -const { trimAllParents, isValidExcludingGit } = require('../../lib/utils') +const { trimAllParents, isGitFile } = require('../../lib/utils') class AbstractProcessor extends BaseHandler { constructor(options) { @@ -150,7 +150,7 @@ class AbstractProcessor extends BaseHandler { */ async filterFiles(location) { const fullList = await this.getFiles(location) - const filteredList = fullList.filter(file => isValidExcludingGit(file)) + const filteredList = fullList.filter(file => file && !isGitFile(file)) return trimAllParents(filteredList, location).filter(x => x) } diff --git a/test/unit/lib/utilsTests.js b/test/unit/lib/utilsTests.js index 3ddc9933..b3cf5028 100644 --- a/test/unit/lib/utilsTests.js +++ b/test/unit/lib/utilsTests.js @@ -3,7 +3,7 @@ const chai = require('chai') const chaiAsPromised = require('chai-as-promised') -const { normalizePath, normalizePaths, trimParents, trimAllParents, extractDate, spawnPromisified, isValidExcludingGit } = require('../../../lib/utils') +const { normalizePath, normalizePaths, trimParents, trimAllParents, extractDate, spawnPromisified, isGitFile } = require('../../../lib/utils') const { promisify } = require('util') const execFile = promisify(require('child_process').execFile) chai.use(chaiAsPromised) @@ -56,20 +56,20 @@ describe('Utils path functions', () => { }) }) -describe('Util isValidExcludingGit', () => { - it('should exclude .git and its contents', () => { - const data = new Map([ - [null, false], - ['/', true], - ['/tmp/tempX/package/src', true], - ['.git', false], - ['/tmp/tempX/package/.git', false], - ['/tmp/tempX/package/.git/hooks/pre-merge-commit.sample', false] - ]) - data.forEach((expected, input) => { - expect(isValidExcludingGit(input)).to.eq(expected) - }) +describe('Util isGitFile', () => { + const entries = new Map([ + [null, false], + ['/', false], + ['/tmp/tempX/package/src', false], + ['.git', true], + ['/tmp/tempX/package/.git', true], + ['/tmp/tempX/package/.git/hooks/pre-merge-commit.sample', true] + ]) + + entries.forEach((expected, file) => { + it(`should return ${expected} for isGitFile given '${file}'`, () => expect(isGitFile(file)).to.eq(expected)) }) + }) describe('Util extractDate', () => {