From f1eddd8185b8b2d45636f8ca36b742b0e548dfcc Mon Sep 17 00:00:00 2001 From: buchslava Date: Thu, 13 Oct 2016 18:27:59 +0300 Subject: [PATCH] feat(generic): simple run & only root directory checking by default Closes #228 #195 --- README.md | 33 +++++++++++++++++++++++---------- lib/data/root.js | 36 +++++++++++++++++++++++++++++++++--- lib/utils/args.js | 18 +++++++++++++----- test/ddf-root.spec.js | 5 ++++- 4 files changed, 73 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index ce5d9f2..24fda4c 100644 --- a/README.md +++ b/README.md @@ -12,24 +12,25 @@ ## Console utility usage -`validate-ddf [options]` +`validate-ddf [root] [options]` ``` Commands: - root DDF Root directory + root DDF Root directory. Current directory will be processed if DDF Root directory is undefined. Options: -i Generate index file -j Fix wrong JSONs --rules print information regarding supported rules --indexless forget about ddf--index.csv and validate + --multidir validate all subdirectories --datapointless forget about datapoint validation --hidden allow hidden folders validation --include-tags Process only issues by selected tags --exclude-tags Process all tags except selected --include-rules Process only issues by selected rules --exclude-rules Process all rules except selected - --exclude-dirs Process all directories except selected + --exclude-dirs Process all directories except selected. Truly only for `--multidir` mode Examples: validate-ddf ../ddf-example validate DDF datasets for the root @@ -37,10 +38,12 @@ Examples: validate-ddf ../ddf-example -j fix JSONs for this DDF dataset validate-ddf --rules print information regarding supported rules validate-ddf ../ddf-example --indexless forget about ddf--index.csv and validate + validate-ddf ../ddf-example --multidir validate `ddf-example` and all subdirectories under "ddf-example" validate-ddf ../ddf-example --datapointless forget about datapoint validation validate-ddf ../ddf-example --hidden allow hidden folders validation - validate-ddf ../ddf-example --include-rules "INCORRECT_JSON_FIELD" Validate only by INCORRECT_JSON_FIELD rule - validate-ddf ../ddf-example --exclude-tags "WARNING" Get all kinds of issues except warnings + validate-ddf ../ddf-example --include-rules "INCORRECT_JSON_FIELD" validate only by INCORRECT_JSON_FIELD rule + validate-ddf ../ddf-example --exclude-tags "WARNING" get all kinds of issues except warnings + validate-ddf ../ddf-example --exclude-dirs "etl foo-dir" validate "ddf-example" and its subdirectories except "etl" and "foo-dir" ``` ## API usage @@ -118,10 +121,17 @@ api.validate(simpleValidator); Also all validators supports validation parameters that corresponds with command line: - * includeTags Process only issues by selected tags - * excludeTags Process all tags except selected - * includeRules Process only issues by selected rules - * excludeRules Process all rules except selected + * includeTags Process only issues by selected tags + * excludeTags Process all tags except selected + * includeRules Process only issues by selected rules + * excludeRules Process all rules except selected + * isIndexGenerationMode `-i` option + * isJsonAutoCorrectionMode `-j` option + * indexlessMode `--indexless` option + * multiDirMode `--multidir` option + * datapointlessMode `--datapointless` option + * isPrintRules `--rules` option + * isCheckHidden `--hidden` option Here is an example: @@ -129,7 +139,10 @@ Here is an example: const api = require('ddf-validation'); const expectedRule = 'INCORRECT_FILE'; const StreamValidator = api.StreamValidator; -const streamValidator = new StreamValidator(path, {includeRules: expectedRule}); +const streamValidator = new StreamValidator(path, { + includeRules: expectedRule, + multiDirMode: true +}); streamValidator.on('issue', issue => { // only one type of issue (INCORRECT_FILE) should be catched diff --git a/lib/data/root.js b/lib/data/root.js index 6bd1c0a..fe3eca6 100644 --- a/lib/data/root.js +++ b/lib/data/root.js @@ -33,7 +33,7 @@ class DDFRoot { this.directoryDescriptors = []; } - getChecks(dirs) { + getChecksMultiDir(dirs) { const actions = []; const defaultExcludes = ['etl']; @@ -57,7 +57,17 @@ class DDFRoot { return actions; } - check(cb) { + getChecksPathOnly() { + return [ + _cb => { + const directoryDescriptor = new DirectoryDescriptor(this.path); + + directoryDescriptor.check(this.settings, () => _cb(null, directoryDescriptor)); + } + ]; + } + + checkMultiDir(cb) { fu.walkDir(this.path, (err, dirs) => { if (err) { this.errors.push(err); @@ -65,7 +75,7 @@ class DDFRoot { return; } - async.parallel(this.getChecks(dirs), (_err, directoryDescriptors) => { + async.parallel(this.getChecksMultiDir(dirs), (_err, directoryDescriptors) => { this.directoryDescriptors = directoryDescriptors; if (_err) { this.errors.push(_err); @@ -76,6 +86,26 @@ class DDFRoot { }); } + checkPathOnly(cb) { + async.parallel(this.getChecksPathOnly(), (_err, directoryDescriptors) => { + this.directoryDescriptors = directoryDescriptors; + if (_err) { + this.errors.push(_err); + } + + cb(); + }); + } + + check(cb) { + if (this.settings.multiDirMode) { + this.checkMultiDir(cb); + return; + } + + this.checkPathOnly(cb); + } + getDdfDirectoriesDescriptors() { return this.directoryDescriptors.filter(desc => desc.isDDF); } diff --git a/lib/utils/args.js b/lib/utils/args.js index 48d131d..be22400 100644 --- a/lib/utils/args.js +++ b/lib/utils/args.js @@ -1,32 +1,39 @@ 'use strict'; const _ = require('lodash'); -const ROOT_PARAMETER_IS_REQUIRED = _.includes(process.argv, '--rules') ? 0 : 1; +// const ROOT_PARAMETER_IS_REQUIRED = _.includes(process.argv, '--rules') ? 0 : 1; const myName = 'validate-ddf'; const argv = require('yargs') - .usage(`Usage: ${myName} [options]`) + .usage(`Usage: ${myName} [root] [options]`) .command('root', 'DDF Root directory') - .demand(ROOT_PARAMETER_IS_REQUIRED) + // .demand(ROOT_PARAMETER_IS_REQUIRED) .example(`${myName} ../ddf-example`, 'validate DDF datasets for the root') .example(`${myName} ../ddf-example -i`, 'generate ddf--index file') .example(`${myName} ../ddf-example -j`, 'fix JSONs for this DDF dataset') .example(`${myName} --rules`, 'print information regarding supported rules') .example(`${myName} ../ddf-example --indexless`, 'forget about ddf--index.csv and validate') + .example(`${myName} ../ddf-example --multidir`, + 'validate `ddf-example` and all subdirectories under "ddf-example"') .example(`${myName} ../ddf-example --hidden`, 'allow hidden folders validation') .example(`${myName} ../ddf-example --include-rules "INCORRECT_JSON_FIELD"`, 'Validate only by INCORRECT_JSON_FIELD rule') - .example(`${myName} ../ddf-example --exclude-tags "WARNING_TAG"`, 'Get all kinds of issues except warnings') + .example(`${myName} ../ddf-example --exclude-tags "WARNING_TAG"`, + 'Get all kinds of issues except warnings') + .example(`${myName} ../ddf-example --exclude-dirs "etl foo-dir"`, + 'validate "ddf-example" and its subdirectories except "etl" and "foo-dir"') .describe('i', 'Generate index file') .describe('j', 'Fix wrong JSONs') .describe('rules', 'print information regarding supported rules') .describe('indexless', 'forget about ddf--index.csv and validate') + .describe('multidir', 'validate all subdirectories') .describe('datapointless', 'forget about datapoint validation') .describe('hidden', 'allow hidden folders validation') .describe('include-tags', 'Process only issues by selected tags') .describe('exclude-tags', 'Process all tags except selected') .describe('include-rules', 'Process only issues by selected rules') .describe('exclude-rules', 'Process all rules except selected') - .describe('exclude-dirs', 'Process all directories except selected') + .describe('exclude-dirs', + 'Process all directories except selected. Truly only for `--multidir` mode') .argv; exports.getDDFRootFolder = () => argv._[0] || process.cwd(); @@ -38,6 +45,7 @@ exports.getSettings = () => { settings.isIndexGenerationMode = !!argv.i; settings.isJsonAutoCorrectionMode = !!argv.j; settings.indexlessMode = !!argv.indexless; + settings.multiDirMode = !!argv.multidir; settings.datapointlessMode = !!argv.datapointless; settings.isPrintRules = !!argv.rules; settings.isCheckHidden = !!argv.hidden; diff --git a/test/ddf-root.spec.js b/test/ddf-root.spec.js index cd4782d..729d1dd 100644 --- a/test/ddf-root.spec.js +++ b/test/ddf-root.spec.js @@ -43,7 +43,10 @@ describe('ddf root folder validation', () => { describe('when good folder with sub-folders (fixtures/good-folder-with-subfolders)', () => { it('count of ddf folders should be greater than 1', done => { - const ddfRoot = new DDFRoot('./test/fixtures/good-folder-with-subfolders'); + const ddfRoot = new DDFRoot( + './test/fixtures/good-folder-with-subfolders', + {multiDirMode: true} + ); ddfRoot.check(() => { expect(ddfRoot.getDdfDirectoriesDescriptors().length)