From f4563ecc8e82f7820ff7e9c4d535fb94ac2c819b Mon Sep 17 00:00:00 2001 From: ghe Date: Wed, 17 Mar 2021 17:27:30 +0000 Subject: [PATCH] feat(@snyk/fix): Extract requirements.txt provenance Function to extract Python provenance data & tests to: - detect -r and -c directive - read the file & extract dependencies - check for -r and -c directive (recursively) - return provenance which is all deps per detected manifest as part of this chain. --- .../extract-version-provenance.ts | 51 +++++++ packages/snyk-fix/src/types.ts | 9 +- .../extract-provenance.spec.ts | 143 ++++++++++++++++++ .../with-multiple-requires/base.txt | 1 + .../workspaces/with-multiple-requires/dev.txt | 4 + .../with-multiple-requires/reqs/base.txt | 1 + .../reqs/constraints.txt | 1 + .../with-recursive-requires/base.txt | 3 + .../with-recursive-requires/constraints.txt | 1 + .../with-recursive-requires/dev.txt | 3 + .../with-require-folder-up/base.txt | 0 .../reqs/requirements.txt | 1 + .../workspaces/with-require/base.txt | 1 + 13 files changed, 215 insertions(+), 4 deletions(-) create mode 100644 packages/snyk-fix/src/plugins/python/handlers/pip-requirements/extract-version-provenance.ts create mode 100644 packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/extract-provenance.spec.ts create mode 100644 packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/workspaces/with-multiple-requires/base.txt create mode 100644 packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/workspaces/with-multiple-requires/dev.txt create mode 100644 packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/workspaces/with-multiple-requires/reqs/base.txt create mode 100644 packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/workspaces/with-multiple-requires/reqs/constraints.txt create mode 100644 packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/workspaces/with-recursive-requires/base.txt create mode 100644 packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/workspaces/with-recursive-requires/constraints.txt create mode 100644 packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/workspaces/with-recursive-requires/dev.txt create mode 100644 packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/workspaces/with-require-folder-up/base.txt create mode 100644 packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/workspaces/with-require-folder-up/reqs/requirements.txt create mode 100644 packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/workspaces/with-require/base.txt diff --git a/packages/snyk-fix/src/plugins/python/handlers/pip-requirements/extract-version-provenance.ts b/packages/snyk-fix/src/plugins/python/handlers/pip-requirements/extract-version-provenance.ts new file mode 100644 index 0000000000..e7f0a445a6 --- /dev/null +++ b/packages/snyk-fix/src/plugins/python/handlers/pip-requirements/extract-version-provenance.ts @@ -0,0 +1,51 @@ +import * as path from 'path'; +import * as debugLib from 'debug'; + +import { containsRequireDirective } from '.'; +import { + parseRequirementsFile, + Requirement, +} from './update-dependencies/requirements-file-parser'; +import { Workspace } from '../../../../types'; + +interface PythonProvenance { + [fileName: string]: Requirement[]; +} + +const debug = debugLib('snyk-fix:python:extract-version-provenance'); + +export async function extractProvenance( + workspace: Workspace, + dir: string, + fileName: string, + provenance: PythonProvenance = {}, +): Promise { + const requirementsTxt = await workspace.readFile(path.join(dir, fileName)); + provenance = { + ...provenance, + [fileName]: parseRequirementsFile(requirementsTxt), + }; + const { containsRequire, matches } = await containsRequireDirective( + requirementsTxt, + ); + if (containsRequire) { + for (const match of matches) { + const requiredFilePath = match[2]; + if (provenance[requiredFilePath]) { + debug('Detected recursive require directive, skipping'); + continue; + } + + provenance = { + ...provenance, + ...(await extractProvenance( + workspace, + dir, + requiredFilePath, + provenance, + )), + }; + } + } + return provenance; +} diff --git a/packages/snyk-fix/src/types.ts b/packages/snyk-fix/src/types.ts index 49548eb712..15bc903369 100644 --- a/packages/snyk-fix/src/types.ts +++ b/packages/snyk-fix/src/types.ts @@ -176,11 +176,12 @@ export enum SEVERITY { export type SupportedScanTypes = 'pip'; +export interface Workspace { + readFile: (path: string) => Promise; + writeFile: (path: string, content: string) => Promise; +} export interface EntityToFix { - readonly workspace: { - readFile: (path: string) => Promise; - writeFile: (path: string, content: string) => Promise; - }; + readonly workspace: Workspace; readonly scanResult: ScanResult; readonly testResult: TestResult; // options diff --git a/packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/extract-provenance.spec.ts b/packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/extract-provenance.spec.ts new file mode 100644 index 0000000000..ef1bc6d1e4 --- /dev/null +++ b/packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/extract-provenance.spec.ts @@ -0,0 +1,143 @@ +import * as fs from 'fs'; +import * as pathLib from 'path'; +import { extractProvenance } from '../../../../../src/plugins/python/handlers/pip-requirements/extract-version-provenance'; +import { parseRequirementsFile } from '../../../../../src/plugins/python/handlers/pip-requirements/update-dependencies/requirements-file-parser'; + +describe('extractProvenance', () => { + const workspacesPath = pathLib.resolve(__dirname, 'workspaces'); + it('can extract and parse 1 required files', async () => { + // Arrange + const targetFile = pathLib.resolve(workspacesPath, 'with-require/dev.txt'); + + const workspace = { + readFile: async (path: string) => { + return fs.readFileSync(pathLib.resolve(workspacesPath, path), 'utf-8'); + }, + writeFile: async () => { + return; + }, + }; + const { dir, base } = pathLib.parse(targetFile); + + // Act + const result = await extractProvenance(workspace, dir, base); + // Assert + const baseTxt = fs.readFileSync( + pathLib.resolve(workspacesPath, 'with-require/base.txt'), + 'utf-8', + ); + const devTxt = fs.readFileSync(targetFile, 'utf-8'); + + expect(result['dev.txt']).toEqual(parseRequirementsFile(devTxt)); + expect(result['base.txt']).toEqual(parseRequirementsFile(baseTxt)); + }); + + it('can extract and parse 1 required files', async () => { + // Arrange + const targetFile = pathLib.resolve( + workspacesPath, + 'with-require-folder-up/reqs/requirements.txt', + ); + + const workspace = { + readFile: async (path: string) => { + return fs.readFileSync(pathLib.resolve(workspacesPath, path), 'utf-8'); + }, + writeFile: async () => { + return; + }, + }; + const { dir, base } = pathLib.parse(targetFile); + + // Act + const result = await extractProvenance(workspace, dir, base); + // Assert + const baseTxt = fs.readFileSync( + pathLib.resolve(workspacesPath, 'with-require-folder-up/base.txt'), + 'utf-8', + ); + const requirementsTxt = fs.readFileSync(targetFile, 'utf-8'); + + expect(result['requirements.txt']).toEqual( + parseRequirementsFile(requirementsTxt), + ); + expect(result['../base.txt']).toEqual(parseRequirementsFile(baseTxt)); + }); + it('can extract and parse all required files with both -r and -c', async () => { + // Arrange + const folder = 'with-multiple-requires'; + const targetFile = pathLib.resolve(workspacesPath, `${folder}/dev.txt`); + + const workspace = { + readFile: async (path: string) => { + return fs.readFileSync(pathLib.resolve(workspacesPath, path), 'utf-8'); + }, + writeFile: async () => { + return; + }, + }; + const { dir, base } = pathLib.parse(targetFile); + + // Act + const result = await extractProvenance(workspace, dir, base); + // Assert + const baseTxt = fs.readFileSync( + pathLib.resolve(workspacesPath, pathLib.join(folder, 'base.txt')), + 'utf-8', + ); + const reqsBaseTxt = fs.readFileSync( + pathLib.resolve(workspacesPath, pathLib.join(folder, 'reqs', 'base.txt')), + 'utf-8', + ); + const devTxt = fs.readFileSync(targetFile, 'utf-8'); + const constraintsTxt = fs.readFileSync( + pathLib.resolve( + workspacesPath, + pathLib.join(folder, 'reqs', 'constraints.txt'), + ), + 'utf-8', + ); + + expect(result['dev.txt']).toEqual(parseRequirementsFile(devTxt)); + expect(result['base.txt']).toEqual(parseRequirementsFile(baseTxt)); + expect(result['reqs/base.txt']).toEqual(parseRequirementsFile(reqsBaseTxt)); + expect(result['reqs/constraints.txt']).toEqual( + parseRequirementsFile(constraintsTxt), + ); + }); + + it('can extract and parse all required files when -r is recursive', async () => { + // Arrange + const folder = 'with-recursive-requires'; + const targetFile = pathLib.resolve(workspacesPath, `${folder}/dev.txt`); + + const workspace = { + readFile: async (path: string) => { + return fs.readFileSync(pathLib.resolve(workspacesPath, path), 'utf-8'); + }, + writeFile: async () => { + return; + }, + }; + const { dir, base } = pathLib.parse(targetFile); + + // Act + const result = await extractProvenance(workspace, dir, base); + // Assert + const baseTxt = fs.readFileSync( + pathLib.resolve(workspacesPath, `${folder}/base.txt`), + 'utf-8', + ); + const devTxt = fs.readFileSync(targetFile, 'utf-8'); + const constraintsTxt = fs.readFileSync( + pathLib.resolve(workspacesPath, `${folder}/constraints.txt`), + 'utf-8', + ); + + expect(result['dev.txt']).toEqual(parseRequirementsFile(devTxt)); + expect(result['base.txt']).toEqual(parseRequirementsFile(baseTxt)); + expect(result['constraints.txt']).toEqual( + parseRequirementsFile(constraintsTxt), + ); + }); +}); diff --git a/packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/workspaces/with-multiple-requires/base.txt b/packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/workspaces/with-multiple-requires/base.txt new file mode 100644 index 0000000000..c3eeef4fb7 --- /dev/null +++ b/packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/workspaces/with-multiple-requires/base.txt @@ -0,0 +1 @@ +Jinja2==2.7.2 diff --git a/packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/workspaces/with-multiple-requires/dev.txt b/packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/workspaces/with-multiple-requires/dev.txt new file mode 100644 index 0000000000..0b8649d430 --- /dev/null +++ b/packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/workspaces/with-multiple-requires/dev.txt @@ -0,0 +1,4 @@ +-r reqs/base.txt +-r base.txt +-c reqs/constraints.txt +Django==1.6.1 diff --git a/packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/workspaces/with-multiple-requires/reqs/base.txt b/packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/workspaces/with-multiple-requires/reqs/base.txt new file mode 100644 index 0000000000..aba69a5a27 --- /dev/null +++ b/packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/workspaces/with-multiple-requires/reqs/base.txt @@ -0,0 +1 @@ +click>7.0 diff --git a/packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/workspaces/with-multiple-requires/reqs/constraints.txt b/packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/workspaces/with-multiple-requires/reqs/constraints.txt new file mode 100644 index 0000000000..697aaaaff3 --- /dev/null +++ b/packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/workspaces/with-multiple-requires/reqs/constraints.txt @@ -0,0 +1 @@ +Django==1.6.7 diff --git a/packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/workspaces/with-recursive-requires/base.txt b/packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/workspaces/with-recursive-requires/base.txt new file mode 100644 index 0000000000..c2a4c98913 --- /dev/null +++ b/packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/workspaces/with-recursive-requires/base.txt @@ -0,0 +1,3 @@ +click>7.0 +# recursive require! +-r dev.txt diff --git a/packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/workspaces/with-recursive-requires/constraints.txt b/packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/workspaces/with-recursive-requires/constraints.txt new file mode 100644 index 0000000000..697aaaaff3 --- /dev/null +++ b/packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/workspaces/with-recursive-requires/constraints.txt @@ -0,0 +1 @@ +Django==1.6.7 diff --git a/packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/workspaces/with-recursive-requires/dev.txt b/packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/workspaces/with-recursive-requires/dev.txt new file mode 100644 index 0000000000..13cef0d323 --- /dev/null +++ b/packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/workspaces/with-recursive-requires/dev.txt @@ -0,0 +1,3 @@ +-r base.txt +-c constraints.txt +Django==1.6.1 diff --git a/packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/workspaces/with-require-folder-up/base.txt b/packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/workspaces/with-require-folder-up/base.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/workspaces/with-require-folder-up/reqs/requirements.txt b/packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/workspaces/with-require-folder-up/reqs/requirements.txt new file mode 100644 index 0000000000..ad2a413c32 --- /dev/null +++ b/packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/workspaces/with-require-folder-up/reqs/requirements.txt @@ -0,0 +1 @@ +-r ../base.txt diff --git a/packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/workspaces/with-require/base.txt b/packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/workspaces/with-require/base.txt new file mode 100644 index 0000000000..aba69a5a27 --- /dev/null +++ b/packages/snyk-fix/test/acceptance/plugins/python/update-dependencies/workspaces/with-require/base.txt @@ -0,0 +1 @@ +click>7.0