From af60a6d6579a7787dfa02200d6b10a7d432ed92c Mon Sep 17 00:00:00 2001 From: Hongxin Liang Date: Wed, 31 Mar 2021 23:58:04 +0200 Subject: [PATCH 01/23] Extend helper to support parsing setup.cfg --- python/helpers/lib/parser.py | 48 ++++++++++++++++++++++++++++++++++++ python/helpers/run.py | 4 ++- 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/python/helpers/lib/parser.py b/python/helpers/lib/parser.py index e9770607982..c8d641a8151 100644 --- a/python/helpers/lib/parser.py +++ b/python/helpers/lib/parser.py @@ -1,3 +1,4 @@ +import configparser import glob import io import json @@ -136,3 +137,50 @@ def fake_open(*args, **kwargs): exec(content) in globals(), locals() return json.dumps({"result": setup_packages}) + + +def parse_setup_cfg(directory): + # Parse the setup.py + setup_packages = [] + if os.path.isfile(directory + '/setup.cfg'): + def version_from_install_req(install_req): + if install_req.is_pinned: + return next(iter(install_req.specifier)).version + + def parse_requirement(req, req_type): + install_req = install_req_from_line(req) + if install_req.original_link: + return + + setup_packages.append({ + "name": install_req.req.name, + "version": version_from_install_req(install_req), + "markers": str(install_req.markers) or None, + "file": "setup.cfg", + "requirement": str(install_req.specifier) or None, + "requirement_type": req_type, + "extras": sorted(list(install_req.extras)) + }) + + def parse_requirements(requires, req_type): + for req in requires: + parse_requirement(req, req_type) + + config = configparser.ConfigParser() + config.read(directory + '/setup.cfg') + + for req_type in ['setup_requires', 'install_requires', 'tests_require']: + requires = [ + r + for r in config.get('options', req_type, fallback='').strip().split('\n') + if r != '' + ] + parse_requirements(requires, req_type) + + extras_require_section = 'options.extras_require' + if config.has_section(extras_require_section): + section = config[extras_require_section] + for key, value in section.items(): + parse_requirements(value.strip().split('\n'), 'extras_require:{}'.format(key)) + + return json.dumps({"result": setup_packages}) diff --git a/python/helpers/run.py b/python/helpers/run.py index c70d3d4a31e..b45c5512cff 100644 --- a/python/helpers/run.py +++ b/python/helpers/run.py @@ -8,8 +8,10 @@ if args["function"] == "parse_requirements": print(parser.parse_requirements(args["args"][0])) - if args["function"] == "parse_setup": + elif args["function"] == "parse_setup": print(parser.parse_setup(args["args"][0])) + elif args["function"] == "parse_setup_cfg": + print(parser.parse_setup_cfg(args["args"][0])) elif args["function"] == "get_dependency_hash": print(hasher.get_dependency_hash(*args["args"])) elif args["function"] == "get_pipfile_hash": From 1efe482319b602d177e229b092a8c6bdbc14522b Mon Sep 17 00:00:00 2001 From: Hongxin Liang Date: Wed, 31 Mar 2021 23:59:01 +0200 Subject: [PATCH 02/23] File parser for setup.cfg --- python/lib/dependabot/python/file_parser.rb | 16 +++++++++++- .../file_parser/setup_cfg_file_parser.rb | 26 +++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 python/lib/dependabot/python/file_parser/setup_cfg_file_parser.rb diff --git a/python/lib/dependabot/python/file_parser.rb b/python/lib/dependabot/python/file_parser.rb index a4d9ab9cd65..ae7fd467ff2 100644 --- a/python/lib/dependabot/python/file_parser.rb +++ b/python/lib/dependabot/python/file_parser.rb @@ -17,6 +17,7 @@ class FileParser < Dependabot::FileParsers::Base require_relative "file_parser/pipfile_files_parser" require_relative "file_parser/poetry_files_parser" require_relative "file_parser/setup_file_parser" + require_relative "file_parser/setup_cfg_file_parser" POETRY_DEPENDENCY_TYPES = %w(tool.poetry.dependencies tool.poetry.dev-dependencies).freeze @@ -45,6 +46,7 @@ def parse dependency_set += poetry_dependencies if using_poetry? dependency_set += requirement_dependencies if requirement_files.any? dependency_set += setup_file_dependencies if setup_file + dependency_set += setup_cfg_file_dependencies if setup_cfg_file dependency_set.dependencies end @@ -137,6 +139,13 @@ def setup_file_dependencies dependency_set end + def setup_cfg_file_dependencies + @setup_cfg_file_dependencies ||= + SetupCfgFileParser. + new(dependency_files: dependency_files). + dependency_set + end + def lockfile_for_pip_compile_file?(filename) return false unless pip_compile_files.any? return false unless filename.end_with?(".txt") @@ -207,8 +216,9 @@ def check_required_files return if pipfile return if pyproject return if setup_file + return if setup_cfg_file - raise "No requirements.txt or setup.py!" + raise "Missing required files!" end def pipfile @@ -248,6 +258,10 @@ def setup_file @setup_file ||= get_original_file("setup.py") end + def setup_cfg_file + @setup_cfg_file ||= get_original_file("setup.cfg") + end + def pip_compile_files @pip_compile_files ||= dependency_files.select { |f| f.name.end_with?(".in") } diff --git a/python/lib/dependabot/python/file_parser/setup_cfg_file_parser.rb b/python/lib/dependabot/python/file_parser/setup_cfg_file_parser.rb new file mode 100644 index 00000000000..b653f0326d9 --- /dev/null +++ b/python/lib/dependabot/python/file_parser/setup_cfg_file_parser.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +require "dependabot/shared_helpers" +require "dependabot/python/native_helpers" +require "dependabot/python/file_parser/setup_file_parser" + +module Dependabot + module Python + class FileParser + class SetupCfgFileParser < Dependabot::Python::FileParser::SetupFileParser + private + + def parsed_setup_file + requirements = SharedHelpers.run_helper_subprocess( + command: "pyenv exec python #{NativeHelpers.python_helper_path}", + function: "parse_setup_cfg", + args: [Dir.pwd] + ) + + check_requirements(requirements) + requirements + end + end + end + end +end From a13be0fdd099712eec07e942e1e1756e66655188 Mon Sep 17 00:00:00 2001 From: Hongxin Liang Date: Thu, 1 Apr 2021 00:16:50 +0200 Subject: [PATCH 03/23] Fetcher and updater --- python/lib/dependabot/python/file_fetcher.rb | 1 + python/lib/dependabot/python/file_updater.rb | 4 +++- .../dependabot/python/update_checker/requirements_updater.rb | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/python/lib/dependabot/python/file_fetcher.rb b/python/lib/dependabot/python/file_fetcher.rb index 48bf3d6a9c8..a73071d822b 100644 --- a/python/lib/dependabot/python/file_fetcher.rb +++ b/python/lib/dependabot/python/file_fetcher.rb @@ -25,6 +25,7 @@ def self.required_files_in?(filenames) # If this repo is using Poetry return true return true if filenames.include?("pyproject.toml") + # FIXME: this assumption is no longer true filenames.include?("setup.py") end diff --git a/python/lib/dependabot/python/file_updater.rb b/python/lib/dependabot/python/file_updater.rb index 16de85814f0..e877634aa3e 100644 --- a/python/lib/dependabot/python/file_updater.rb +++ b/python/lib/dependabot/python/file_updater.rb @@ -19,6 +19,7 @@ def self.updated_files_regex /.*\.txt$/, /.*\.in$/, /^setup\.py$/, + /^setup\.cfg$/, /^pyproject\.toml$/, /^pyproject\.lock$/ ] @@ -113,8 +114,9 @@ def check_required_files return if pipfile return if pyproject return if get_original_file("setup.py") + return if get_original_file("setup.cfg") - raise "No requirements.txt or setup.py!" + raise "Missing required files!" end def pipfile diff --git a/python/lib/dependabot/python/update_checker/requirements_updater.rb b/python/lib/dependabot/python/update_checker/requirements_updater.rb index 903b7cf088e..6c977cd89db 100644 --- a/python/lib/dependabot/python/update_checker/requirements_updater.rb +++ b/python/lib/dependabot/python/update_checker/requirements_updater.rb @@ -32,7 +32,7 @@ def initialize(requirements:, update_strategy:, has_lockfile:, def updated_requirements requirements.map do |req| case req[:file] - when "setup.py" then updated_setup_requirement(req) + when /setup\.(?:py|cfg)$/ then updated_setup_requirement(req) when "pyproject.toml" then updated_pyproject_requirement(req) when "Pipfile" then updated_pipfile_requirement(req) when /\.txt$|\.in$/ then updated_requirement(req) From 14dd6a5363a1b338ad77a31ea54bbaa0e7bbc07c Mon Sep 17 00:00:00 2001 From: Hongxin Liang Date: Thu, 1 Apr 2021 10:43:19 +0200 Subject: [PATCH 04/23] Fix flake8 --- python/helpers/lib/parser.py | 44 ++++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/python/helpers/lib/parser.py b/python/helpers/lib/parser.py index c8d641a8151..d0dfdde66be 100644 --- a/python/helpers/lib/parser.py +++ b/python/helpers/lib/parser.py @@ -140,9 +140,9 @@ def fake_open(*args, **kwargs): def parse_setup_cfg(directory): - # Parse the setup.py setup_packages = [] - if os.path.isfile(directory + '/setup.cfg'): + if os.path.isfile(directory + "/setup.cfg"): + def version_from_install_req(install_req): if install_req.is_pinned: return next(iter(install_req.specifier)).version @@ -152,35 +152,45 @@ def parse_requirement(req, req_type): if install_req.original_link: return - setup_packages.append({ - "name": install_req.req.name, - "version": version_from_install_req(install_req), - "markers": str(install_req.markers) or None, - "file": "setup.cfg", - "requirement": str(install_req.specifier) or None, - "requirement_type": req_type, - "extras": sorted(list(install_req.extras)) - }) + setup_packages.append( + { + "name": install_req.req.name, + "version": version_from_install_req(install_req), + "markers": str(install_req.markers) or None, + "file": "setup.cfg", + "requirement": str(install_req.specifier) or None, + "requirement_type": req_type, + "extras": sorted(list(install_req.extras)), + } + ) def parse_requirements(requires, req_type): for req in requires: parse_requirement(req, req_type) config = configparser.ConfigParser() - config.read(directory + '/setup.cfg') + config.read(directory + "/setup.cfg") - for req_type in ['setup_requires', 'install_requires', 'tests_require']: + for req_type in [ + "setup_requires", + "install_requires", + "tests_require", + ]: requires = [ r - for r in config.get('options', req_type, fallback='').strip().split('\n') - if r != '' + for r in config.get("options", req_type, fallback="") + .strip() + .split("\n") + if r != "" ] parse_requirements(requires, req_type) - extras_require_section = 'options.extras_require' + extras_require_section = "options.extras_require" if config.has_section(extras_require_section): section = config[extras_require_section] for key, value in section.items(): - parse_requirements(value.strip().split('\n'), 'extras_require:{}'.format(key)) + parse_requirements( + value.strip().split("\n"), "extras_require:{}".format(key) + ) return json.dumps({"result": setup_packages}) From 59a45e93588c020af754f549d3a132903715db6d Mon Sep 17 00:00:00 2001 From: Hongxin Liang Date: Thu, 1 Apr 2021 23:50:49 +0200 Subject: [PATCH 05/23] Test file parser --- .gitignore | 2 + .../file_parser/setup_cfg_file_parser_spec.rb | 118 ++++++++++++++++++ .../fixtures/setup_files/illformed_req.cfg | 28 +++++ .../fixtures/setup_files/no_tests_require.cfg | 22 ++++ .../setup_files/setup_with_requires.cfg | 29 +++++ 5 files changed, 199 insertions(+) create mode 100644 python/spec/dependabot/python/file_parser/setup_cfg_file_parser_spec.rb create mode 100644 python/spec/fixtures/setup_files/illformed_req.cfg create mode 100644 python/spec/fixtures/setup_files/no_tests_require.cfg create mode 100644 python/spec/fixtures/setup_files/setup_with_requires.cfg diff --git a/.gitignore b/.gitignore index 4422e0a2042..6a7604a8228 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,5 @@ vendor **/bin/helper /.core-bash_history coverage/ +.ruby-gemset +.ruby-version diff --git a/python/spec/dependabot/python/file_parser/setup_cfg_file_parser_spec.rb b/python/spec/dependabot/python/file_parser/setup_cfg_file_parser_spec.rb new file mode 100644 index 00000000000..856285ad1ee --- /dev/null +++ b/python/spec/dependabot/python/file_parser/setup_cfg_file_parser_spec.rb @@ -0,0 +1,118 @@ +# frozen_string_literal: true + +require "spec_helper" +require "dependabot/dependency_file" +require "dependabot/python/file_parser/setup_cfg_file_parser" + +RSpec.describe Dependabot::Python::FileParser::SetupCfgFileParser do + let(:parser) { described_class.new(dependency_files: files) } + + let(:files) { [setup_cfg_file] } + let(:setup_cfg_file) do + Dependabot::DependencyFile.new( + name: "setup.cfg", + content: setup_cfg_file_body + ) + end + let(:setup_cfg_file_body) do + fixture("setup_files", setup_cfg_file_fixture_name) + end + let(:setup_cfg_file_fixture_name) { "setup_with_requires.cfg" } + + describe "parse" do + subject(:dependencies) { parser.dependency_set.dependencies } + + its(:length) { is_expected.to eq(15) } + + describe "an install_requires dependencies" do + subject(:dependency) { dependencies.find { |d| d.name == "boto3" } } + + it "has the right details" do + expect(dependency).to be_a(Dependabot::Dependency) + expect(dependency.name).to eq("boto3") + expect(dependency.version).to eq("1.3.1") + expect(dependency.requirements).to eq( + [{ + requirement: "==1.3.1", + file: "setup.cfg", + groups: ["install_requires"], + source: nil + }] + ) + end + end + + describe "a setup_requires dependencies" do + subject(:dependency) { dependencies.find { |d| d.name == "numpy" } } + + it "has the right details" do + expect(dependency).to be_a(Dependabot::Dependency) + expect(dependency.name).to eq("numpy") + expect(dependency.version).to eq("1.11.0") + expect(dependency.requirements).to eq( + [{ + requirement: "==1.11.0", + file: "setup.cfg", + groups: ["setup_requires"], + source: nil + }] + ) + end + end + + describe "a tests_require dependencies" do + subject(:dependency) { dependencies.find { |d| d.name == "responses" } } + + it "has the right details" do + expect(dependency).to be_a(Dependabot::Dependency) + expect(dependency.name).to eq("responses") + expect(dependency.version).to eq("0.5.1") + expect(dependency.requirements).to eq( + [{ + requirement: "==0.5.1", + file: "setup.cfg", + groups: ["tests_require"], + source: nil + }] + ) + end + end + + describe "an extras_require dependencies" do + subject(:dependency) { dependencies.find { |d| d.name == "flask" } } + + it "has the right details" do + expect(dependency).to be_a(Dependabot::Dependency) + expect(dependency.name).to eq("flask") + expect(dependency.version).to eq("0.12.2") + expect(dependency.requirements).to eq( + [{ + requirement: "==0.12.2", + file: "setup.cfg", + groups: ["extras_require:API"], + source: nil + }] + ) + end + end + + context "without a `tests_require` key" do + let(:setup_cfg_file_fixture_name) { "no_tests_require.cfg" } + its(:length) { is_expected.to eq(12) } + end + + context "with an illformed_requirement" do + let(:setup_cfg_file_fixture_name) { "illformed_req.cfg" } + + it "raises a helpful error" do + expect { parser.dependency_set }. + to raise_error do |error| + expect(error.class). + to eq(Dependabot::DependencyFileNotEvaluatable) + expect(error.message). + to eq('Illformed requirement ["==2.6.1raven==5.32.0"]') + end + end + end + end +end diff --git a/python/spec/fixtures/setup_files/illformed_req.cfg b/python/spec/fixtures/setup_files/illformed_req.cfg new file mode 100644 index 00000000000..842e8f8dc41 --- /dev/null +++ b/python/spec/fixtures/setup_files/illformed_req.cfg @@ -0,0 +1,28 @@ +name = python-package +version = 0.0 +description = Example setup.cfg +url = httos://github.com/example/python-package +author = Dependabot + +[options] +packages = find: +setup_requires = + numpy==1.11.0 + pytest-runner +install_requires = + boto3==1.3.1 + flake8 > 2.5.4, < 3.0.0 + gocardless_pro + pandas==0.19.2 + pep8==1.7.0 + psycopg2==2.6.1raven == 5.32.0 + requests==2.12.* + scipy==0.18.1 + scikit-learn==0.18.1 + +tests_require = + pytest==2.9.1 + responses==0.5.1 + +[options.extras_require] +API = flask==0.12.2 diff --git a/python/spec/fixtures/setup_files/no_tests_require.cfg b/python/spec/fixtures/setup_files/no_tests_require.cfg new file mode 100644 index 00000000000..e63d47289c4 --- /dev/null +++ b/python/spec/fixtures/setup_files/no_tests_require.cfg @@ -0,0 +1,22 @@ +name = python-package +version = 0.0 +description = Example setup.cfg +url = httos://github.com/example/python-package +author = Dependabot + +[options] +packages = find: +setup_requires = + numpy==1.11.0 + pytest-runner +install_requires = + boto3==1.3.1 + flake8 > 2.5.4, < 3.0.0 + gocardless_pro + pandas==0.19.2 + pep8==1.7.0 + psycopg2==2.6.1 + raven == 5.32.0 + requests==2.12.* + scipy==0.18.1 + scikit-learn==0.18.1 diff --git a/python/spec/fixtures/setup_files/setup_with_requires.cfg b/python/spec/fixtures/setup_files/setup_with_requires.cfg new file mode 100644 index 00000000000..fa8526bfc16 --- /dev/null +++ b/python/spec/fixtures/setup_files/setup_with_requires.cfg @@ -0,0 +1,29 @@ +name = python-package +version = 0.0 +description = Example setup.cfg +url = httos://github.com/example/python-package +author = Dependabot + +[options] +packages = find: +setup_requires = + numpy==1.11.0 + pytest-runner +install_requires = + boto3==1.3.1 + flake8 > 2.5.4, < 3.0.0 + gocardless_pro + pandas==0.19.2 + pep8==1.7.0 + psycopg2==2.6.1 + raven == 5.32.0 + requests==2.12.* + scipy==0.18.1 + scikit-learn==0.18.1 + +tests_require = + pytest==2.9.1 + responses==0.5.1 + +[options.extras_require] +API = flask==0.12.2 From 93d67ebf49671826199e92b8491bdb8aaf09b2a4 Mon Sep 17 00:00:00 2001 From: Hongxin Liang Date: Fri, 2 Apr 2021 00:32:25 +0200 Subject: [PATCH 06/23] Test file parser --- .../dependabot/python/file_parser_spec.rb | 86 +++++++++++++++++++ python/spec/fixtures/setup_files/extras.cfg | 22 +++++ python/spec/fixtures/setup_files/markers.cfg | 30 +++++++ 3 files changed, 138 insertions(+) create mode 100644 python/spec/fixtures/setup_files/extras.cfg create mode 100644 python/spec/fixtures/setup_files/markers.cfg diff --git a/python/spec/dependabot/python/file_parser_spec.rb b/python/spec/dependabot/python/file_parser_spec.rb index 2fdc123bd37..6fb4f792126 100644 --- a/python/spec/dependabot/python/file_parser_spec.rb +++ b/python/spec/dependabot/python/file_parser_spec.rb @@ -979,6 +979,92 @@ end end + context "with a setup.cfg" do + let(:files) { [setup_cfg_file] } + let(:setup_cfg_file) do + Dependabot::DependencyFile.new( + name: "setup.cfg", + content: fixture("setup_files", "setup_with_requires.cfg") + ) + end + + its(:length) { is_expected.to eq(15) } + + describe "an install_requires dependencies" do + subject(:dependency) { dependencies.find { |d| d.name == "boto3" } } + + it "has the right details" do + expect(dependency).to be_a(Dependabot::Dependency) + expect(dependency.name).to eq("boto3") + expect(dependency.version).to eq("1.3.1") + expect(dependency.requirements).to eq( + [{ + requirement: "==1.3.1", + file: "setup.cfg", + groups: ["install_requires"], + source: nil + }] + ) + end + end + + context "with markers" do + let(:setup_cfg_file) do + Dependabot::DependencyFile.new( + name: "setup.cfg", + content: fixture("setup_files", "markers.cfg") + ) + end + + describe "a dependency with markers" do + subject(:dependency) { dependencies.find { |d| d.name == "boto3" } } + + it "has the right details" do + expect(dependency).to be_a(Dependabot::Dependency) + expect(dependency.name).to eq("boto3") + expect(dependency.version).to eq("1.3.1") + expect(dependency.requirements).to eq( + [{ + requirement: "==1.3.1", + file: "setup.cfg", + groups: ["install_requires"], + source: nil + }] + ) + end + end + end + + context "with extras" do + let(:setup_cfg_file) do + Dependabot::DependencyFile.new( + name: "setup.cfg", + content: fixture("setup_files", "extras.cfg") + ) + end + + describe "a dependency with extras" do + subject(:dependency) do + dependencies.find { |d| d.name == "requests[security]" } + end + + it "has the right details" do + expect(dependency).to be_a(Dependabot::Dependency) + expect(dependency.name).to eq("requests[security]") + expect(dependency.version).to be_nil + expect(dependency.requirements).to eq( + [{ + requirement: "==2.12.*", + file: "setup.cfg", + groups: ["install_requires"], + source: nil + }] + ) + end + end + end + end + context "with a Pipfile and Pipfile.lock" do let(:files) { [pipfile, lockfile] } let(:pipfile) do diff --git a/python/spec/fixtures/setup_files/extras.cfg b/python/spec/fixtures/setup_files/extras.cfg new file mode 100644 index 00000000000..6f8cc195d67 --- /dev/null +++ b/python/spec/fixtures/setup_files/extras.cfg @@ -0,0 +1,22 @@ +name = python-package +version = 0.0 +description = Example setup.cfg +url = httos://github.com/example/python-package +author = Dependabot + +[options] +packages = find: +setup_requires = + numpy==1.11.0 + pytest-runner +install_requires = + requests[security] == 2.12.* + scipy==0.18.1 + scikit-learn==0.18.1 + +tests_require = + pytest==2.9.1 + responses==0.5.1 + +[options.extras_require] +API = flask==0.12.2 diff --git a/python/spec/fixtures/setup_files/markers.cfg b/python/spec/fixtures/setup_files/markers.cfg new file mode 100644 index 00000000000..bc2451b074e --- /dev/null +++ b/python/spec/fixtures/setup_files/markers.cfg @@ -0,0 +1,30 @@ +name = python-package +version = 0.0 +description = Example setup.cfg +url = httos://github.com/example/python-package +author = Dependabot + +[options] +packages = find: +setup_requires = + numpy==1.11.0 + pytest-runner +install_requires = + boto3==1.3.1;python_version<="2.6" + boto3==1.3.1;python_version>="2.7" + flake8 > 2.5.4, < 3.0.0 + gocardless_pro + pandas==0.19.2 + pep8==1.7.0 + psycopg2==2.6.1 + raven == 5.32.0 + requests==2.12.* + scipy==0.18.1 + scikit-learn==0.18.1 + +tests_require = + pytest==2.9.1 + responses==0.5.1 + +[options.extras_require] +API = flask==0.12.2 From 39353d19527ed2b7c80817c8451ab9dabeffaa86 Mon Sep 17 00:00:00 2001 From: Hongxin Liang Date: Fri, 2 Apr 2021 00:43:41 +0200 Subject: [PATCH 07/23] Fix fixture file --- python/spec/fixtures/setup_files/setup_with_requires.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/python/spec/fixtures/setup_files/setup_with_requires.cfg b/python/spec/fixtures/setup_files/setup_with_requires.cfg index fa8526bfc16..6ff01959226 100644 --- a/python/spec/fixtures/setup_files/setup_with_requires.cfg +++ b/python/spec/fixtures/setup_files/setup_with_requires.cfg @@ -1,3 +1,4 @@ +[metadata] name = python-package version = 0.0 description = Example setup.cfg From f42d221cffb384042c92944f4ae0d45850a92906 Mon Sep 17 00:00:00 2001 From: Hongxin Liang Date: Fri, 2 Apr 2021 00:44:13 +0200 Subject: [PATCH 08/23] Check setup.cfg as well --- python/lib/dependabot/python/file_fetcher.rb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/python/lib/dependabot/python/file_fetcher.rb b/python/lib/dependabot/python/file_fetcher.rb index a73071d822b..8dbf2d416aa 100644 --- a/python/lib/dependabot/python/file_fetcher.rb +++ b/python/lib/dependabot/python/file_fetcher.rb @@ -25,12 +25,13 @@ def self.required_files_in?(filenames) # If this repo is using Poetry return true return true if filenames.include?("pyproject.toml") - # FIXME: this assumption is no longer true - filenames.include?("setup.py") + return true if filenames.include?("setup.py") + + filename.include?("setup.cfg") end def self.required_files_message - "Repo must contain a requirements.txt, setup.py, pyproject.toml, "\ + "Repo must contain a requirements.txt, setup.py, setup.cfg, pyproject.toml, "\ "or a Pipfile." end From 7995c92117de4156e05a0aed109c98855d51b839 Mon Sep 17 00:00:00 2001 From: Hongxin Liang Date: Fri, 2 Apr 2021 01:00:17 +0200 Subject: [PATCH 09/23] Test file fetcher --- .../dependabot/python/file_fetcher_spec.rb | 26 ++ .../contents_python_only_setup_cfg.json | 418 ++++++++++++++++++ .../fixtures/github/setup_cfg_content.json | 18 + 3 files changed, 462 insertions(+) create mode 100644 python/spec/fixtures/github/contents_python_only_setup_cfg.json create mode 100644 python/spec/fixtures/github/setup_cfg_content.json diff --git a/python/spec/dependabot/python/file_fetcher_spec.rb b/python/spec/dependabot/python/file_fetcher_spec.rb index 0f51cd841d9..de70f4a74ec 100644 --- a/python/spec/dependabot/python/file_fetcher_spec.rb +++ b/python/spec/dependabot/python/file_fetcher_spec.rb @@ -20,6 +20,11 @@ it { is_expected.to eq(true) } end + context "with only a setup.cfg" do + let(:filenames) { %w(setup.cfg) } + it { is_expected.to eq(true) } + end + context "with only a requirements folder" do let(:filenames) { %w(requirements) } it { is_expected.to eq(true) } @@ -211,6 +216,27 @@ end end + context "with only a setup.cfg file" do + let(:repo_contents) do + fixture("github", "contents_python_only_setup_cfg.json") + end + before do + stub_request(:get, url + "setup.cfg?ref=sha"). + with(headers: { "Authorization" => "token token" }). + to_return( + status: 200, + body: fixture("github", "setup_cfg_content.json"), + headers: { "content-type" => "application/json" } + ) + end + + it "fetches the setup.cfg file" do + expect(file_fetcher_instance.files.count).to eq(1) + expect(file_fetcher_instance.files.map(&:name)). + to eq(["setup.cfg"]) + end + end + context "with only a Pipfile and Pipfile.lock" do let(:repo_contents) do fixture("github", "contents_python_only_pipfile_and_lockfile.json") diff --git a/python/spec/fixtures/github/contents_python_only_setup_cfg.json b/python/spec/fixtures/github/contents_python_only_setup_cfg.json new file mode 100644 index 00000000000..4fed78216f7 --- /dev/null +++ b/python/spec/fixtures/github/contents_python_only_setup_cfg.json @@ -0,0 +1,418 @@ +[ + { + "name": ".codeclimate.yml", + "path": ".codeclimate.yml", + "sha": "6393602fac96cfe31d64f89476014124b4a13b85", + "size": 416, + "url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/.codeclimate.yml?ref=develop", + "html_url": "https://github.com/CityOfNewYork/NYCOpenRecords/blob/develop/.codeclimate.yml", + "git_url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/blobs/6393602fac96cfe31d64f89476014124b4a13b85", + "download_url": "https://raw.githubusercontent.com/CityOfNewYork/NYCOpenRecords/develop/.codeclimate.yml", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/.codeclimate.yml?ref=develop", + "git": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/blobs/6393602fac96cfe31d64f89476014124b4a13b85", + "html": "https://github.com/CityOfNewYork/NYCOpenRecords/blob/develop/.codeclimate.yml" + } + }, + { + "name": ".coveragerc", + "path": ".coveragerc", + "sha": "be7fdc86067d0924f3e1e92c1a3ed92d9486fcbb", + "size": 646, + "url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/.coveragerc?ref=develop", + "html_url": "https://github.com/CityOfNewYork/NYCOpenRecords/blob/develop/.coveragerc", + "git_url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/blobs/be7fdc86067d0924f3e1e92c1a3ed92d9486fcbb", + "download_url": "https://raw.githubusercontent.com/CityOfNewYork/NYCOpenRecords/develop/.coveragerc", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/.coveragerc?ref=develop", + "git": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/blobs/be7fdc86067d0924f3e1e92c1a3ed92d9486fcbb", + "html": "https://github.com/CityOfNewYork/NYCOpenRecords/blob/develop/.coveragerc" + } + }, + { + "name": ".csslintrc", + "path": ".csslintrc", + "sha": "aacba956e5bbede1c195ce554fcad3e200b24ff8", + "size": 107, + "url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/.csslintrc?ref=develop", + "html_url": "https://github.com/CityOfNewYork/NYCOpenRecords/blob/develop/.csslintrc", + "git_url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/blobs/aacba956e5bbede1c195ce554fcad3e200b24ff8", + "download_url": "https://raw.githubusercontent.com/CityOfNewYork/NYCOpenRecords/develop/.csslintrc", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/.csslintrc?ref=develop", + "git": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/blobs/aacba956e5bbede1c195ce554fcad3e200b24ff8", + "html": "https://github.com/CityOfNewYork/NYCOpenRecords/blob/develop/.csslintrc" + } + }, + { + "name": ".env.example", + "path": ".env.example", + "sha": "d9f599b33ecd834ea88979dcc3daf4fcafacf4e7", + "size": 2617, + "url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/.env.example?ref=develop", + "html_url": "https://github.com/CityOfNewYork/NYCOpenRecords/blob/develop/.env.example", + "git_url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/blobs/d9f599b33ecd834ea88979dcc3daf4fcafacf4e7", + "download_url": "https://raw.githubusercontent.com/CityOfNewYork/NYCOpenRecords/develop/.env.example", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/.env.example?ref=develop", + "git": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/blobs/d9f599b33ecd834ea88979dcc3daf4fcafacf4e7", + "html": "https://github.com/CityOfNewYork/NYCOpenRecords/blob/develop/.env.example" + } + }, + { + "name": ".eslintignore", + "path": ".eslintignore", + "sha": "96212a3593bac8c93f624b77185a8d11018efd96", + "size": 16, + "url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/.eslintignore?ref=develop", + "html_url": "https://github.com/CityOfNewYork/NYCOpenRecords/blob/develop/.eslintignore", + "git_url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/blobs/96212a3593bac8c93f624b77185a8d11018efd96", + "download_url": "https://raw.githubusercontent.com/CityOfNewYork/NYCOpenRecords/develop/.eslintignore", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/.eslintignore?ref=develop", + "git": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/blobs/96212a3593bac8c93f624b77185a8d11018efd96", + "html": "https://github.com/CityOfNewYork/NYCOpenRecords/blob/develop/.eslintignore" + } + }, + { + "name": ".eslintrc.yml", + "path": ".eslintrc.yml", + "sha": "a6a0ce9c44238e06ff55ca335a66a4e16810da38", + "size": 6056, + "url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/.eslintrc.yml?ref=develop", + "html_url": "https://github.com/CityOfNewYork/NYCOpenRecords/blob/develop/.eslintrc.yml", + "git_url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/blobs/a6a0ce9c44238e06ff55ca335a66a4e16810da38", + "download_url": "https://raw.githubusercontent.com/CityOfNewYork/NYCOpenRecords/develop/.eslintrc.yml", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/.eslintrc.yml?ref=develop", + "git": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/blobs/a6a0ce9c44238e06ff55ca335a66a4e16810da38", + "html": "https://github.com/CityOfNewYork/NYCOpenRecords/blob/develop/.eslintrc.yml" + } + }, + { + "name": ".gitignore", + "path": ".gitignore", + "sha": "93923ac3a463bd17d0aefca2de9d2810f243d747", + "size": 1073, + "url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/.gitignore?ref=develop", + "html_url": "https://github.com/CityOfNewYork/NYCOpenRecords/blob/develop/.gitignore", + "git_url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/blobs/93923ac3a463bd17d0aefca2de9d2810f243d747", + "download_url": "https://raw.githubusercontent.com/CityOfNewYork/NYCOpenRecords/develop/.gitignore", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/.gitignore?ref=develop", + "git": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/blobs/93923ac3a463bd17d0aefca2de9d2810f243d747", + "html": "https://github.com/CityOfNewYork/NYCOpenRecords/blob/develop/.gitignore" + } + }, + { + "name": "LICENSE.md", + "path": "LICENSE.md", + "sha": "8dada3edaf50dbc082c9a125058f25def75e625a", + "size": 11357, + "url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/LICENSE.md?ref=develop", + "html_url": "https://github.com/CityOfNewYork/NYCOpenRecords/blob/develop/LICENSE.md", + "git_url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/blobs/8dada3edaf50dbc082c9a125058f25def75e625a", + "download_url": "https://raw.githubusercontent.com/CityOfNewYork/NYCOpenRecords/develop/LICENSE.md", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/LICENSE.md?ref=develop", + "git": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/blobs/8dada3edaf50dbc082c9a125058f25def75e625a", + "html": "https://github.com/CityOfNewYork/NYCOpenRecords/blob/develop/LICENSE.md" + } + }, + { + "name": "Procfile", + "path": "Procfile", + "sha": "72e4ef1be7c64eb0f874344bc8a6cf859bd39e00", + "size": 26, + "url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/Procfile?ref=develop", + "html_url": "https://github.com/CityOfNewYork/NYCOpenRecords/blob/develop/Procfile", + "git_url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/blobs/72e4ef1be7c64eb0f874344bc8a6cf859bd39e00", + "download_url": "https://raw.githubusercontent.com/CityOfNewYork/NYCOpenRecords/develop/Procfile", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/Procfile?ref=develop", + "git": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/blobs/72e4ef1be7c64eb0f874344bc8a6cf859bd39e00", + "html": "https://github.com/CityOfNewYork/NYCOpenRecords/blob/develop/Procfile" + } + }, + { + "name": "README.md", + "path": "README.md", + "sha": "01aae7d562fe164d2cee644e8ef5fba82d2b8e81", + "size": 1589, + "url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/README.md?ref=develop", + "html_url": "https://github.com/CityOfNewYork/NYCOpenRecords/blob/develop/README.md", + "git_url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/blobs/01aae7d562fe164d2cee644e8ef5fba82d2b8e81", + "download_url": "https://raw.githubusercontent.com/CityOfNewYork/NYCOpenRecords/develop/README.md", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/README.md?ref=develop", + "git": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/blobs/01aae7d562fe164d2cee644e8ef5fba82d2b8e81", + "html": "https://github.com/CityOfNewYork/NYCOpenRecords/blob/develop/README.md" + } + }, + { + "name": "Vagrantfile.example", + "path": "Vagrantfile.example", + "sha": "54be29ea9e1e2ecdbfa642656cded8b3cb85c910", + "size": 4329, + "url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/Vagrantfile.example?ref=develop", + "html_url": "https://github.com/CityOfNewYork/NYCOpenRecords/blob/develop/Vagrantfile.example", + "git_url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/blobs/54be29ea9e1e2ecdbfa642656cded8b3cb85c910", + "download_url": "https://raw.githubusercontent.com/CityOfNewYork/NYCOpenRecords/develop/Vagrantfile.example", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/Vagrantfile.example?ref=develop", + "git": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/blobs/54be29ea9e1e2ecdbfa642656cded8b3cb85c910", + "html": "https://github.com/CityOfNewYork/NYCOpenRecords/blob/develop/Vagrantfile.example" + } + }, + { + "name": "app", + "path": "app", + "sha": "f9e84e24364972f3aa4f92b869a622db1f260fa1", + "size": 0, + "url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/app?ref=develop", + "html_url": "https://github.com/CityOfNewYork/NYCOpenRecords/tree/develop/app", + "git_url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/trees/f9e84e24364972f3aa4f92b869a622db1f260fa1", + "download_url": null, + "type": "dir", + "_links": { + "self": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/app?ref=develop", + "git": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/trees/f9e84e24364972f3aa4f92b869a622db1f260fa1", + "html": "https://github.com/CityOfNewYork/NYCOpenRecords/tree/develop/app" + } + }, + { + "name": "build_scripts", + "path": "build_scripts", + "sha": "be8e80f054e47f0c9a6fc9cfe5061346c6f8b2d0", + "size": 0, + "url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/build_scripts?ref=develop", + "html_url": "https://github.com/CityOfNewYork/NYCOpenRecords/tree/develop/build_scripts", + "git_url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/trees/be8e80f054e47f0c9a6fc9cfe5061346c6f8b2d0", + "download_url": null, + "type": "dir", + "_links": { + "self": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/build_scripts?ref=develop", + "git": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/trees/be8e80f054e47f0c9a6fc9cfe5061346c6f8b2d0", + "html": "https://github.com/CityOfNewYork/NYCOpenRecords/tree/develop/build_scripts" + } + }, + { + "name": "celery_worker.py", + "path": "celery_worker.py", + "sha": "c84c73c1f1c1aa42550a08ae4f32dc955ced5f18", + "size": 279, + "url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/celery_worker.py?ref=develop", + "html_url": "https://github.com/CityOfNewYork/NYCOpenRecords/blob/develop/celery_worker.py", + "git_url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/blobs/c84c73c1f1c1aa42550a08ae4f32dc955ced5f18", + "download_url": "https://raw.githubusercontent.com/CityOfNewYork/NYCOpenRecords/develop/celery_worker.py", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/celery_worker.py?ref=develop", + "git": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/blobs/c84c73c1f1c1aa42550a08ae4f32dc955ced5f18", + "html": "https://github.com/CityOfNewYork/NYCOpenRecords/blob/develop/celery_worker.py" + } + }, + { + "name": "config.py", + "path": "config.py", + "sha": "2886556fc4844e708472a99a85b42b4f5b51d509", + "size": 8250, + "url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/config.py?ref=develop", + "html_url": "https://github.com/CityOfNewYork/NYCOpenRecords/blob/develop/config.py", + "git_url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/blobs/2886556fc4844e708472a99a85b42b4f5b51d509", + "download_url": "https://raw.githubusercontent.com/CityOfNewYork/NYCOpenRecords/develop/config.py", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/config.py?ref=develop", + "git": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/blobs/2886556fc4844e708472a99a85b42b4f5b51d509", + "html": "https://github.com/CityOfNewYork/NYCOpenRecords/blob/develop/config.py" + } + }, + { + "name": "crontab", + "path": "crontab", + "sha": "1e2c1da613e8272df6ece759565524c93bc76780", + "size": 300, + "url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/crontab?ref=develop", + "html_url": "https://github.com/CityOfNewYork/NYCOpenRecords/blob/develop/crontab", + "git_url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/blobs/1e2c1da613e8272df6ece759565524c93bc76780", + "download_url": "https://raw.githubusercontent.com/CityOfNewYork/NYCOpenRecords/develop/crontab", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/crontab?ref=develop", + "git": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/blobs/1e2c1da613e8272df6ece759565524c93bc76780", + "html": "https://github.com/CityOfNewYork/NYCOpenRecords/blob/develop/crontab" + } + }, + { + "name": "data", + "path": "data", + "sha": "7bceca86ff0e88cb53c8828d441f5e0725776395", + "size": 0, + "url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/data?ref=develop", + "html_url": "https://github.com/CityOfNewYork/NYCOpenRecords/tree/develop/data", + "git_url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/trees/7bceca86ff0e88cb53c8828d441f5e0725776395", + "download_url": null, + "type": "dir", + "_links": { + "self": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/data?ref=develop", + "git": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/trees/7bceca86ff0e88cb53c8828d441f5e0725776395", + "html": "https://github.com/CityOfNewYork/NYCOpenRecords/tree/develop/data" + } + }, + { + "name": "gunicorn_config.py", + "path": "gunicorn_config.py", + "sha": "7b1a455133c2e78611550e45e8d2072d5c5c4ad7", + "size": 2504, + "url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/gunicorn_config.py?ref=develop", + "html_url": "https://github.com/CityOfNewYork/NYCOpenRecords/blob/develop/gunicorn_config.py", + "git_url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/blobs/7b1a455133c2e78611550e45e8d2072d5c5c4ad7", + "download_url": "https://raw.githubusercontent.com/CityOfNewYork/NYCOpenRecords/develop/gunicorn_config.py", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/gunicorn_config.py?ref=develop", + "git": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/blobs/7b1a455133c2e78611550e45e8d2072d5c5c4ad7", + "html": "https://github.com/CityOfNewYork/NYCOpenRecords/blob/develop/gunicorn_config.py" + } + }, + { + "name": "jobs.py", + "path": "jobs.py", + "sha": "79f2957e2c94e44786f6ca3dfc7384aaceb80c0a", + "size": 6213, + "url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/jobs.py?ref=develop", + "html_url": "https://github.com/CityOfNewYork/NYCOpenRecords/blob/develop/jobs.py", + "git_url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/blobs/79f2957e2c94e44786f6ca3dfc7384aaceb80c0a", + "download_url": "https://raw.githubusercontent.com/CityOfNewYork/NYCOpenRecords/develop/jobs.py", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/jobs.py?ref=develop", + "git": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/blobs/79f2957e2c94e44786f6ca3dfc7384aaceb80c0a", + "html": "https://github.com/CityOfNewYork/NYCOpenRecords/blob/develop/jobs.py" + } + }, + { + "name": "magic", + "path": "magic", + "sha": "6342094c1d4b1af948867ffba67cf0b866e5613c", + "size": 659195, + "url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/magic?ref=develop", + "html_url": "https://github.com/CityOfNewYork/NYCOpenRecords/blob/develop/magic", + "git_url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/blobs/6342094c1d4b1af948867ffba67cf0b866e5613c", + "download_url": "https://raw.githubusercontent.com/CityOfNewYork/NYCOpenRecords/develop/magic", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/magic?ref=develop", + "git": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/blobs/6342094c1d4b1af948867ffba67cf0b866e5613c", + "html": "https://github.com/CityOfNewYork/NYCOpenRecords/blob/develop/magic" + } + }, + { + "name": "manage.py", + "path": "manage.py", + "sha": "a0b80d4c7b5fd7e50d6dcb5bcbdcd995b8da27f0", + "size": 15323, + "url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/manage.py?ref=develop", + "html_url": "https://github.com/CityOfNewYork/NYCOpenRecords/blob/develop/manage.py", + "git_url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/blobs/a0b80d4c7b5fd7e50d6dcb5bcbdcd995b8da27f0", + "download_url": "https://raw.githubusercontent.com/CityOfNewYork/NYCOpenRecords/develop/manage.py", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/manage.py?ref=develop", + "git": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/blobs/a0b80d4c7b5fd7e50d6dcb5bcbdcd995b8da27f0", + "html": "https://github.com/CityOfNewYork/NYCOpenRecords/blob/develop/manage.py" + } + }, + { + "name": "migrations", + "path": "migrations", + "sha": "da206a9809330d01f4800718bca2a20c05038b44", + "size": 0, + "url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/migrations?ref=develop", + "html_url": "https://github.com/CityOfNewYork/NYCOpenRecords/tree/develop/migrations", + "git_url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/trees/da206a9809330d01f4800718bca2a20c05038b44", + "download_url": null, + "type": "dir", + "_links": { + "self": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/migrations?ref=develop", + "git": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/trees/da206a9809330d01f4800718bca2a20c05038b44", + "html": "https://github.com/CityOfNewYork/NYCOpenRecords/tree/develop/migrations" + } + }, + { + "name": "setup.cfg", + "path": "setup.cfg", + "sha": "88b4e0a1c8093fae2b4fa52534035f9f85ed0956", + "size": 2433, + "url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/setup.cfg?ref=develop", + "html_url": "https://github.com/CityOfNewYork/NYCOpenRecords/tree/develop/setup.cfg", + "git_url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/trees/88b4e0a1c8093fae2b4fa52534035f9f85ed0956", + "download_url": "https://raw.githubusercontent.com/CityOfNewYork/NYCOpenRecords/develop/setup.cfg", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/setup.cfg?ref=develop", + "git": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/trees/88b4e0a1c8093fae2b4fa52534035f9f85ed0956", + "html": "https://github.com/CityOfNewYork/NYCOpenRecords/tree/develop/setup.cfg" + } + }, + { + "name": "tests", + "path": "tests", + "sha": "88b4e0a1c8093fae2b4fa52534035f9f85ed0956", + "size": 0, + "url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/tests?ref=develop", + "html_url": "https://github.com/CityOfNewYork/NYCOpenRecords/tree/develop/tests", + "git_url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/trees/88b4e0a1c8093fae2b4fa52534035f9f85ed0956", + "download_url": null, + "type": "dir", + "_links": { + "self": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/tests?ref=develop", + "git": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/trees/88b4e0a1c8093fae2b4fa52534035f9f85ed0956", + "html": "https://github.com/CityOfNewYork/NYCOpenRecords/tree/develop/tests" + } + }, + { + "name": "todo.txt", + "path": "todo.txt", + "sha": "3e369beef1c9e64f0c10735d35aa8ad755daddc6", + "size": 1147, + "url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/todo.txt?ref=develop", + "html_url": "https://github.com/CityOfNewYork/NYCOpenRecords/blob/develop/todo.txt", + "git_url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/blobs/3e369beef1c9e64f0c10735d35aa8ad755daddc6", + "download_url": "https://raw.githubusercontent.com/CityOfNewYork/NYCOpenRecords/develop/todo.txt", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/todo.txt?ref=develop", + "git": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/blobs/3e369beef1c9e64f0c10735d35aa8ad755daddc6", + "html": "https://github.com/CityOfNewYork/NYCOpenRecords/blob/develop/todo.txt" + } + }, + { + "name": "update_definitions.sh", + "path": "update_definitions.sh", + "sha": "fbde266ea5ebd519b94d18f8f78ab825b02766df", + "size": 7877, + "url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/update_definitions.sh?ref=develop", + "html_url": "https://github.com/CityOfNewYork/NYCOpenRecords/blob/develop/update_definitions.sh", + "git_url": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/blobs/fbde266ea5ebd519b94d18f8f78ab825b02766df", + "download_url": "https://raw.githubusercontent.com/CityOfNewYork/NYCOpenRecords/develop/update_definitions.sh", + "type": "file", + "_links": { + "self": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/contents/update_definitions.sh?ref=develop", + "git": "https://api.github.com/repos/CityOfNewYork/NYCOpenRecords/git/blobs/fbde266ea5ebd519b94d18f8f78ab825b02766df", + "html": "https://github.com/CityOfNewYork/NYCOpenRecords/blob/develop/update_definitions.sh" + } + } +] diff --git a/python/spec/fixtures/github/setup_cfg_content.json b/python/spec/fixtures/github/setup_cfg_content.json new file mode 100644 index 00000000000..16e45106e5e --- /dev/null +++ b/python/spec/fixtures/github/setup_cfg_content.json @@ -0,0 +1,18 @@ +{ + "name": "setup.cfg", + "path": "setup.cfg", + "sha": "a99ba2c724c136387dc17eefcda2f9c003b5d705", + "size": 29, + "url": "https://api.github.com/repos/gocardless/bump/contents/setup.cfg?ref=master", + "html_url": "https://github.com/gocardless/bump/blob/master/setup.cfg", + "git_url": "https://api.github.com/repos/gocardless/bump/git/blobs/a99ba2c724c136387dc17eefcda2f9c003b5d705", + "download_url": "https://raw.githubusercontent.com/gocardless/bump/master/setup.cfg?token=ABMwe8vXxd4DBCEsxz3jMusk5sr_fFsoks5WJWE-wA%3D%3D", + "type": "file", + "content": "W21ldGFkYXRhXQpuYW1lID0gcHl0aG9uLXBhY2thZ2UKdmVyc2lvbiA9IDAuMApkZXNjcmlwdGlv\nbiA9IEV4YW1wbGUgc2V0dXAuY2ZnCnVybCA9IGh0dG9zOi8vZ2l0aHViLmNvbS9leGFtcGxlL3B5\ndGhvbi1wYWNrYWdlCmF1dGhvciA9IERlcGVuZGFib3QKCltvcHRpb25zXQpwYWNrYWdlcyA9IGZp\nbmQ6CnNldHVwX3JlcXVpcmVzID0KICAgIG51bXB5PT0xLjExLjAKICAgIHB5dGVzdC1ydW5uZXIK\naW5zdGFsbF9yZXF1aXJlcyA9CiAgICBib3RvMz09MS4zLjEKICAgIGZsYWtlOCA+IDIuNS40LCA8\nIDMuMC4wCiAgICBnb2NhcmRsZXNzX3BybwogICAgcGFuZGFzPT0wLjE5LjIKICAgIHBlcDg9PTEu\nNy4wCiAgICBwc3ljb3BnMj09Mi42LjEKICAgIHJhdmVuID09IDUuMzIuMAogICAgcmVxdWVzdHM9\nPTIuMTIuKgogICAgc2NpcHk9PTAuMTguMQogICAgc2Npa2l0LWxlYXJuPT0wLjE4LjEKCnRlc3Rz\nX3JlcXVpcmUgPQogICAgcHl0ZXN0PT0yLjkuMQogICAgcmVzcG9uc2VzPT0wLjUuMQoKW29wdGlv\nbnMuZXh0cmFzX3JlcXVpcmVdCkFQSSA9IGZsYXNrPT0wLjEyLjIK\n", + "encoding": "base64", + "_links": { + "self": "https://api.github.com/repos/gocardless/bump/contents/setup.cfg?ref=master", + "git": "https://api.github.com/repos/gocardless/bump/git/blobs/a99ba2c724c136387dc17eefcda2f9c003b5d705", + "html": "https://github.com/gocardless/bump/blob/master/setup.cfg" + } +} From 5aec97701e2db1aa4eb259a5653fa0d6b4856ed7 Mon Sep 17 00:00:00 2001 From: Hongxin Liang Date: Fri, 2 Apr 2021 10:12:45 +0200 Subject: [PATCH 10/23] Fix typo --- python/lib/dependabot/python/file_fetcher.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lib/dependabot/python/file_fetcher.rb b/python/lib/dependabot/python/file_fetcher.rb index 8dbf2d416aa..11fcad79d1d 100644 --- a/python/lib/dependabot/python/file_fetcher.rb +++ b/python/lib/dependabot/python/file_fetcher.rb @@ -27,7 +27,7 @@ def self.required_files_in?(filenames) return true if filenames.include?("setup.py") - filename.include?("setup.cfg") + filenames.include?("setup.cfg") end def self.required_files_message From 48f13541aa09f274352c3030c9211c78fa0c5f26 Mon Sep 17 00:00:00 2001 From: Hongxin Liang Date: Fri, 2 Apr 2021 11:58:05 +0200 Subject: [PATCH 11/23] Fix test cases --- python/helpers/lib/parser.py | 55 +++++---- python/lib/dependabot/python/file_fetcher.rb | 8 +- .../file_parser/setup_cfg_file_parser.rb | 17 +-- .../python/file_parser/setup_file_parser.rb | 89 +-------------- .../file_parser/setup_file_parser_base.rb | 104 ++++++++++++++++++ .../file_parser/setup_cfg_file_parser_spec.rb | 2 +- python/spec/fixtures/setup_files/extras.cfg | 1 + .../fixtures/setup_files/illformed_req.cfg | 1 + python/spec/fixtures/setup_files/markers.cfg | 1 + .../fixtures/setup_files/no_tests_require.cfg | 1 + 10 files changed, 151 insertions(+), 128 deletions(-) create mode 100644 python/lib/dependabot/python/file_parser/setup_file_parser_base.rb diff --git a/python/helpers/lib/parser.py b/python/helpers/lib/parser.py index d0dfdde66be..82ca4d9a81e 100644 --- a/python/helpers/lib/parser.py +++ b/python/helpers/lib/parser.py @@ -168,29 +168,36 @@ def parse_requirements(requires, req_type): for req in requires: parse_requirement(req, req_type) - config = configparser.ConfigParser() - config.read(directory + "/setup.cfg") - - for req_type in [ - "setup_requires", - "install_requires", - "tests_require", - ]: - requires = [ - r - for r in config.get("options", req_type, fallback="") - .strip() - .split("\n") - if r != "" - ] - parse_requirements(requires, req_type) - - extras_require_section = "options.extras_require" - if config.has_section(extras_require_section): - section = config[extras_require_section] - for key, value in section.items(): - parse_requirements( - value.strip().split("\n"), "extras_require:{}".format(key) - ) + try: + config = configparser.ConfigParser() + # preserve case + config.optionxform = str + config.read(directory + "/setup.cfg") + + for req_type in [ + "setup_requires", + "install_requires", + "tests_require", + ]: + requires = [ + r + for r in config.get("options", req_type, fallback="") + .strip() + .split("\n") + if r != "" + ] + parse_requirements(requires, req_type) + + extras_require_section = "options.extras_require" + if config.has_section(extras_require_section): + section = config[extras_require_section] + for key, value in section.items(): + parse_requirements( + value.strip().split("\n"), + "extras_require:{}".format(key), + ) + except Exception as e: + print(json.dumps({"error": repr(e)})) + exit(1) return json.dumps({"result": setup_packages}) diff --git a/python/lib/dependabot/python/file_fetcher.rb b/python/lib/dependabot/python/file_fetcher.rb index 11fcad79d1d..cff7bdef11e 100644 --- a/python/lib/dependabot/python/file_fetcher.rb +++ b/python/lib/dependabot/python/file_fetcher.rb @@ -47,7 +47,7 @@ def fetch_files fetched_files += requirement_files if requirements_txt_files.any? fetched_files << setup_file if setup_file - fetched_files << setup_cfg if setup_cfg + fetched_files << setup_cfg_file if setup_cfg_file fetched_files += path_setup_files fetched_files << pip_conf if pip_conf fetched_files << python_version if python_version @@ -79,7 +79,7 @@ def requirement_files end def check_required_files_present - return if requirements_txt_files.any? || setup_file || pipfile || pyproject + return if requirements_txt_files.any? || setup_file || setup_cfg_file || pipfile || pyproject path = Pathname.new(File.join(directory, "requirements.txt")). cleanpath.to_path @@ -90,8 +90,8 @@ def setup_file @setup_file ||= fetch_file_if_present("setup.py") end - def setup_cfg - @setup_cfg ||= fetch_file_if_present("setup.cfg") + def setup_cfg_file + @setup_cfg_file ||= fetch_file_if_present("setup.cfg") end def pip_conf diff --git a/python/lib/dependabot/python/file_parser/setup_cfg_file_parser.rb b/python/lib/dependabot/python/file_parser/setup_cfg_file_parser.rb index b653f0326d9..6038d1c3289 100644 --- a/python/lib/dependabot/python/file_parser/setup_cfg_file_parser.rb +++ b/python/lib/dependabot/python/file_parser/setup_cfg_file_parser.rb @@ -1,24 +1,15 @@ # frozen_string_literal: true -require "dependabot/shared_helpers" -require "dependabot/python/native_helpers" -require "dependabot/python/file_parser/setup_file_parser" +require_relative "./setup_file_parser_base" module Dependabot module Python class FileParser - class SetupCfgFileParser < Dependabot::Python::FileParser::SetupFileParser + class SetupCfgFileParser < SetupFileParserBase private - def parsed_setup_file - requirements = SharedHelpers.run_helper_subprocess( - command: "pyenv exec python #{NativeHelpers.python_helper_path}", - function: "parse_setup_cfg", - args: [Dir.pwd] - ) - - check_requirements(requirements) - requirements + def function + "parse_setup_cfg" end end end diff --git a/python/lib/dependabot/python/file_parser/setup_file_parser.rb b/python/lib/dependabot/python/file_parser/setup_file_parser.rb index ea9f4900791..14d168394da 100644 --- a/python/lib/dependabot/python/file_parser/setup_file_parser.rb +++ b/python/lib/dependabot/python/file_parser/setup_file_parser.rb @@ -1,17 +1,11 @@ # frozen_string_literal: true -require "dependabot/dependency" -require "dependabot/errors" -require "dependabot/file_parsers/base/dependency_set" -require "dependabot/shared_helpers" -require "dependabot/python/file_parser" -require "dependabot/python/native_helpers" -require "dependabot/python/name_normaliser" +require_relative "./setup_file_parser_base" module Dependabot module Python class FileParser - class SetupFileParser + class SetupFileParser < SetupFileParserBase INSTALL_REQUIRES_REGEX = /install_requires\s*=\s*\[/m.freeze SETUP_REQUIRES_REGEX = /setup_requires\s*=\s*\[/m.freeze TESTS_REQUIRE_REGEX = /tests_require\s*=\s*\[/m.freeze @@ -19,67 +13,14 @@ class SetupFileParser CLOSING_BRACKET = { "[" => "]", "{" => "}" }.freeze - def initialize(dependency_files:) - @dependency_files = dependency_files - end - - def dependency_set - dependencies = Dependabot::FileParsers::Base::DependencySet.new - - parsed_setup_file.each do |dep| - # If a requirement has a `<` or `<=` marker then updating it is - # probably blocked. Ignore it. - next if dep["markers"].include?("<") - - # If the requirement is our inserted version, ignore it - # (we wouldn't be able to update it) - next if dep["version"] == "0.0.1+dependabot" - - dependencies << - Dependency.new( - name: normalised_name(dep["name"], dep["extras"]), - version: dep["version"]&.include?("*") ? nil : dep["version"], - requirements: [{ - requirement: dep["requirement"], - file: Pathname.new(dep["file"]).cleanpath.to_path, - source: nil, - groups: [dep["requirement_type"]] - }], - package_manager: "pip" - ) - end - dependencies - end - private - attr_reader :dependency_files - - def parsed_setup_file - SharedHelpers.in_a_temporary_directory do - write_temporary_dependency_files - - requirements = SharedHelpers.run_helper_subprocess( - command: "pyenv exec python #{NativeHelpers.python_helper_path}", - function: "parse_setup", - args: [Dir.pwd] - ) - - check_requirements(requirements) - requirements - end - rescue SharedHelpers::HelperSubprocessFailed => e - raise Dependabot::DependencyFileNotEvaluatable, e.message if e.message.start_with?("InstallationError") - - parsed_sanitized_setup_file - end - def parsed_sanitized_setup_file SharedHelpers.in_a_temporary_directory do write_sanitized_setup_file requirements = SharedHelpers.run_helper_subprocess( - command: "pyenv exec python #{NativeHelpers.python_helper_path}", + command: "python #{NativeHelpers.python_helper_path}", function: "parse_setup", args: [Dir.pwd] ) @@ -95,26 +36,6 @@ def parsed_sanitized_setup_file [] end - def check_requirements(requirements) - requirements.each do |dep| - next unless dep["requirement"] - - Python::Requirement.new(dep["requirement"].split(",")) - rescue Gem::Requirement::BadRequirementError => e - raise Dependabot::DependencyFileNotEvaluatable, e.message - end - end - - def write_temporary_dependency_files - dependency_files. - reject { |f| f.name == ".python-version" }. - each do |file| - path = file.name - FileUtils.mkdir_p(Pathname.new(path).dirname) - File.write(path, file.content) - end - end - # Write a setup.py with only entries for the requires fields. # # This sanitization is far from perfect (it will fail if any of the @@ -162,10 +83,6 @@ def closing_bracket_index(string, bracket) 0 end - def normalised_name(name, extras) - NameNormaliser.normalise_including_extras(name, extras) - end - def setup_file dependency_files.find { |f| f.name == "setup.py" } end diff --git a/python/lib/dependabot/python/file_parser/setup_file_parser_base.rb b/python/lib/dependabot/python/file_parser/setup_file_parser_base.rb new file mode 100644 index 00000000000..45a682d7fe7 --- /dev/null +++ b/python/lib/dependabot/python/file_parser/setup_file_parser_base.rb @@ -0,0 +1,104 @@ +# frozen_string_literal: true + +require "dependabot/dependency" +require "dependabot/errors" +require "dependabot/file_parsers/base/dependency_set" +require "dependabot/shared_helpers" +require "dependabot/python/requirement" +require "dependabot/python/native_helpers" +require "dependabot/python/name_normaliser" + +module Dependabot + module Python + class FileParser + class SetupFileParserBase + def initialize(dependency_files:) + @dependency_files = dependency_files + end + + def dependency_set + dependencies = Dependabot::FileParsers::Base::DependencySet.new + + parsed_setup_file.each do |dep| + # If a requirement has a `<` or `<=` marker then updating it is + # probably blocked. Ignore it. + next if dep["markers"].include?("<") + + # If the requirement is our inserted version, ignore it + # (we wouldn't be able to update it) + next if dep["version"] == "0.0.1+dependabot" + + dependencies << + Dependency.new( + name: normalised_name(dep["name"], dep["extras"]), + version: dep["version"]&.include?("*") ? nil : dep["version"], + requirements: [{ + requirement: dep["requirement"], + file: Pathname.new(dep["file"]).cleanpath.to_path, + source: nil, + groups: [dep["requirement_type"]] + }], + package_manager: "pip" + ) + end + dependencies + end + + private + + attr_reader :dependency_files + + def function + "parse_setup" + end + + def parsed_setup_file + SharedHelpers.in_a_temporary_directory do + write_temporary_dependency_files + + requirements = SharedHelpers.run_helper_subprocess( + command: "python #{NativeHelpers.python_helper_path}", + function: function, + args: [Dir.pwd] + ) + + check_requirements(requirements) + requirements + end + rescue SharedHelpers::HelperSubprocessFailed => e + raise Dependabot::DependencyFileNotEvaluatable, e.message if e.message.start_with?("InstallationError") + + parsed_sanitized_setup_file + end + + def parsed_sanitized_setup_file + [] + end + + def check_requirements(requirements) + requirements.each do |dep| + next unless dep["requirement"] + + Python::Requirement.new(dep["requirement"].split(",")) + rescue Gem::Requirement::BadRequirementError => e + raise Dependabot::DependencyFileNotEvaluatable, e.message + end + end + + def write_temporary_dependency_files + dependency_files. + reject { |f| f.name == ".python-version" }. + each do |file| + path = file.name + FileUtils.mkdir_p(Pathname.new(path).dirname) + File.write(path, file.content) + end + end + + def normalised_name(name, extras) + NameNormaliser.normalise_including_extras(name, extras) + end + end + end + end +end diff --git a/python/spec/dependabot/python/file_parser/setup_cfg_file_parser_spec.rb b/python/spec/dependabot/python/file_parser/setup_cfg_file_parser_spec.rb index 856285ad1ee..2c0548bbde7 100644 --- a/python/spec/dependabot/python/file_parser/setup_cfg_file_parser_spec.rb +++ b/python/spec/dependabot/python/file_parser/setup_cfg_file_parser_spec.rb @@ -110,7 +110,7 @@ expect(error.class). to eq(Dependabot::DependencyFileNotEvaluatable) expect(error.message). - to eq('Illformed requirement ["==2.6.1raven==5.32.0"]') + to eq("InstallationError(\"Invalid requirement: 'psycopg2==2.6.1raven == 5.32.0'\")") end end end diff --git a/python/spec/fixtures/setup_files/extras.cfg b/python/spec/fixtures/setup_files/extras.cfg index 6f8cc195d67..1b5c2078a8d 100644 --- a/python/spec/fixtures/setup_files/extras.cfg +++ b/python/spec/fixtures/setup_files/extras.cfg @@ -1,3 +1,4 @@ +[metadata] name = python-package version = 0.0 description = Example setup.cfg diff --git a/python/spec/fixtures/setup_files/illformed_req.cfg b/python/spec/fixtures/setup_files/illformed_req.cfg index 842e8f8dc41..a07d919e87f 100644 --- a/python/spec/fixtures/setup_files/illformed_req.cfg +++ b/python/spec/fixtures/setup_files/illformed_req.cfg @@ -1,3 +1,4 @@ +[metadata] name = python-package version = 0.0 description = Example setup.cfg diff --git a/python/spec/fixtures/setup_files/markers.cfg b/python/spec/fixtures/setup_files/markers.cfg index bc2451b074e..0163ee9ac6b 100644 --- a/python/spec/fixtures/setup_files/markers.cfg +++ b/python/spec/fixtures/setup_files/markers.cfg @@ -1,3 +1,4 @@ +[metadata] name = python-package version = 0.0 description = Example setup.cfg diff --git a/python/spec/fixtures/setup_files/no_tests_require.cfg b/python/spec/fixtures/setup_files/no_tests_require.cfg index e63d47289c4..dcc0de1ce0d 100644 --- a/python/spec/fixtures/setup_files/no_tests_require.cfg +++ b/python/spec/fixtures/setup_files/no_tests_require.cfg @@ -1,3 +1,4 @@ +[metadata] name = python-package version = 0.0 description = Example setup.cfg From bc0c4c735eff346830782b76aa3c1754d692c105 Mon Sep 17 00:00:00 2001 From: Hongxin Liang Date: Fri, 2 Apr 2021 12:12:33 +0200 Subject: [PATCH 12/23] Restore temp hack --- .../lib/dependabot/python/file_parser/setup_file_parser_base.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lib/dependabot/python/file_parser/setup_file_parser_base.rb b/python/lib/dependabot/python/file_parser/setup_file_parser_base.rb index 45a682d7fe7..b40e607371f 100644 --- a/python/lib/dependabot/python/file_parser/setup_file_parser_base.rb +++ b/python/lib/dependabot/python/file_parser/setup_file_parser_base.rb @@ -57,7 +57,7 @@ def parsed_setup_file write_temporary_dependency_files requirements = SharedHelpers.run_helper_subprocess( - command: "python #{NativeHelpers.python_helper_path}", + command: "pyenv exec python #{NativeHelpers.python_helper_path}", function: function, args: [Dir.pwd] ) From 4859f069d4abd0cedd5378a0125c9cab889d0a31 Mon Sep 17 00:00:00 2001 From: Hongxin Liang Date: Fri, 2 Apr 2021 12:31:21 +0200 Subject: [PATCH 13/23] Test file updater --- .../file_parser/setup_cfg_file_parser.rb | 2 +- .../python/file_parser/setup_file_parser.rb | 2 +- .../requirement_file_updater_spec.rb | 107 ++++++++++++++++++ 3 files changed, 109 insertions(+), 2 deletions(-) diff --git a/python/lib/dependabot/python/file_parser/setup_cfg_file_parser.rb b/python/lib/dependabot/python/file_parser/setup_cfg_file_parser.rb index 6038d1c3289..bebe1b7c574 100644 --- a/python/lib/dependabot/python/file_parser/setup_cfg_file_parser.rb +++ b/python/lib/dependabot/python/file_parser/setup_cfg_file_parser.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require_relative "./setup_file_parser_base" +require_relative "setup_file_parser_base" module Dependabot module Python diff --git a/python/lib/dependabot/python/file_parser/setup_file_parser.rb b/python/lib/dependabot/python/file_parser/setup_file_parser.rb index 14d168394da..dd5652e7e2e 100644 --- a/python/lib/dependabot/python/file_parser/setup_file_parser.rb +++ b/python/lib/dependabot/python/file_parser/setup_file_parser.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require_relative "./setup_file_parser_base" +require_relative "setup_file_parser_base" module Dependabot module Python diff --git a/python/spec/dependabot/python/file_updater/requirement_file_updater_spec.rb b/python/spec/dependabot/python/file_updater/requirement_file_updater_spec.rb index 531975abc3d..909531cc83e 100644 --- a/python/spec/dependabot/python/file_updater/requirement_file_updater_spec.rb +++ b/python/spec/dependabot/python/file_updater/requirement_file_updater_spec.rb @@ -434,6 +434,113 @@ end end + context "with only a setup.cfg" do + subject(:updated_setup_cfg_file) do + updated_files.find { |f| f.name == "setup.cfg" } + end + let(:dependency_files) { [setup_cfg] } + let(:setup_cfg) do + Dependabot::DependencyFile.new( + content: fixture("setup_files", "setup_with_requires.cfg"), + name: "setup.cfg" + ) + end + let(:dependency) do + Dependabot::Dependency.new( + name: "psycopg2", + version: "2.8.1", + requirements: [{ + file: "setup.cfg", + requirement: "==2.8.1", + groups: [], + source: nil + }], + previous_requirements: [{ + file: "setup.cfg", + requirement: "==2.6.1", + groups: [], + source: nil + }], + package_manager: "pip" + ) + end + + its(:content) { is_expected.to include "psycopg2==2.8.1\n" } + its(:content) { is_expected.to include "pep8==1.7.0" } + + context "with non-standard formatting" do + let(:dependency) do + Dependabot::Dependency.new( + name: "raven", + version: "5.34.0", + requirements: [{ + file: "setup.cfg", + requirement: "==5.34.0", + groups: [], + source: nil + }], + previous_requirements: [{ + file: "setup.cfg", + requirement: "==5.32.0", + groups: [], + source: nil + }], + package_manager: "pip" + ) + end + + its(:content) { is_expected.to include "raven == 5.34.0\n" } + end + + context "with a prefix-matcher" do + let(:dependency) do + Dependabot::Dependency.new( + name: "requests", + version: nil, + requirements: [{ + file: "setup.cfg", + requirement: "==2.13.*", + groups: [], + source: nil + }], + previous_requirements: [{ + file: "setup.cfg", + requirement: "==2.12.*", + groups: [], + source: nil + }], + package_manager: "pip" + ) + end + + its(:content) { is_expected.to include "requests==2.13.*\n" } + end + + context "with a range requirement" do + let(:dependency) do + Dependabot::Dependency.new( + name: "flake8", + version: nil, + requirements: [{ + file: "setup.cfg", + requirement: ">2.5.4,<3.4.0", + groups: [], + source: nil + }], + previous_requirements: [{ + file: "setup.cfg", + requirement: "<3.0.0,>2.5.4", + groups: [], + source: nil + }], + package_manager: "pip" + ) + end + + its(:content) { is_expected.to include "flake8 > 2.5.4, < 3.4.0\n" } + end + end + context "when the dependency is in constraints.txt and requirement.txt" do let(:dependency_files) { [requirements, constraints] } let(:requirements_fixture_name) { "specific_with_constraints.txt" } From 484dcd62274d0208b2cb606739fe4a7d3de4ac8e Mon Sep 17 00:00:00 2001 From: Hongxin Liang Date: Fri, 2 Apr 2021 12:38:54 +0200 Subject: [PATCH 14/23] Test updater --- .../requirements_updater_spec.rb | 98 ++++++++++++++++++- 1 file changed, 97 insertions(+), 1 deletion(-) diff --git a/python/spec/dependabot/python/update_checker/requirements_updater_spec.rb b/python/spec/dependabot/python/update_checker/requirements_updater_spec.rb index ff00393f0f0..afb4cb99962 100644 --- a/python/spec/dependabot/python/update_checker/requirements_updater_spec.rb +++ b/python/spec/dependabot/python/update_checker/requirements_updater_spec.rb @@ -14,7 +14,7 @@ end let(:update_strategy) { :bump_versions } - let(:requirements) { [requirement_txt_req, setup_py_req].compact } + let(:requirements) { [requirement_txt_req, setup_py_req, setup_cfg_req].compact } let(:requirement_txt_req) do { file: "requirements.txt", @@ -31,8 +31,17 @@ source: nil } end + let(:setup_cfg_req) do + { + file: "setup.cfg", + requirement: setup_cfg_req_string, + groups: [], + source: nil + } + end let(:requirement_txt_req_string) { "==1.4.0" } let(:setup_py_req_string) { ">= 1.4.0" } + let(:setup_cfg_req_string) { ">= 1.4.0" } let(:has_lockfile) { true } let(:latest_resolvable_version) { "1.5.0" } @@ -253,6 +262,93 @@ end end + context "for a setup.cfg dependency" do + subject { updated_requirements.find { |r| r[:file] == "setup.cfg" } } + + context "when there is no resolvable version" do + let(:latest_resolvable_version) { nil } + it { is_expected.to eq(setup_cfg_req) } + end + + context "when there is a resolvable version" do + let(:latest_resolvable_version) { "1.5.0" } + + context "and a full version was previously pinned" do + let(:setup_cfg_req_string) { "==1.4.0" } + its([:requirement]) { is_expected.to eq("==1.5.0") } + + context "that has fewer digits than the new version" do + let(:setup_cfg_req_string) { "==1.4.0" } + let(:latest_resolvable_version) { "1.5.0.1" } + its([:requirement]) { is_expected.to eq("==1.5.0.1") } + end + + context "without leading == (technically invalid)" do + let(:setup_cfg_req_string) { "1.4.0" } + its([:requirement]) { is_expected.to eq("1.5.0") } + end + end + + context "and no requirement was specified" do + let(:setup_cfg_req_string) { nil } + it { is_expected.to eq(setup_cfg_req) } + end + + context "and a range requirement was specified" do + let(:setup_cfg_req_string) { ">=1.3.0" } + it { is_expected.to eq(setup_cfg_req) } + + context "that is too high" do + let(:setup_cfg_req_string) { ">=2.0.0" } + its([:requirement]) { is_expected.to eq(:unfixable) } + end + + context "with an upper bound" do + let(:setup_cfg_req_string) { ">=1.3.0, <=1.5.0" } + it { is_expected.to eq(setup_cfg_req) } + + context "that needs updating" do + let(:setup_cfg_req_string) { ">=1.3.0, <1.5" } + its([:requirement]) { is_expected.to eq(">=1.3.0,<1.6") } + end + end + end + + context "and a compatibility requirement was specified" do + let(:setup_cfg_req_string) { "~=1.3.0" } + its([:requirement]) { is_expected.to eq(">=1.3,<1.6") } + + context "that supports the new version" do + let(:setup_cfg_req_string) { "~=1.3" } + it { is_expected.to eq(setup_cfg_req) } + end + + context "that needs to be updated and maintain its precision" do + let(:setup_cfg_req_string) { "~=1.3" } + let(:latest_resolvable_version) { "2.1.0" } + its([:requirement]) { is_expected.to eq(">=1.3,<3.0") } + end + end + + context "and a prefix match was specified" do + context "that is satisfied" do + let(:setup_cfg_req_string) { "==1.*.*" } + it { is_expected.to eq(setup_cfg_req) } + end + + context "that needs updating" do + let(:setup_cfg_req_string) { "==1.4.*" } + its([:requirement]) { is_expected.to eq(">=1.4,<1.6") } + end + + context "along with an exact match" do + let(:setup_cfg_req_string) { "==1.4.*, ==1.4.1" } + its([:requirement]) { is_expected.to eq("==1.5.0") } + end + end + end + end + context "for a pyproject.toml dependency" do let(:requirements) { [pyproject_req].compact } let(:pyproject_req) do From c7642e440dc11ed462a2d3215d763561ae6079af Mon Sep 17 00:00:00 2001 From: Hongxin Liang Date: Fri, 2 Apr 2021 15:38:25 +0200 Subject: [PATCH 15/23] Disable other test suite temporarily --- .github/workflows/ci.yml | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 75e87b886f5..cc13a404097 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,25 +16,7 @@ jobs: fail-fast: false matrix: suite: - - { path: bundler, name: bundler1 } - - { path: bundler, name: bundler2 } - - { path: cargo, name: cargo } - - { path: common, name: common } - - { path: composer, name: composer } - - { path: dep, name: dep } - - { path: docker, name: docker } - - { path: elm, name: elm } - - { path: git_submodules, name: git_submodules } - - { path: github_actions, name: github_actions } - - { path: go_modules, name: go_modules } - - { path: gradle, name: gradle } - - { path: hex, name: hex } - - { path: maven, name: maven } - - { path: npm_and_yarn, name: npm_and_yarn } - - { path: nuget, name: nuget } - - { path: omnibus, name: omnibus } - { path: python, name: python } - - { path: terraform, name: terraform } steps: - name: Checkout code uses: actions/checkout@v2 From 67bec2790873046134a75f5d7a0fd95f5a601cba Mon Sep 17 00:00:00 2001 From: Hongxin Liang Date: Fri, 2 Apr 2021 15:56:50 +0200 Subject: [PATCH 16/23] Restore temporary hack --- python/lib/dependabot/python/file_parser/setup_file_parser.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lib/dependabot/python/file_parser/setup_file_parser.rb b/python/lib/dependabot/python/file_parser/setup_file_parser.rb index dd5652e7e2e..84cf0e61b90 100644 --- a/python/lib/dependabot/python/file_parser/setup_file_parser.rb +++ b/python/lib/dependabot/python/file_parser/setup_file_parser.rb @@ -20,7 +20,7 @@ def parsed_sanitized_setup_file write_sanitized_setup_file requirements = SharedHelpers.run_helper_subprocess( - command: "python #{NativeHelpers.python_helper_path}", + command: "pyenv exec python #{NativeHelpers.python_helper_path}", function: "parse_setup", args: [Dir.pwd] ) From 3d2e76a4566239af8912b76beec8c20604c8276b Mon Sep 17 00:00:00 2001 From: Hongxin Liang Date: Fri, 2 Apr 2021 18:00:51 +0200 Subject: [PATCH 17/23] Revert "Disable other test suite temporarily" This reverts commit d4f8c264f4c5a41b2e4879eeecf25438a32c634b. --- .github/workflows/ci.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cc13a404097..75e87b886f5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,25 @@ jobs: fail-fast: false matrix: suite: + - { path: bundler, name: bundler1 } + - { path: bundler, name: bundler2 } + - { path: cargo, name: cargo } + - { path: common, name: common } + - { path: composer, name: composer } + - { path: dep, name: dep } + - { path: docker, name: docker } + - { path: elm, name: elm } + - { path: git_submodules, name: git_submodules } + - { path: github_actions, name: github_actions } + - { path: go_modules, name: go_modules } + - { path: gradle, name: gradle } + - { path: hex, name: hex } + - { path: maven, name: maven } + - { path: npm_and_yarn, name: npm_and_yarn } + - { path: nuget, name: nuget } + - { path: omnibus, name: omnibus } - { path: python, name: python } + - { path: terraform, name: terraform } steps: - name: Checkout code uses: actions/checkout@v2 From 1277169b1514d27767cce7c589cc25536c3fe4b4 Mon Sep 17 00:00:00 2001 From: Hongxin Liang Date: Tue, 6 Apr 2021 09:14:48 +0200 Subject: [PATCH 18/23] Use setuptools to parse setup.cfg --- python/helpers/lib/parser.py | 31 +++++++------------ .../file_parser/setup_cfg_file_parser_spec.rb | 2 +- 2 files changed, 13 insertions(+), 20 deletions(-) diff --git a/python/helpers/lib/parser.py b/python/helpers/lib/parser.py index 82ca4d9a81e..1b24842df05 100644 --- a/python/helpers/lib/parser.py +++ b/python/helpers/lib/parser.py @@ -169,33 +169,26 @@ def parse_requirements(requires, req_type): parse_requirement(req, req_type) try: - config = configparser.ConfigParser() - # preserve case - config.optionxform = str - config.read(directory + "/setup.cfg") + config = setuptools.config.read_configuration( + directory + "/setup.cfg" + ) for req_type in [ "setup_requires", "install_requires", "tests_require", ]: - requires = [ - r - for r in config.get("options", req_type, fallback="") - .strip() - .split("\n") - if r != "" - ] + requires = config.get("options", {}).get(req_type, []) parse_requirements(requires, req_type) - extras_require_section = "options.extras_require" - if config.has_section(extras_require_section): - section = config[extras_require_section] - for key, value in section.items(): - parse_requirements( - value.strip().split("\n"), - "extras_require:{}".format(key), - ) + extras_require = config.get("options", {}).get( + "extras_require", {} + ) + for key, value in extras_require.items(): + parse_requirements( + value, + "extras_require:{}".format(key), + ) except Exception as e: print(json.dumps({"error": repr(e)})) exit(1) diff --git a/python/spec/dependabot/python/file_parser/setup_cfg_file_parser_spec.rb b/python/spec/dependabot/python/file_parser/setup_cfg_file_parser_spec.rb index 2c0548bbde7..62e86fede55 100644 --- a/python/spec/dependabot/python/file_parser/setup_cfg_file_parser_spec.rb +++ b/python/spec/dependabot/python/file_parser/setup_cfg_file_parser_spec.rb @@ -89,7 +89,7 @@ [{ requirement: "==0.12.2", file: "setup.cfg", - groups: ["extras_require:API"], + groups: ["extras_require:api"], source: nil }] ) From 433fe24cd45c8a61f3e14281b3a3ca38c7f58d3f Mon Sep 17 00:00:00 2001 From: Hongxin Liang Date: Fri, 16 Apr 2021 20:35:54 +0200 Subject: [PATCH 19/23] Merge parse_setup and parse_setup_cfg --- python/helpers/lib/parser.py | 134 +++++++++++++++-------------------- python/helpers/run.py | 2 - 2 files changed, 57 insertions(+), 79 deletions(-) diff --git a/python/helpers/lib/parser.py b/python/helpers/lib/parser.py index 1b24842df05..1f221005b44 100644 --- a/python/helpers/lib/parser.py +++ b/python/helpers/lib/parser.py @@ -1,4 +1,3 @@ -import configparser import glob import io import json @@ -58,38 +57,48 @@ def version_from_install_req(install_req): def parse_setup(directory): - # Parse the setup.py - setup_packages = [] - if os.path.isfile(directory + '/setup.py'): - def version_from_install_req(install_req): - if install_req.is_pinned: - return next(iter(install_req.specifier)).version + def version_from_install_req(install_req): + if install_req.is_pinned: + return next(iter(install_req.specifier)).version - def parse_requirement(req, req_type): - install_req = install_req_from_line(req) - if install_req.original_link: - return + def parse_requirement(req, req_type, filename): + install_req = install_req_from_line(req) + if install_req.original_link: + return - setup_packages.append({ + setup_packages.append( + { "name": install_req.req.name, "version": version_from_install_req(install_req), "markers": str(install_req.markers) or None, - "file": "setup.py", + "file": filename, "requirement": str(install_req.specifier) or None, "requirement_type": req_type, - "extras": sorted(list(install_req.extras)) - }) + "extras": sorted(list(install_req.extras)), + } + ) + + def parse_requirements(requires, req_type, filename): + for req in requires: + parse_requirement(req, req_type, filename) + + # Parse the setup.py and setup.cfg + setup_py = "setup.py" + setup_cfg = "setup.cfg" + setup_packages = [] + + if os.path.isfile(os.path.join(directory, setup_py)): def setup(*args, **kwargs): - for arg in ['setup_requires', 'install_requires', 'tests_require']: - if not kwargs.get(arg): - continue - for req in kwargs.get(arg): - parse_requirement(req, arg) - extras_require_dict = kwargs.get('extras_require', {}) - for key in extras_require_dict: - for req in extras_require_dict[key]: - parse_requirement(req, 'extras_require:{}'.format(key)) + for arg in ["setup_requires", "install_requires", "tests_require"]: + requires = kwargs.get(arg, []) + parse_requirements(requires, arg, setup_py) + extras_require_dict = kwargs.get("extras_require", {}) + for key, value in extras_require_dict.items(): + parse_requirements( + value, "extras_require:{}".format(key), setup_py + ) + setuptools.setup = setup def noop(*args, **kwargs): @@ -101,17 +110,19 @@ def fake_parse(*args, **kwargs): global fake_open def fake_open(*args, **kwargs): - content = ("VERSION = ('0', '0', '1+dependabot')\n" - "__version__ = '0.0.1+dependabot'\n" - "__author__ = 'someone'\n" - "__title__ = 'something'\n" - "__description__ = 'something'\n" - "__author_email__ = 'something'\n" - "__license__ = 'something'\n" - "__url__ = 'something'\n") + content = ( + "VERSION = ('0', '0', '1+dependabot')\n" + "__version__ = '0.0.1+dependabot'\n" + "__author__ = 'someone'\n" + "__title__ = 'something'\n" + "__description__ = 'something'\n" + "__author_email__ = 'something'\n" + "__license__ = 'something'\n" + "__url__ = 'something'\n" + ) return io.StringIO(content) - content = open(directory + '/setup.py', 'r').read() + content = open(directory + "/setup.py", "r").read() # Remove `print`, `open`, `log` and import statements content = re.sub(r"print\s*\(", "noop(", content) @@ -122,55 +133,24 @@ def fake_open(*args, **kwargs): content = re.sub(version_re, "", content) # Set variables likely to be imported - __version__ = '0.0.1+dependabot' - __author__ = 'someone' - __title__ = 'something' - __description__ = 'something' - __author_email__ = 'something' - __license__ = 'something' - __url__ = 'something' + __version__ = "0.0.1+dependabot" + __author__ = "someone" + __title__ = "something" + __description__ = "something" + __author_email__ = "something" + __license__ = "something" + __url__ = "something" # Run as main (since setup.py is a script) - __name__ = '__main__' + __name__ = "__main__" # Exec the setup.py exec(content) in globals(), locals() - return json.dumps({"result": setup_packages}) - - -def parse_setup_cfg(directory): - setup_packages = [] - if os.path.isfile(directory + "/setup.cfg"): - - def version_from_install_req(install_req): - if install_req.is_pinned: - return next(iter(install_req.specifier)).version - - def parse_requirement(req, req_type): - install_req = install_req_from_line(req) - if install_req.original_link: - return - - setup_packages.append( - { - "name": install_req.req.name, - "version": version_from_install_req(install_req), - "markers": str(install_req.markers) or None, - "file": "setup.cfg", - "requirement": str(install_req.specifier) or None, - "requirement_type": req_type, - "extras": sorted(list(install_req.extras)), - } - ) - - def parse_requirements(requires, req_type): - for req in requires: - parse_requirement(req, req_type) - + if os.path.isfile(os.path.join(directory, setup_cfg)): try: config = setuptools.config.read_configuration( - directory + "/setup.cfg" + os.path.join(directory, setup_cfg) ) for req_type in [ @@ -179,18 +159,18 @@ def parse_requirements(requires, req_type): "tests_require", ]: requires = config.get("options", {}).get(req_type, []) - parse_requirements(requires, req_type) + parse_requirements(requires, req_type, setup_cfg) extras_require = config.get("options", {}).get( "extras_require", {} ) for key, value in extras_require.items(): parse_requirements( - value, - "extras_require:{}".format(key), + value, "extras_require:{}".format(key), setup_cfg ) except Exception as e: print(json.dumps({"error": repr(e)})) exit(1) return json.dumps({"result": setup_packages}) + diff --git a/python/helpers/run.py b/python/helpers/run.py index b45c5512cff..26961e502d2 100644 --- a/python/helpers/run.py +++ b/python/helpers/run.py @@ -10,8 +10,6 @@ print(parser.parse_requirements(args["args"][0])) elif args["function"] == "parse_setup": print(parser.parse_setup(args["args"][0])) - elif args["function"] == "parse_setup_cfg": - print(parser.parse_setup_cfg(args["args"][0])) elif args["function"] == "get_dependency_hash": print(hasher.get_dependency_hash(*args["args"])) elif args["function"] == "get_pipfile_hash": From ca4d2921ef24cdc774252e9bbbba8f9f811b5d38 Mon Sep 17 00:00:00 2001 From: Hongxin Liang Date: Fri, 16 Apr 2021 20:59:59 +0200 Subject: [PATCH 20/23] Parse both setup.py and setup.cfg --- python/lib/dependabot/python/file_parser.rb | 11 +- .../python/file_parser/setup_file_parser.rb | 89 ++++- .../file_parser/setup_file_parser_base.rb | 104 ------ .../file_parser/setup_cfg_file_parser_spec.rb | 118 ------ .../file_parser/setup_file_parser_spec.rb | 339 ++++++++++++------ 5 files changed, 314 insertions(+), 347 deletions(-) delete mode 100644 python/lib/dependabot/python/file_parser/setup_file_parser_base.rb delete mode 100644 python/spec/dependabot/python/file_parser/setup_cfg_file_parser_spec.rb diff --git a/python/lib/dependabot/python/file_parser.rb b/python/lib/dependabot/python/file_parser.rb index ae7fd467ff2..4ea73a61f42 100644 --- a/python/lib/dependabot/python/file_parser.rb +++ b/python/lib/dependabot/python/file_parser.rb @@ -17,7 +17,6 @@ class FileParser < Dependabot::FileParsers::Base require_relative "file_parser/pipfile_files_parser" require_relative "file_parser/poetry_files_parser" require_relative "file_parser/setup_file_parser" - require_relative "file_parser/setup_cfg_file_parser" POETRY_DEPENDENCY_TYPES = %w(tool.poetry.dependencies tool.poetry.dev-dependencies).freeze @@ -45,8 +44,7 @@ def parse dependency_set += pipenv_dependencies if pipfile dependency_set += poetry_dependencies if using_poetry? dependency_set += requirement_dependencies if requirement_files.any? - dependency_set += setup_file_dependencies if setup_file - dependency_set += setup_cfg_file_dependencies if setup_cfg_file + dependency_set += setup_file_dependencies if setup_file || setup_cfg_file dependency_set.dependencies end @@ -139,13 +137,6 @@ def setup_file_dependencies dependency_set end - def setup_cfg_file_dependencies - @setup_cfg_file_dependencies ||= - SetupCfgFileParser. - new(dependency_files: dependency_files). - dependency_set - end - def lockfile_for_pip_compile_file?(filename) return false unless pip_compile_files.any? return false unless filename.end_with?(".txt") diff --git a/python/lib/dependabot/python/file_parser/setup_file_parser.rb b/python/lib/dependabot/python/file_parser/setup_file_parser.rb index 84cf0e61b90..73734b6f853 100644 --- a/python/lib/dependabot/python/file_parser/setup_file_parser.rb +++ b/python/lib/dependabot/python/file_parser/setup_file_parser.rb @@ -1,11 +1,17 @@ # frozen_string_literal: true -require_relative "setup_file_parser_base" +require "dependabot/dependency" +require "dependabot/errors" +require "dependabot/file_parsers/base/dependency_set" +require "dependabot/shared_helpers" +require "dependabot/python/file_parser" +require "dependabot/python/native_helpers" +require "dependabot/python/name_normaliser" module Dependabot module Python class FileParser - class SetupFileParser < SetupFileParserBase + class SetupFileParser INSTALL_REQUIRES_REGEX = /install_requires\s*=\s*\[/m.freeze SETUP_REQUIRES_REGEX = /setup_requires\s*=\s*\[/m.freeze TESTS_REQUIRE_REGEX = /tests_require\s*=\s*\[/m.freeze @@ -13,8 +19,63 @@ class SetupFileParser < SetupFileParserBase CLOSING_BRACKET = { "[" => "]", "{" => "}" }.freeze + def initialize(dependency_files:) + @dependency_files = dependency_files + end + + def dependency_set + dependencies = Dependabot::FileParsers::Base::DependencySet.new + + parsed_setup_file.each do |dep| + # If a requirement has a `<` or `<=` marker then updating it is + # probably blocked. Ignore it. + next if dep["markers"].include?("<") + + # If the requirement is our inserted version, ignore it + # (we wouldn't be able to update it) + next if dep["version"] == "0.0.1+dependabot" + + dependencies << + Dependency.new( + name: normalised_name(dep["name"], dep["extras"]), + version: dep["version"]&.include?("*") ? nil : dep["version"], + requirements: [{ + requirement: dep["requirement"], + file: Pathname.new(dep["file"]).cleanpath.to_path, + source: nil, + groups: [dep["requirement_type"]] + }], + package_manager: "pip" + ) + end + dependencies + end + private + attr_reader :dependency_files + + def parsed_setup_file + SharedHelpers.in_a_temporary_directory do + write_temporary_dependency_files + + requirements = SharedHelpers.run_helper_subprocess( + command: "pyenv exec python #{NativeHelpers.python_helper_path}", + function: "parse_setup", + args: [Dir.pwd] + ) + + check_requirements(requirements) + requirements + end + rescue SharedHelpers::HelperSubprocessFailed => e + raise Dependabot::DependencyFileNotEvaluatable, e.message if e.message.start_with?("InstallationError") + + return [] unless setup_file + + parsed_sanitized_setup_file + end + def parsed_sanitized_setup_file SharedHelpers.in_a_temporary_directory do write_sanitized_setup_file @@ -36,6 +97,26 @@ def parsed_sanitized_setup_file [] end + def check_requirements(requirements) + requirements.each do |dep| + next unless dep["requirement"] + + Python::Requirement.new(dep["requirement"].split(",")) + rescue Gem::Requirement::BadRequirementError => e + raise Dependabot::DependencyFileNotEvaluatable, e.message + end + end + + def write_temporary_dependency_files + dependency_files. + reject { |f| f.name == ".python-version" }. + each do |file| + path = file.name + FileUtils.mkdir_p(Pathname.new(path).dirname) + File.write(path, file.content) + end + end + # Write a setup.py with only entries for the requires fields. # # This sanitization is far from perfect (it will fail if any of the @@ -83,6 +164,10 @@ def closing_bracket_index(string, bracket) 0 end + def normalised_name(name, extras) + NameNormaliser.normalise_including_extras(name, extras) + end + def setup_file dependency_files.find { |f| f.name == "setup.py" } end diff --git a/python/lib/dependabot/python/file_parser/setup_file_parser_base.rb b/python/lib/dependabot/python/file_parser/setup_file_parser_base.rb deleted file mode 100644 index b40e607371f..00000000000 --- a/python/lib/dependabot/python/file_parser/setup_file_parser_base.rb +++ /dev/null @@ -1,104 +0,0 @@ -# frozen_string_literal: true - -require "dependabot/dependency" -require "dependabot/errors" -require "dependabot/file_parsers/base/dependency_set" -require "dependabot/shared_helpers" -require "dependabot/python/requirement" -require "dependabot/python/native_helpers" -require "dependabot/python/name_normaliser" - -module Dependabot - module Python - class FileParser - class SetupFileParserBase - def initialize(dependency_files:) - @dependency_files = dependency_files - end - - def dependency_set - dependencies = Dependabot::FileParsers::Base::DependencySet.new - - parsed_setup_file.each do |dep| - # If a requirement has a `<` or `<=` marker then updating it is - # probably blocked. Ignore it. - next if dep["markers"].include?("<") - - # If the requirement is our inserted version, ignore it - # (we wouldn't be able to update it) - next if dep["version"] == "0.0.1+dependabot" - - dependencies << - Dependency.new( - name: normalised_name(dep["name"], dep["extras"]), - version: dep["version"]&.include?("*") ? nil : dep["version"], - requirements: [{ - requirement: dep["requirement"], - file: Pathname.new(dep["file"]).cleanpath.to_path, - source: nil, - groups: [dep["requirement_type"]] - }], - package_manager: "pip" - ) - end - dependencies - end - - private - - attr_reader :dependency_files - - def function - "parse_setup" - end - - def parsed_setup_file - SharedHelpers.in_a_temporary_directory do - write_temporary_dependency_files - - requirements = SharedHelpers.run_helper_subprocess( - command: "pyenv exec python #{NativeHelpers.python_helper_path}", - function: function, - args: [Dir.pwd] - ) - - check_requirements(requirements) - requirements - end - rescue SharedHelpers::HelperSubprocessFailed => e - raise Dependabot::DependencyFileNotEvaluatable, e.message if e.message.start_with?("InstallationError") - - parsed_sanitized_setup_file - end - - def parsed_sanitized_setup_file - [] - end - - def check_requirements(requirements) - requirements.each do |dep| - next unless dep["requirement"] - - Python::Requirement.new(dep["requirement"].split(",")) - rescue Gem::Requirement::BadRequirementError => e - raise Dependabot::DependencyFileNotEvaluatable, e.message - end - end - - def write_temporary_dependency_files - dependency_files. - reject { |f| f.name == ".python-version" }. - each do |file| - path = file.name - FileUtils.mkdir_p(Pathname.new(path).dirname) - File.write(path, file.content) - end - end - - def normalised_name(name, extras) - NameNormaliser.normalise_including_extras(name, extras) - end - end - end - end -end diff --git a/python/spec/dependabot/python/file_parser/setup_cfg_file_parser_spec.rb b/python/spec/dependabot/python/file_parser/setup_cfg_file_parser_spec.rb deleted file mode 100644 index 62e86fede55..00000000000 --- a/python/spec/dependabot/python/file_parser/setup_cfg_file_parser_spec.rb +++ /dev/null @@ -1,118 +0,0 @@ -# frozen_string_literal: true - -require "spec_helper" -require "dependabot/dependency_file" -require "dependabot/python/file_parser/setup_cfg_file_parser" - -RSpec.describe Dependabot::Python::FileParser::SetupCfgFileParser do - let(:parser) { described_class.new(dependency_files: files) } - - let(:files) { [setup_cfg_file] } - let(:setup_cfg_file) do - Dependabot::DependencyFile.new( - name: "setup.cfg", - content: setup_cfg_file_body - ) - end - let(:setup_cfg_file_body) do - fixture("setup_files", setup_cfg_file_fixture_name) - end - let(:setup_cfg_file_fixture_name) { "setup_with_requires.cfg" } - - describe "parse" do - subject(:dependencies) { parser.dependency_set.dependencies } - - its(:length) { is_expected.to eq(15) } - - describe "an install_requires dependencies" do - subject(:dependency) { dependencies.find { |d| d.name == "boto3" } } - - it "has the right details" do - expect(dependency).to be_a(Dependabot::Dependency) - expect(dependency.name).to eq("boto3") - expect(dependency.version).to eq("1.3.1") - expect(dependency.requirements).to eq( - [{ - requirement: "==1.3.1", - file: "setup.cfg", - groups: ["install_requires"], - source: nil - }] - ) - end - end - - describe "a setup_requires dependencies" do - subject(:dependency) { dependencies.find { |d| d.name == "numpy" } } - - it "has the right details" do - expect(dependency).to be_a(Dependabot::Dependency) - expect(dependency.name).to eq("numpy") - expect(dependency.version).to eq("1.11.0") - expect(dependency.requirements).to eq( - [{ - requirement: "==1.11.0", - file: "setup.cfg", - groups: ["setup_requires"], - source: nil - }] - ) - end - end - - describe "a tests_require dependencies" do - subject(:dependency) { dependencies.find { |d| d.name == "responses" } } - - it "has the right details" do - expect(dependency).to be_a(Dependabot::Dependency) - expect(dependency.name).to eq("responses") - expect(dependency.version).to eq("0.5.1") - expect(dependency.requirements).to eq( - [{ - requirement: "==0.5.1", - file: "setup.cfg", - groups: ["tests_require"], - source: nil - }] - ) - end - end - - describe "an extras_require dependencies" do - subject(:dependency) { dependencies.find { |d| d.name == "flask" } } - - it "has the right details" do - expect(dependency).to be_a(Dependabot::Dependency) - expect(dependency.name).to eq("flask") - expect(dependency.version).to eq("0.12.2") - expect(dependency.requirements).to eq( - [{ - requirement: "==0.12.2", - file: "setup.cfg", - groups: ["extras_require:api"], - source: nil - }] - ) - end - end - - context "without a `tests_require` key" do - let(:setup_cfg_file_fixture_name) { "no_tests_require.cfg" } - its(:length) { is_expected.to eq(12) } - end - - context "with an illformed_requirement" do - let(:setup_cfg_file_fixture_name) { "illformed_req.cfg" } - - it "raises a helpful error" do - expect { parser.dependency_set }. - to raise_error do |error| - expect(error.class). - to eq(Dependabot::DependencyFileNotEvaluatable) - expect(error.message). - to eq("InstallationError(\"Invalid requirement: 'psycopg2==2.6.1raven == 5.32.0'\")") - end - end - end - end -end diff --git a/python/spec/dependabot/python/file_parser/setup_file_parser_spec.rb b/python/spec/dependabot/python/file_parser/setup_file_parser_spec.rb index 46e904e8bd7..60676db82bc 100644 --- a/python/spec/dependabot/python/file_parser/setup_file_parser_spec.rb +++ b/python/spec/dependabot/python/file_parser/setup_file_parser_spec.rb @@ -7,144 +7,257 @@ RSpec.describe Dependabot::Python::FileParser::SetupFileParser do let(:parser) { described_class.new(dependency_files: files) } - let(:files) { [setup_file] } - let(:setup_file) do - Dependabot::DependencyFile.new( - name: "setup.py", - content: setup_file_body - ) - end - let(:setup_file_body) do - fixture("setup_files", setup_file_fixture_name) - end - let(:setup_file_fixture_name) { "setup.py" } + describe "for setup.py" do + let(:files) { [setup_file] } + let(:setup_file) do + Dependabot::DependencyFile.new( + name: "setup.py", + content: setup_file_body + ) + end + let(:setup_file_body) do + fixture("setup_files", setup_file_fixture_name) + end + let(:setup_file_fixture_name) { "setup.py" } - describe "parse" do - subject(:dependencies) { parser.dependency_set.dependencies } + describe "parse" do + subject(:dependencies) { parser.dependency_set.dependencies } - its(:length) { is_expected.to eq(15) } + its(:length) { is_expected.to eq(15) } - describe "an install_requires dependencies" do - subject(:dependency) { dependencies.find { |d| d.name == "boto3" } } + describe "an install_requires dependencies" do + subject(:dependency) { dependencies.find { |d| d.name == "boto3" } } - it "has the right details" do - expect(dependency).to be_a(Dependabot::Dependency) - expect(dependency.name).to eq("boto3") - expect(dependency.version).to eq("1.3.1") - expect(dependency.requirements).to eq( - [{ - requirement: "==1.3.1", - file: "setup.py", - groups: ["install_requires"], - source: nil - }] - ) + it "has the right details" do + expect(dependency).to be_a(Dependabot::Dependency) + expect(dependency.name).to eq("boto3") + expect(dependency.version).to eq("1.3.1") + expect(dependency.requirements).to eq( + [{ + requirement: "==1.3.1", + file: "setup.py", + groups: ["install_requires"], + source: nil + }] + ) + end end - end - describe "a setup_requires dependencies" do - subject(:dependency) { dependencies.find { |d| d.name == "numpy" } } - - it "has the right details" do - expect(dependency).to be_a(Dependabot::Dependency) - expect(dependency.name).to eq("numpy") - expect(dependency.version).to eq("1.11.0") - expect(dependency.requirements).to eq( - [{ - requirement: "==1.11.0", - file: "setup.py", - groups: ["setup_requires"], - source: nil - }] - ) + describe "a setup_requires dependencies" do + subject(:dependency) { dependencies.find { |d| d.name == "numpy" } } + + it "has the right details" do + expect(dependency).to be_a(Dependabot::Dependency) + expect(dependency.name).to eq("numpy") + expect(dependency.version).to eq("1.11.0") + expect(dependency.requirements).to eq( + [{ + requirement: "==1.11.0", + file: "setup.py", + groups: ["setup_requires"], + source: nil + }] + ) + end end - end - describe "a tests_require dependencies" do - subject(:dependency) { dependencies.find { |d| d.name == "responses" } } - - it "has the right details" do - expect(dependency).to be_a(Dependabot::Dependency) - expect(dependency.name).to eq("responses") - expect(dependency.version).to eq("0.5.1") - expect(dependency.requirements).to eq( - [{ - requirement: "==0.5.1", - file: "setup.py", - groups: ["tests_require"], - source: nil - }] - ) + describe "a tests_require dependencies" do + subject(:dependency) { dependencies.find { |d| d.name == "responses" } } + + it "has the right details" do + expect(dependency).to be_a(Dependabot::Dependency) + expect(dependency.name).to eq("responses") + expect(dependency.version).to eq("0.5.1") + expect(dependency.requirements).to eq( + [{ + requirement: "==0.5.1", + file: "setup.py", + groups: ["tests_require"], + source: nil + }] + ) + end end - end - describe "an extras_require dependencies" do - subject(:dependency) { dependencies.find { |d| d.name == "flask" } } - - it "has the right details" do - expect(dependency).to be_a(Dependabot::Dependency) - expect(dependency.name).to eq("flask") - expect(dependency.version).to eq("0.12.2") - expect(dependency.requirements).to eq( - [{ - requirement: "==0.12.2", - file: "setup.py", - groups: ["extras_require:API"], - source: nil - }] - ) + describe "an extras_require dependencies" do + subject(:dependency) { dependencies.find { |d| d.name == "flask" } } + + it "has the right details" do + expect(dependency).to be_a(Dependabot::Dependency) + expect(dependency.name).to eq("flask") + expect(dependency.version).to eq("0.12.2") + expect(dependency.requirements).to eq( + [{ + requirement: "==0.12.2", + file: "setup.py", + groups: ["extras_require:API"], + source: nil + }] + ) + end end - end - context "without a `tests_require` key" do - let(:setup_file_fixture_name) { "no_tests_require.py" } - its(:length) { is_expected.to eq(12) } - end + context "without a `tests_require` key" do + let(:setup_file_fixture_name) { "no_tests_require.py" } + its(:length) { is_expected.to eq(12) } + end - context "with a `print` statement" do - let(:setup_file_fixture_name) { "with_print.py" } - its(:length) { is_expected.to eq(14) } - end + context "with a `print` statement" do + let(:setup_file_fixture_name) { "with_print.py" } + its(:length) { is_expected.to eq(14) } + end - context "with an import statements that can't be handled" do - let(:setup_file_fixture_name) { "impossible_imports.py" } - its(:length) { is_expected.to eq(12) } - end + context "with an import statements that can't be handled" do + let(:setup_file_fixture_name) { "impossible_imports.py" } + its(:length) { is_expected.to eq(12) } + end + + context "with an illformed_requirement" do + let(:setup_file_fixture_name) { "illformed_req.py" } + + it "raises a helpful error" do + expect { parser.dependency_set }. + to raise_error do |error| + expect(error.class). + to eq(Dependabot::DependencyFileNotEvaluatable) + expect(error.message). + to eq('Illformed requirement ["==2.6.1raven==5.32.0"]') + end + end + end + + context "with an `open` statement" do + let(:setup_file_fixture_name) { "with_open.py" } + its(:length) { is_expected.to eq(14) } + end + + context "with the setup.py from requests" do + let(:setup_file_fixture_name) { "requests_setup.py" } + its(:length) { is_expected.to eq(13) } + end + + context "with an import of a config file" do + let(:setup_file_fixture_name) { "imports_version.py" } + its(:length) { is_expected.to eq(14) } - context "with an illformed_requirement" do - let(:setup_file_fixture_name) { "illformed_req.py" } + context "with a inserted version" do + let(:setup_file_fixture_name) { "imports_version_for_dep.py" } - it "raises a helpful error" do - expect { parser.dependency_set }. - to raise_error do |error| - expect(error.class). - to eq(Dependabot::DependencyFileNotEvaluatable) - expect(error.message). - to eq('Illformed requirement ["==2.6.1raven==5.32.0"]') + it "excludes the dependency importing a version" do + expect(dependencies.count).to eq(14) + expect(dependencies.map(&:name)).to_not include("acme") end + end end end + end - context "with an `open` statement" do - let(:setup_file_fixture_name) { "with_open.py" } - its(:length) { is_expected.to eq(14) } + describe "for setup.cfg" do + let(:files) { [setup_cfg_file] } + let(:setup_cfg_file) do + Dependabot::DependencyFile.new( + name: "setup.cfg", + content: setup_cfg_file_body + ) end - - context "with the setup.py from requests" do - let(:setup_file_fixture_name) { "requests_setup.py" } - its(:length) { is_expected.to eq(13) } + let(:setup_cfg_file_body) do + fixture("setup_files", setup_cfg_file_fixture_name) end + let(:setup_cfg_file_fixture_name) { "setup_with_requires.cfg" } + + describe "parse" do + subject(:dependencies) { parser.dependency_set.dependencies } + + its(:length) { is_expected.to eq(15) } + + describe "an install_requires dependencies" do + subject(:dependency) { dependencies.find { |d| d.name == "boto3" } } + + it "has the right details" do + expect(dependency).to be_a(Dependabot::Dependency) + expect(dependency.name).to eq("boto3") + expect(dependency.version).to eq("1.3.1") + expect(dependency.requirements).to eq( + [{ + requirement: "==1.3.1", + file: "setup.cfg", + groups: ["install_requires"], + source: nil + }] + ) + end + end - context "with an import of a config file" do - let(:setup_file_fixture_name) { "imports_version.py" } - its(:length) { is_expected.to eq(14) } + describe "a setup_requires dependencies" do + subject(:dependency) { dependencies.find { |d| d.name == "numpy" } } + + it "has the right details" do + expect(dependency).to be_a(Dependabot::Dependency) + expect(dependency.name).to eq("numpy") + expect(dependency.version).to eq("1.11.0") + expect(dependency.requirements).to eq( + [{ + requirement: "==1.11.0", + file: "setup.cfg", + groups: ["setup_requires"], + source: nil + }] + ) + end + end + + describe "a tests_require dependencies" do + subject(:dependency) { dependencies.find { |d| d.name == "responses" } } + + it "has the right details" do + expect(dependency).to be_a(Dependabot::Dependency) + expect(dependency.name).to eq("responses") + expect(dependency.version).to eq("0.5.1") + expect(dependency.requirements).to eq( + [{ + requirement: "==0.5.1", + file: "setup.cfg", + groups: ["tests_require"], + source: nil + }] + ) + end + end + + describe "an extras_require dependencies" do + subject(:dependency) { dependencies.find { |d| d.name == "flask" } } + + it "has the right details" do + expect(dependency).to be_a(Dependabot::Dependency) + expect(dependency.name).to eq("flask") + expect(dependency.version).to eq("0.12.2") + expect(dependency.requirements).to eq( + [{ + requirement: "==0.12.2", + file: "setup.cfg", + groups: ["extras_require:api"], + source: nil + }] + ) + end + end + + context "without a `tests_require` key" do + let(:setup_cfg_file_fixture_name) { "no_tests_require.cfg" } + its(:length) { is_expected.to eq(12) } + end - context "with a inserted version" do - let(:setup_file_fixture_name) { "imports_version_for_dep.py" } + context "with an illformed_requirement" do + let(:setup_cfg_file_fixture_name) { "illformed_req.cfg" } - it "excludes the dependency importing a version" do - expect(dependencies.count).to eq(14) - expect(dependencies.map(&:name)).to_not include("acme") + it "raises a helpful error" do + expect { parser.dependency_set }. + to raise_error do |error| + expect(error.class). + to eq(Dependabot::DependencyFileNotEvaluatable) + expect(error.message). + to eq("InstallationError(\"Invalid requirement: 'psycopg2==2.6.1raven == 5.32.0'\")") + end end end end From f5c96b94ea9437d0e7c4aacc011dc80482638ed9 Mon Sep 17 00:00:00 2001 From: Hongxin Liang Date: Fri, 16 Apr 2021 21:02:41 +0200 Subject: [PATCH 21/23] Remove empty line --- python/helpers/lib/parser.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/helpers/lib/parser.py b/python/helpers/lib/parser.py index 1f221005b44..b67f376522e 100644 --- a/python/helpers/lib/parser.py +++ b/python/helpers/lib/parser.py @@ -173,4 +173,3 @@ def fake_open(*args, **kwargs): exit(1) return json.dumps({"result": setup_packages}) - From dd4669c467dfab2b994e41d79deab9f40a20d1b4 Mon Sep 17 00:00:00 2001 From: Hongxin Liang Date: Fri, 16 Apr 2021 21:03:17 +0200 Subject: [PATCH 22/23] Delete unused class --- .../python/file_parser/setup_cfg_file_parser.rb | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 python/lib/dependabot/python/file_parser/setup_cfg_file_parser.rb diff --git a/python/lib/dependabot/python/file_parser/setup_cfg_file_parser.rb b/python/lib/dependabot/python/file_parser/setup_cfg_file_parser.rb deleted file mode 100644 index bebe1b7c574..00000000000 --- a/python/lib/dependabot/python/file_parser/setup_cfg_file_parser.rb +++ /dev/null @@ -1,17 +0,0 @@ -# frozen_string_literal: true - -require_relative "setup_file_parser_base" - -module Dependabot - module Python - class FileParser - class SetupCfgFileParser < SetupFileParserBase - private - - def function - "parse_setup_cfg" - end - end - end - end -end From 9fd5b108b1eb0848934514fb2c7da66855aae676 Mon Sep 17 00:00:00 2001 From: Hongxin Liang Date: Fri, 16 Apr 2021 21:10:39 +0200 Subject: [PATCH 23/23] Simplify path --- python/helpers/lib/parser.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/python/helpers/lib/parser.py b/python/helpers/lib/parser.py index b67f376522e..7bc39c4943d 100644 --- a/python/helpers/lib/parser.py +++ b/python/helpers/lib/parser.py @@ -84,10 +84,12 @@ def parse_requirements(requires, req_type, filename): # Parse the setup.py and setup.cfg setup_py = "setup.py" + setup_py_path = os.path.join(directory, setup_py) setup_cfg = "setup.cfg" + setup_cfg_path = os.path.join(directory, setup_cfg) setup_packages = [] - if os.path.isfile(os.path.join(directory, setup_py)): + if os.path.isfile(setup_py_path): def setup(*args, **kwargs): for arg in ["setup_requires", "install_requires", "tests_require"]: @@ -122,7 +124,7 @@ def fake_open(*args, **kwargs): ) return io.StringIO(content) - content = open(directory + "/setup.py", "r").read() + content = open(setup_py_path, "r").read() # Remove `print`, `open`, `log` and import statements content = re.sub(r"print\s*\(", "noop(", content) @@ -147,11 +149,9 @@ def fake_open(*args, **kwargs): # Exec the setup.py exec(content) in globals(), locals() - if os.path.isfile(os.path.join(directory, setup_cfg)): + if os.path.isfile(setup_cfg_path): try: - config = setuptools.config.read_configuration( - os.path.join(directory, setup_cfg) - ) + config = setuptools.config.read_configuration(setup_cfg_path) for req_type in [ "setup_requires",