From 953c27994e0d434f519698dcfdd90e107e5ee4af Mon Sep 17 00:00:00 2001 From: edX requirements bot Date: Tue, 9 Nov 2021 00:41:04 +0500 Subject: [PATCH 1/2] feat: advertise constraints in setup.py --- MANIFEST.in | 1 + setup.py | 68 +++++++++++++++++++++++++++++++++++++++++------------ 2 files changed, 54 insertions(+), 15 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index a69668c..c357a1e 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1,2 @@ include requirements/base.in +include requirements/constraints.txt diff --git a/setup.py b/setup.py index 1e364c5..421f93d 100644 --- a/setup.py +++ b/setup.py @@ -40,29 +40,67 @@ def package_data(pkg, root_list): def load_requirements(*requirements_paths): """ Load all requirements from the specified requirements files. + + Requirements will include any constraints from files specified + with -c in the requirements files. Returns a list of requirement strings. """ - requirements = set() + # UPDATED VIA SEMGREP - if you need to remove/modify this method remove this line and add a comment specifying why. + + requirements = {} + constraint_files = set() + + # groups "my-package-name<=x.y.z,..." into ("my-package-name", "<=x.y.z,...") + requirement_line_regex = re.compile(r"([a-zA-Z0-9-_.]+)([<>=][^#\s]+)?") + + def add_version_constraint_or_raise(current_line, current_requirements, add_if_not_present): + regex_match = requirement_line_regex.match(current_line) + if regex_match: + package = regex_match.group(1) + version_constraints = regex_match.group(2) + existing_version_constraints = current_requirements.get(package, None) + # it's fine to add constraints to an unconstrained package, but raise an error if there are already + # constraints in place + if existing_version_constraints and existing_version_constraints != version_constraints: + raise BaseException(f'Multiple constraint definitions found for {package}:' + f' "{existing_version_constraints}" and "{version_constraints}".' + f'Combine constraints into one location with {package}' + f'{existing_version_constraints},{version_constraints}.') + if add_if_not_present or package in current_requirements: + current_requirements[package] = version_constraints + + # process .in files and store the path to any constraint files that are pulled in for path in requirements_paths: - requirements.update( - line.split('#')[0].strip() for line in open(path).readlines() - if is_requirement(line.strip()) - ) - return list(requirements) + with open(path) as reqs: + for line in reqs: + if is_requirement(line): + add_version_constraint_or_raise(line, requirements, True) + if line and line.startswith('-c') and not line.startswith('-c http'): + constraint_files.add(os.path.dirname(path) + '/' + line.split('#')[0].replace('-c', '').strip()) + + # process constraint files and add any new constraints found to existing requirements + for constraint_file in constraint_files: + with open(constraint_file) as reader: + for line in reader: + if is_requirement(line): + add_version_constraint_or_raise(line, requirements, False) + + # process back into list of pkg><=constraints strings + constrained_requirements = [f'{pkg}{version or ""}' for (pkg, version) in sorted(requirements.items())] + return constrained_requirements def is_requirement(line): """ - Return True if the requirement line is a package requirement; - that is, it is not blank, a comment, a URL, or an included file. + Return True if the requirement line is a package requirement. + + Returns: + bool: True if the line is not blank, a comment, + a URL, or an included file """ - return not ( - line == '' or - line.startswith('-r') or - line.startswith('#') or - line.startswith('-e') or - line.startswith('git+') or - line.startswith('-c') + # UPDATED VIA SEMGREP - if you need to remove/modify this method remove this line and add a comment specifying why + + return line and line.strip() and not line.startswith(('-r', '#', '-e', 'git+', '-c')) ) From ad47132df2c0c761b161fd4e2e2af849b85dc7ef Mon Sep 17 00:00:00 2001 From: Rebecca Graber Date: Mon, 8 Nov 2021 14:56:55 -0500 Subject: [PATCH 2/2] fix: rm extraneous parens --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 421f93d..1efa702 100644 --- a/setup.py +++ b/setup.py @@ -101,7 +101,6 @@ def is_requirement(line): # UPDATED VIA SEMGREP - if you need to remove/modify this method remove this line and add a comment specifying why return line and line.strip() and not line.startswith(('-r', '#', '-e', 'git+', '-c')) - ) def get_version(*file_paths):