Merge pull request #199 from openedx/usamasadiq/bom-2133-requirement-…

…files-structure

fix: standardise requirement files structure in cookiecutter templates

CHANGELOG.rst

@@ -5,6 +5,14 @@ Change Log
    This file loosely adheres to the structure of https://keepachangelog.com/,
    but in reStructuredText instead of Markdown.
 
+2022-07-13
+----------
+
+Fixed
+~~~~~
+
+- Standardised the Requirements file structure in all templates.
+
 2022-07-12
 ----------
 

cookiecutter-django-app/{{cookiecutter.repo_name}}/Makefile

@@ -1,6 +1,6 @@
 .PHONY: clean compile_translations coverage diff_cover docs dummy_translations \
         extract_translations fake_translations help pii_check pull_translations push_translations \
-        quality requirements selfcheck test test-all upgrade validate
+        quality requirements selfcheck test test-all upgrade validate install_transifex_client
 
 .DEFAULT_GOAL := help
 
@@ -30,13 +30,16 @@ docs: ## generate Sphinx HTML documentation, including API docs
 	$(BROWSER)docs/_build/html/index.html
 
 # Define PIP_COMPILE_OPTS=-v to get more information during make upgrade.
-PIP_COMPILE = pip-compile --rebuild --upgrade $(PIP_COMPILE_OPTS)
+PIP_COMPILE = pip-compile --upgrade $(PIP_COMPILE_OPTS)
 
 upgrade: export CUSTOM_COMPILE_COMMAND=make upgrade
 upgrade: ## update the requirements/*.txt files with the latest packages satisfying requirements/*.in
 	pip install -qr requirements/pip-tools.txt
 	# Make sure to compile files after any other files they include!
+	$(PIP_COMPILE) --allow-unsafe -o requirements/pip.txt requirements/pip.in
 	$(PIP_COMPILE) -o requirements/pip-tools.txt requirements/pip-tools.in
+	pip install -qr requirements/pip.txt
+	pip install -qr requirements/pip-tools.txt
 	$(PIP_COMPILE) -o requirements/base.txt requirements/base.in
 	$(PIP_COMPILE) -o requirements/test.txt requirements/test.in
 	$(PIP_COMPILE) -o requirements/doc.txt requirements/doc.in
@@ -100,3 +103,10 @@ dummy_translations: ## generate dummy translation (.po) files
 build_dummy_translations: extract_translations dummy_translations compile_translations ## generate and compile dummy translation files
 
 validate_translations: build_dummy_translations detect_changed_source_translations ## validate translations
+
+install_transifex_client: ## Install the Transifex client
+	# Instaling client will skip CHANGELOG and LICENSE files from git changes
+	# so remind the user to commit the change first before installing client.
+	git diff -s --exit-code HEAD || { echo "Please commit changes first."; exit 1; }
+	curl -o- https://raw.githubusercontent.com/transifex/cli/master/install.sh | bash
+	git checkout -- LICENSE README.md ## overwritten by Transifex installer

cookiecutter-django-app/{{cookiecutter.repo_name}}/requirements/ci.in

@@ -0,0 +1,7 @@
+# Requirements for running tests in CI
+
+-c constraints.txt
+
+codecov                   # Code coverage reporting
+tox                       # Virtualenv management for tests
+tox-battery               # Makes tox aware of requirements file changes

cookiecutter-django-app/{{cookiecutter.repo_name}}/requirements/dev.in

@@ -7,4 +7,3 @@
 
 diff-cover                # Changeset diff test coverage
 edx-i18n-tools            # For i18n_tool dummy
-tox-battery               # Makes tox aware of requirements file changes

cookiecutter-django-app/{{cookiecutter.repo_name}}/requirements/doc.in

@@ -0,0 +1,10 @@
+# Requirements for documentation validation
+
+-c constraints.txt
+
+-r test.txt               # Core and testing dependencies for this package
+
+doc8                      # reStructuredText style checker
+twine                     # Validates README.rst for usage on PyPI
+build                     # Needed to build the wheel for twine README check
+Sphinx                    # Documentation builder

cookiecutter-django-app/{{cookiecutter.repo_name}}/requirements/pip-tools.in

@@ -0,0 +1,5 @@
+# Just the dependencies to run pip-tools, mainly for the "upgrade" make target
+
+-c constraints.txt
+
+pip-tools                 # Contains pip-compile, used to generate pip requirements files

cookiecutter-django-app/{{cookiecutter.repo_name}}/requirements/pip.in

@@ -0,0 +1,6 @@
+# Core dependencies for installing other packages
+-c constraints.txt
+
+pip
+setuptools
+wheel

cookiecutter-django-app/{{cookiecutter.repo_name}}/requirements/quality.in

@@ -0,0 +1,10 @@
+# Requirements for code quality checks
+
+-c constraints.txt
+
+-r test.txt               # Core and testing dependencies for this package
+
+edx-lint                  # edX pylint rules and plugins
+isort                     # to standardize order of imports
+pycodestyle               # PEP 8 compliance validation
+pydocstyle                # PEP 257 compliance validation

cookiecutter-django-ida/{{cookiecutter.repo_name}}/Makefile

@@ -1,6 +1,6 @@
 .DEFAULT_GOAL := help
 
-.PHONY: help clean piptools requirements ci_requirements dev_requirements \
+.PHONY: help clean requirements ci_requirements dev_requirements \
         validation_requirements doc_requirements prod_requirements static shell \
         test coverage isort_check isort style lint quality pii_check validate \
         migrate html_coverage upgrade extract_translation dummy_translations \
@@ -32,14 +32,10 @@ clean: ## delete generated byte code and coverage reports
 	rm -rf assets
 	rm -rf pii_report
 
-piptools: ## install pinned version of pip-compile and pip-sync
-	pip install -r requirements/pip.txt
-	pip install -r requirements/pip-tools.txt
-
 clean_pycrypto: ## temporary (?) hack to deal with the pycrypto dep that's installed via setup-tools
 	ls -d /usr/lib/python3/dist-packages/* | grep 'pycrypto\|pygobject\|pyxdg' | xargs rm -f
 
-requirements: clean_pycrypto piptools dev_requirements ## sync to default requirements
+requirements: clean_pycrypto dev_requirements ## sync to default requirements
 
 ci_requirements: validation_requirements ## sync to requirements needed for CI checks
 
@@ -99,17 +95,25 @@ migrate: ## apply database migrations
 html_coverage: ## generate and view HTML coverage report
 	coverage html && open htmlcov/index.html
 
+# Define PIP_COMPILE_OPTS=-v to get more information during make upgrade.
+PIP_COMPILE = pip-compile --upgrade $(PIP_COMPILE_OPTS)
+
 upgrade: export CUSTOM_COMPILE_COMMAND=make upgrade
-upgrade: piptools ## update the requirements/*.txt files with the latest packages satisfying requirements/*.in
+upgrade: ## update the requirements/*.txt files with the latest packages satisfying requirements/*.in
+	pip install -qr requirements/pip-tools.txt
 	# Make sure to compile files after any other files they include!
-	pip-compile --upgrade -o requirements/pip-tools.txt requirements/pip-tools.in
-	pip-compile --upgrade -o requirements/base.txt requirements/base.in
-	pip-compile --upgrade -o requirements/test.txt requirements/test.in
-	pip-compile --upgrade -o requirements/doc.txt requirements/doc.in
-	pip-compile --upgrade -o requirements/quality.txt requirements/quality.in
-	pip-compile --upgrade -o requirements/validation.txt requirements/validation.in
-	pip-compile --upgrade -o requirements/dev.txt requirements/dev.in
-	pip-compile --upgrade -o requirements/production.txt requirements/production.in
+	$(PIP_COMPILE) --allow-unsafe -o requirements/pip.txt requirements/pip.in
+	$(PIP_COMPILE) -o requirements/pip-tools.txt requirements/pip-tools.in
+	pip install -qr requirements/pip.txt
+	pip install -qr requirements/pip-tools.txt
+	$(PIP_COMPILE) -o requirements/base.txt requirements/base.in
+	$(PIP_COMPILE) -o requirements/test.txt requirements/test.in
+	$(PIP_COMPILE) -o requirements/doc.txt requirements/doc.in
+	$(PIP_COMPILE) -o requirements/quality.txt requirements/quality.in
+	$(PIP_COMPILE) -o requirements/ci.txt requirements/ci.in
+	$(PIP_COMPILE) -o requirements/dev.txt requirements/dev.in
+	$(PIP_COMPILE) -o requirements/validation.txt requirements/validation.in
+	$(PIP_COMPILE) -o requirements/production.txt requirements/production.in
 	# Let tox control the Django version for tests
 	grep -e "^django==" requirements/base.txt > requirements/django.txt
 	sed '/^[dD]jango==/d' requirements/test.txt > requirements/test.tmp

cookiecutter-django-ida/{{cookiecutter.repo_name}}/requirements/ci.in

@@ -0,0 +1,7 @@
+# Requirements for running tests in CI
+
+-c constraints.txt
+
+codecov                   # Code coverage reporting
+tox                       # Virtualenv management for tests
+tox-battery               # Makes tox aware of requirements file changes

cookiecutter-django-ida/{{cookiecutter.repo_name}}/requirements/doc.in

@@ -0,0 +1,10 @@
+# Requirements for documentation validation
+
+-c constraints.txt
+
+-r test.txt               # Core and testing dependencies for this package
+
+doc8                      # reStructuredText style checker
+twine                     # Validates README.rst for usage on PyPI
+build                     # Needed to build the wheel for twine README check
+Sphinx                    # Documentation builder

cookiecutter-django-ida/{{cookiecutter.repo_name}}/requirements/pip-tools.in

@@ -0,0 +1,5 @@
+# Just the dependencies to run pip-tools, mainly for the "upgrade" make target
+
+-c constraints.txt
+
+pip-tools                 # Contains pip-compile, used to generate pip requirements files

cookiecutter-django-ida/{{cookiecutter.repo_name}}/requirements/pip.in

@@ -0,0 +1,6 @@
+# Core dependencies for installing other packages
+-c constraints.txt
+
+pip
+setuptools
+wheel

cookiecutter-xblock/{{cookiecutter.repo_name}}/Makefile

@@ -22,17 +22,21 @@ PIP_COMPILE = pip-compile --upgrade $(PIP_COMPILE_OPTS)
 
 upgrade: export CUSTOM_COMPILE_COMMAND=make upgrade
 upgrade: ## update the requirements/*.txt files with the latest packages satisfying requirements/*.in
-	pip install -r requirements/pip-tools.txt
+	pip install -qr requirements/pip-tools.txt
 	# Make sure to compile files after any other files they include!
-	$(PIP_COMPILE) --allow-unsafe --rebuild -o requirements/pip.txt requirements/pip.in
+	$(PIP_COMPILE) --allow-unsafe -o requirements/pip.txt requirements/pip.in
 	$(PIP_COMPILE) -o requirements/pip-tools.txt requirements/pip-tools.in
-	pip install -r requirements/pip-tools.txt
+	pip install -qr requirements/pip.txt
+	pip install -qr requirements/pip-tools.txt
 	$(PIP_COMPILE) -o requirements/base.txt requirements/base.in
 	$(PIP_COMPILE) -o requirements/test.txt requirements/test.in
 	$(PIP_COMPILE) -o requirements/doc.txt requirements/doc.in
 	$(PIP_COMPILE) -o requirements/quality.txt requirements/quality.in
 	$(PIP_COMPILE) -o requirements/ci.txt requirements/ci.in
 	$(PIP_COMPILE) -o requirements/dev.txt requirements/dev.in
+	# Let tox control the Django version for tests
+	sed '/^[dD]jango==/d' requirements/test.txt > requirements/test.tmp
+	mv requirements/test.tmp requirements/test.txt
 
 piptools: ## install pinned version of pip-compile and pip-sync
 	pip install -r requirements/pip.txt

cookiecutter-xblock/{{cookiecutter.repo_name}}/requirements/base.in

@@ -6,4 +6,3 @@ edx-i18n-tools
 Mako
 XBlock 
 xblock-utils
-

cookiecutter-xblock/{{cookiecutter.repo_name}}/requirements/ci.in

@@ -0,0 +1,7 @@
+# Requirements for running tests in CI
+
+-c constraints.txt
+
+codecov                   # Code coverage reporting
+tox                       # Virtualenv management for tests
+tox-battery               # Makes tox aware of requirements file changes

cookiecutter-xblock/{{cookiecutter.repo_name}}/requirements/constraints.txt

@@ -0,0 +1,12 @@
+# Version constraints for pip-installation.
+#
+# This file doesn't install any packages. It specifies version constraints
+# that will be applied if a package is needed.
+#
+# When pinning something here, please provide an explanation of why.  Ideally,
+# link to other information that will help people in the future to remove the
+# pin when possible.  Writing an issue against the offending project and
+# linking to it here is good.
+
+# Common constraints for edx repos
+-c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt

cookiecutter-xblock/{{cookiecutter.repo_name}}/requirements/dev.in

@@ -0,0 +1,9 @@
+# Additional requirements for development of this application
+-c constraints.txt
+
+-r pip-tools.txt          # pip-tools and its dependencies, for managing requirements files
+-r quality.txt            # Core and quality check dependencies
+-r ci.txt                 # dependencies for setting up testing in CI
+
+diff-cover                # Changeset diff test coverage
+edx-i18n-tools            # For i18n_tool dummy

cookiecutter-xblock/{{cookiecutter.repo_name}}/requirements/doc.in

@@ -0,0 +1,10 @@
+# Requirements for documentation validation
+
+-c constraints.txt
+
+-r test.txt               # Core and testing dependencies for this package
+
+doc8                      # reStructuredText style checker
+twine                     # Validates README.rst for usage on PyPI
+build                     # Needed to build the wheel for twine README check
+Sphinx                    # Documentation builder

cookiecutter-xblock/{{cookiecutter.repo_name}}/requirements/pip-tools.in

@@ -0,0 +1,5 @@
+# Just the dependencies to run pip-tools, mainly for the "upgrade" make target
+
+-c constraints.txt
+
+pip-tools                 # Contains pip-compile, used to generate pip requirements files

cookiecutter-xblock/{{cookiecutter.repo_name}}/requirements/pip.in

@@ -0,0 +1,6 @@
+# Core dependencies for installing other packages
+-c constraints.txt
+
+pip
+setuptools
+wheel

cookiecutter-xblock/{{cookiecutter.repo_name}}/requirements/quality.in

@@ -0,0 +1,10 @@
+# Requirements for code quality checks
+
+-c constraints.txt
+
+-r test.txt               # Core and testing dependencies for this package
+
+edx-lint                  # edX pylint rules and plugins
+isort                     # to standardize order of imports
+pycodestyle               # PEP 8 compliance validation
+pydocstyle                # PEP 257 compliance validation

cookiecutter-xblock/{{cookiecutter.repo_name}}/requirements/test.in

@@ -0,0 +1,8 @@
+# Requirements for test runs.
+-c constraints.txt
+
+-r base.txt               # Core dependencies for this package
+
+pytest-cov                # pytest extension for code coverage statistics
+pytest-django             # pytest extension for better Django support
+code-annotations          # provides commands used by the pii_check make target.