build: Deduplicate mismatched pip.txt files that caused build failure (…

…openedx#32081)

There was a `requirements/pip.txt` with old versions, and a newer
`requirements/edx/pip.txt` managed via a `pip.in` file. The old one was
used in most places, but came out of sync with pip-tools.txt, which was
managed properly. Eventually this caused a `pip check` failure due to the
mismatch.

This should resolve at least part of edx/edx-arch-experiments#267

This PR moves pip.in and pip-tools.in and their corresponding pin files
up to the `requirements/` dir, since they should be shared between the edx
and sandbox environments. This also has the effect of upgrading pip to
match the version in the file we've been uselessly upgrading.

Other improvements:

- Remove `-q` option from pip and pip-sync calls, as it was hiding some
  debugging information that would have resolved this sooner.
- Depend on `pre-requirements` from `compile-requirements`, rather than
  from `upgrade`. (The base target is the one that actually needs it.)
  This also lets us remove the explicit `pip install pip-tools` line.
- Install the recompiled pip and pip-tools files right away, not after the
  loop. When we upgrade pip-tools, we want to use the upgraded version,
  not the previous version. This requires moving the pip-tools.txt
  recompilation outside of the loop and into its own explicit line.
- Don't upgrade pip if we're not running `make upgrade` (respect the
  compile options).
- Remove apparently-unneeded `--no-emit-trusted-host --no-emit-index-url`
  options (we don't pass trusted-host or index-url options).

(cherry picked from commit b852344)

Makefile

@@ -64,8 +64,8 @@ pull: ## update the Docker image used by "make shell"
 	docker pull edxops/edxapp:latest
 
 pre-requirements: ## install Python requirements for running pip-tools
-	pip install -qr requirements/pip.txt
-	pip install -qr requirements/edx/pip-tools.txt
+	pip install -r requirements/pip.txt
+	pip install -r requirements/pip-tools.txt
 
 local-requirements:
 # 	edx-platform installs some Python projects from within the edx-platform repo itself.
@@ -74,7 +74,7 @@ local-requirements:
 dev-requirements: pre-requirements
 	@# The "$(wildcard..)" is to include private.txt if it exists, and make no mention
 	@# of it if it does not.  Shell wildcarding can't do that with default options.
-	pip-sync -q requirements/edx/development.txt $(wildcard requirements/edx/private.txt)
+	pip-sync requirements/edx/development.txt $(wildcard requirements/edx/private.txt)
 	make local-requirements
 
 base-requirements: pre-requirements
@@ -96,7 +96,6 @@ shell: ## launch a bash shell in a Docker container with all edx-platform depend
 
 # Order is very important in this list: files must appear after everything they include!
 REQ_FILES = \
-	requirements/edx/pip-tools \
 	requirements/edx/coverage \
 	requirements/edx/doc \
 	requirements/edx/paver \
@@ -117,23 +116,26 @@ $(COMMON_CONSTRAINTS_TXT):
 	echo "$(COMMON_CONSTRAINTS_TEMP_COMMENT)" | cat - $(@) > temp && mv temp $(@)
 
 compile-requirements: export CUSTOM_COMPILE_COMMAND=make upgrade
-compile-requirements: $(COMMON_CONSTRAINTS_TXT) ## Re-compile *.in requirements to *.txt
-	pip install -q pip-tools
-	pip-compile --allow-unsafe --upgrade -o requirements/edx/pip.txt requirements/edx/pip.in
+compile-requirements: pre-requirements $(COMMON_CONSTRAINTS_TXT) ## Re-compile *.in requirements to *.txt
+	@# Bootstrapping: Rebuild pip and pip-tools first, and then install them
+	@# so that if there are any failures we'll know now, rather than the next
+	@# time someone tries to use the outputs.
+	pip-compile -v --allow-unsafe ${COMPILE_OPTS} -o requirements/pip.txt requirements/pip.in
+	pip install -r requirements/pip.txt
+
+	pip-compile -v ${COMPILE_OPTS} -o requirements/pip-tools.txt requirements/pip-tools.in
+	pip install -r requirements/pip-tools.txt
 
 	@ export REBUILD='--rebuild'; \
 	for f in $(REQ_FILES); do \
 		echo ; \
 		echo "== $$f ===============================" ; \
-		echo "pip-compile -v --no-emit-trusted-host --no-emit-index-url $$REBUILD ${COMPILE_OPTS} -o $$f.txt $$f.in"; \
-		pip-compile -v --no-emit-trusted-host --no-emit-index-url $$REBUILD ${COMPILE_OPTS} -o $$f.txt $$f.in || exit 1; \
+		echo "pip-compile -v $$REBUILD ${COMPILE_OPTS} -o $$f.txt $$f.in"; \
+		pip-compile -v $$REBUILD ${COMPILE_OPTS} -o $$f.txt $$f.in || exit 1; \
 		export REBUILD=''; \
 	done
 
-	pip install -qr requirements/edx/pip.txt
-	pip install -qr requirements/edx/pip-tools.txt
-
-upgrade: pre-requirements  ## update the pip requirements files to use the latest releases satisfying our constraints
+upgrade:  ## update the pip requirements files to use the latest releases satisfying our constraints
 	$(MAKE) compile-requirements COMPILE_OPTS="--upgrade"
 
 check-types: ## run static type-checking tests

requirements/edx/development.in

@@ -10,7 +10,7 @@
 
 -c ../constraints.txt
 
--r pip-tools.txt                        # pip-tools and its dependencies, for managing requirements files
+-r ../pip-tools.txt                     # pip-tools and its dependencies, for managing requirements files
 -r testing.txt                          # Dependencies for running the various test suites
 
 click                                   # Used for perf_tests utilities in modulestore

requirements/edx/development.txt

@@ -125,7 +125,7 @@ bridgekeeper==0.9
     # via -r requirements/edx/testing.txt
 build==0.10.0
     # via
-    #   -r requirements/edx/pip-tools.txt
+    #   -r requirements/edx/../pip-tools.txt
     #   pip-tools
 celery==5.2.7
     # via
@@ -168,8 +168,8 @@ chem==1.2.0
 click==8.1.3
     # via
     #   -c requirements/edx/../constraints.txt
+    #   -r requirements/edx/../pip-tools.txt
     #   -r requirements/edx/development.in
-    #   -r requirements/edx/pip-tools.txt
     #   -r requirements/edx/testing.txt
     #   celery
     #   click-didyoumean
@@ -1047,7 +1047,7 @@ outcome-surveys==2.4.0
     # via -r requirements/edx/testing.txt
 packaging==23.0
     # via
-    #   -r requirements/edx/pip-tools.txt
+    #   -r requirements/edx/../pip-tools.txt
     #   -r requirements/edx/testing.txt
     #   build
     #   drf-yasg
@@ -1093,7 +1093,7 @@ pillow==9.4.0
     #   edx-enterprise
     #   edx-organizations
 pip-tools==6.12.3
-    # via -r requirements/edx/pip-tools.txt
+    # via -r requirements/edx/../pip-tools.txt
 pkgutil-resolve-name==1.3.10
     # via
     #   -r requirements/edx/testing.txt
@@ -1237,7 +1237,7 @@ pyparsing==3.0.9
     #   openedx-calc
 pyproject-hooks==1.0.0
     # via
-    #   -r requirements/edx/pip-tools.txt
+    #   -r requirements/edx/../pip-tools.txt
     #   build
 pyquery==2.0.0
     # via -r requirements/edx/testing.txt
@@ -1584,7 +1584,7 @@ toml==0.10.2
     # via vulture
 tomli==2.0.1
     # via
-    #   -r requirements/edx/pip-tools.txt
+    #   -r requirements/edx/../pip-tools.txt
     #   -r requirements/edx/testing.txt
     #   build
     #   coverage
@@ -1691,7 +1691,7 @@ webob==1.8.7
     #   xblock
 wheel==0.40.0
     # via
-    #   -r requirements/edx/pip-tools.txt
+    #   -r requirements/edx/../pip-tools.txt
     #   pip-tools
 wrapt==1.15.0
     # via

requirements/edx/pip-tools.in → requirements/pip-tools.in

@@ -7,6 +7,6 @@
 #   * confirm that it has no system requirements beyond what we already install
 #   * run "make upgrade" to update the detailed requirements files
 
--c ../constraints.txt
+-c constraints.txt
 
 pip-tools                 # Contains pip-compile, used to generate pip requirements files

requirements/edx/pip-tools.txt → requirements/pip-tools.txt

@@ -8,12 +8,12 @@ build==0.10.0
     # via pip-tools
 click==8.1.3
     # via
-    #   -c requirements/edx/../constraints.txt
+    #   -c requirements/constraints.txt
     #   pip-tools
 packaging==23.0
     # via build
 pip-tools==6.12.3
-    # via -r requirements/edx/pip-tools.in
+    # via -r requirements/pip-tools.in
 pyproject-hooks==1.0.0
     # via build
 tomli==2.0.1

requirements/edx/pip.in → requirements/pip.in

@@ -1,4 +1,4 @@
--c ../constraints.txt
+-c constraints.txt
 # Core dependencies for installing other dependencies
 
 pip

requirements/pip.txt

@@ -1,2 +1,14 @@
-pip==22.1
-wheel==0.37.1
+#
+# This file is autogenerated by pip-compile with Python 3.8
+# by the following command:
+#
+#    make upgrade
+#
+wheel==0.40.0
+    # via -r requirements/pip.in
+
+# The following packages are considered to be unsafe in a requirements file:
+pip==23.0.1
+    # via -r requirements/pip.in
+setuptools==67.6.0
+    # via -r requirements/pip.in

scripts/ci-runner.Dockerfile

@@ -46,7 +46,7 @@ COPY openedx/core/lib openedx/core/lib
 COPY lms lms
 COPY cms cms
 COPY requirements/pip.txt requirements/pip.txt
-COPY requirements/edx/pip-tools.txt requirements/edx/pip-tools.txt
+COPY requirements/pip-tools.txt requirements/pip-tools.txt
 COPY requirements/edx/testing.txt requirements/edx/testing.txt
 COPY Makefile Makefile
 RUN make test-requirements