build: Deduplicate mismatched pip.txt files that caused build failure

Makefile

@@ -64,8 +64,8 @@ pull: ## update the Docker image used by "make shell"
 	docker pull edxops/edxapp:latest
 
 pre-requirements: ## install Python requirements for running pip-tools
-	pip install -qr requirements/pip.txt
-	pip install -qr requirements/edx/pip-tools.txt
+	pip install -r requirements/pip.txt
+	pip install -r requirements/pip-tools.txt
 
 local-requirements:
 # 	edx-platform installs some Python projects from within the edx-platform repo itself.
@@ -74,7 +74,7 @@ local-requirements:
 dev-requirements: pre-requirements
 	@# The "$(wildcard..)" is to include private.txt if it exists, and make no mention
 	@# of it if it does not.  Shell wildcarding can't do that with default options.
-	pip-sync -q requirements/edx/development.txt $(wildcard requirements/edx/private.txt)
+	pip-sync requirements/edx/development.txt $(wildcard requirements/edx/private.txt)
 	make local-requirements
 
 base-requirements: pre-requirements
@@ -96,7 +96,6 @@ shell: ## launch a bash shell in a Docker container with all edx-platform depend
 
 # Order is very important in this list: files must appear after everything they include!
 REQ_FILES = \
-	requirements/edx/pip-tools \
 	requirements/edx/coverage \
 	requirements/edx/doc \
 	requirements/edx/paver \
@@ -117,23 +116,26 @@ $(COMMON_CONSTRAINTS_TXT):
 	echo "$(COMMON_CONSTRAINTS_TEMP_COMMENT)" | cat - $(@) > temp && mv temp $(@)
 
 compile-requirements: export CUSTOM_COMPILE_COMMAND=make upgrade
-compile-requirements: $(COMMON_CONSTRAINTS_TXT) ## Re-compile *.in requirements to *.txt
-	pip install -q pip-tools
-	pip-compile --allow-unsafe --upgrade -o requirements/edx/pip.txt requirements/edx/pip.in
+compile-requirements: pre-requirements $(COMMON_CONSTRAINTS_TXT) ## Re-compile *.in requirements to *.txt
+	@# Bootstrapping: Rebuild pip and pip-tools first, and then install them
+	@# so that if there are any failures we'll know now, rather than the next
+	@# time someone tries to use the outputs.
+	pip-compile -v --allow-unsafe ${COMPILE_OPTS} -o requirements/pip.txt requirements/pip.in
+	pip install -r requirements/pip.txt
+
+	pip-compile -v ${COMPILE_OPTS} -o requirements/pip-tools.txt requirements/pip-tools.in
+	pip install -r requirements/pip-tools.txt
 
 	@ export REBUILD='--rebuild'; \
 	for f in $(REQ_FILES); do \
 		echo ; \
 		echo "== $$f ===============================" ; \
-		echo "pip-compile -v --no-emit-trusted-host --no-emit-index-url $$REBUILD ${COMPILE_OPTS} -o $$f.txt $$f.in"; \
-		pip-compile -v --no-emit-trusted-host --no-emit-index-url $$REBUILD ${COMPILE_OPTS} -o $$f.txt $$f.in || exit 1; \
+		echo "pip-compile -v $$REBUILD ${COMPILE_OPTS} -o $$f.txt $$f.in"; \
+		pip-compile -v $$REBUILD ${COMPILE_OPTS} -o $$f.txt $$f.in || exit 1; \
 		export REBUILD=''; \
 	done
 
-	pip install -qr requirements/edx/pip.txt
-	pip install -qr requirements/edx/pip-tools.txt
-
-upgrade: pre-requirements  ## update the pip requirements files to use the latest releases satisfying our constraints
+upgrade:  ## update the pip requirements files to use the latest releases satisfying our constraints
 	$(MAKE) compile-requirements COMPILE_OPTS="--upgrade"
 
 check-types: ## run static type-checking tests

requirements/edx/development.in

@@ -10,7 +10,7 @@
 
 -c ../constraints.txt
 
--r pip-tools.txt                        # pip-tools and its dependencies, for managing requirements files
+-r ../pip-tools.txt                        # pip-tools and its dependencies, for managing requirements files
 -r testing.txt                          # Dependencies for running the various test suites
 
 click                                   # Used for perf_tests utilities in modulestore

requirements/edx/development.txt

@@ -125,7 +125,7 @@ bridgekeeper==0.9
     # via -r requirements/edx/testing.txt
 build==0.10.0
     # via
-    #   -r requirements/edx/pip-tools.txt
+    #   -r requirements/edx/../pip-tools.txt
     #   pip-tools
 celery==5.2.7
     # via
@@ -168,8 +168,8 @@ chem==1.2.0
 click==8.1.3
     # via
     #   -c requirements/edx/../constraints.txt
+    #   -r requirements/edx/../pip-tools.txt
     #   -r requirements/edx/development.in
-    #   -r requirements/edx/pip-tools.txt
     #   -r requirements/edx/testing.txt
     #   celery
     #   click-didyoumean
@@ -1047,7 +1047,7 @@ outcome-surveys==2.4.0
     # via -r requirements/edx/testing.txt
 packaging==23.0
     # via
-    #   -r requirements/edx/pip-tools.txt
+    #   -r requirements/edx/../pip-tools.txt
     #   -r requirements/edx/testing.txt
     #   build
     #   drf-yasg
@@ -1093,7 +1093,7 @@ pillow==9.4.0
     #   edx-enterprise
     #   edx-organizations
 pip-tools==6.12.3
-    # via -r requirements/edx/pip-tools.txt
+    # via -r requirements/edx/../pip-tools.txt
 pkgutil-resolve-name==1.3.10
     # via
     #   -r requirements/edx/testing.txt
@@ -1237,7 +1237,7 @@ pyparsing==3.0.9
     #   openedx-calc
 pyproject-hooks==1.0.0
     # via
-    #   -r requirements/edx/pip-tools.txt
+    #   -r requirements/edx/../pip-tools.txt
     #   build
 pyquery==2.0.0
     # via -r requirements/edx/testing.txt
@@ -1584,7 +1584,7 @@ toml==0.10.2
     # via vulture
 tomli==2.0.1
     # via
-    #   -r requirements/edx/pip-tools.txt
+    #   -r requirements/edx/../pip-tools.txt
     #   -r requirements/edx/testing.txt
     #   build
     #   coverage
@@ -1691,7 +1691,7 @@ webob==1.8.7
     #   xblock
 wheel==0.40.0
     # via
-    #   -r requirements/edx/pip-tools.txt
+    #   -r requirements/edx/../pip-tools.txt
     #   pip-tools
 wrapt==1.15.0
     # via

requirements/edx/pip-tools.in → requirements/pip-tools.in

@@ -7,6 +7,6 @@
 #   * confirm that it has no system requirements beyond what we already install
 #   * run "make upgrade" to update the detailed requirements files
 
--c ../constraints.txt
+-c constraints.txt
 
 pip-tools                 # Contains pip-compile, used to generate pip requirements files

requirements/edx/pip-tools.txt → requirements/pip-tools.txt

@@ -8,12 +8,12 @@ build==0.10.0
     # via pip-tools
 click==8.1.3
     # via
-    #   -c requirements/edx/../constraints.txt
+    #   -c requirements/constraints.txt
     #   pip-tools
 packaging==23.0
     # via build
 pip-tools==6.12.3
-    # via -r requirements/edx/pip-tools.in
+    # via -r requirements/pip-tools.in
 pyproject-hooks==1.0.0
     # via build
 tomli==2.0.1

requirements/edx/pip.in → requirements/pip.in

@@ -1,4 +1,4 @@
--c ../constraints.txt
+-c constraints.txt
 # Core dependencies for installing other dependencies
 
 pip

requirements/pip.txt

@@ -1,2 +1,14 @@
-pip==22.1
-wheel==0.37.1
+#
+# This file is autogenerated by pip-compile with Python 3.8
+# by the following command:
+#
+#    make upgrade
+#
+wheel==0.40.0
+    # via -r requirements/pip.in
+
+# The following packages are considered to be unsafe in a requirements file:
+pip==23.0.1
+    # via -r requirements/pip.in
+setuptools==67.6.0
+    # via -r requirements/pip.in

scripts/ci-runner.Dockerfile

@@ -46,7 +46,7 @@ COPY openedx/core/lib openedx/core/lib
 COPY lms lms
 COPY cms cms
 COPY requirements/pip.txt requirements/pip.txt
-COPY requirements/edx/pip-tools.txt requirements/edx/pip-tools.txt
+COPY requirements/pip-tools.txt requirements/pip-tools.txt
 COPY requirements/edx/testing.txt requirements/edx/testing.txt
 COPY Makefile Makefile
 RUN make test-requirements