Merge pull request #54822 from s0undt3ch/hotfix/consolidate-requireme…

…nts-master

[master] Consolidate static requirements

.pre-commit-config.yaml

@@ -6,13 +6,12 @@ repos:
       - id: pip-tools-compile
         alias: compile-linux-py2.7-zmq-requirements
         name: Linux Py2.7 ZeroMQ Requirements
-        files: ^requirements/((base|zeromq|pytest)\.txt|static/(.*)\.in)$
+        files: ^requirements/((base|zeromq|pytest)\.txt|static/linux\.in)$
         exclude: ^requirements/static/(lint|cloud|docs|osx|windows)\.in$
         args:
           - -v
           - --py-version=2.7
           - --platform=linux
-          - --out-prefix=zeromq
           - --include=requirements/base.txt
           - --include=requirements/zeromq.txt
           - --include=requirements/pytest.txt
@@ -25,7 +24,6 @@ repos:
           - -v
           - --py-version=2.7
           - --platform=darwin
-          - --out-prefix=zeromq
           - --include=pkg/osx/req.txt
           - --include=pkg/osx/req_ext.txt
           - --include=requirements/base.txt
@@ -41,7 +39,6 @@ repos:
           - -v
           - --py-version=2.7
           - --platform=windows
-          - --out-prefix=zeromq
           - --include=pkg/windows/req.txt
           - --include=pkg/windows/req_win.txt
           - --include=requirements/base.txt
@@ -70,13 +67,12 @@ repos:
       - id: pip-tools-compile
         alias: compile-linux-py3.4-zmq-requirements
         name: Linux Py3.4 ZeroMQ Requirements
-        files: ^requirements/((base|zeromq|pytest)\.txt|static/(.*)\.in)$
+        files: ^requirements/((base|zeromq|pytest)\.txt|static/linux\.in)$
         exclude: ^requirements/static/(centos-6|amzn-2018\.03|lint|cloud|docs|osx|windows)\.in$
         args:
           - -v
           - --py-version=3.4
           - --platform=linux
-          - --out-prefix=zeromq
           - --include=requirements/base.txt
           - --include=requirements/zeromq.txt
           - --include=requirements/pytest.txt
@@ -93,13 +89,12 @@ repos:
       - id: pip-tools-compile
         alias: compile-linux-py3.5-zmq-requirements
         name: Linux Py3.5 ZeroMQ Requirements
-        files: ^requirements/((base|zeromq|pytest)\.txt|static/(.*)\.in)$
+        files: ^requirements/((base|zeromq|pytest)\.txt|static/linux\.in)$
         exclude: ^requirements/static/(centos-6|amzn-2018\.03|lint|cloud|docs|osx|windows)\.in$
         args:
           - -v
           - --py-version=3.5
           - --platform=linux
-          - --out-prefix=zeromq
           - --include=requirements/base.txt
           - --include=requirements/zeromq.txt
           - --include=requirements/pytest.txt
@@ -112,7 +107,6 @@ repos:
           - -v
           - --py-version=3.5
           - --platform=darwin
-          - --out-prefix=zeromq
           - --include=pkg/osx/req.txt
           - --include=pkg/osx/req_ext.txt
           - --include=requirements/base.txt
@@ -128,7 +122,6 @@ repos:
           - -v
           - --py-version=3.5
           - --platform=windows
-          - --out-prefix=zeromq
           - --include=pkg/windows/req.txt
           - --include=pkg/windows/req_win.txt
           - --include=requirements/base.txt
@@ -156,13 +149,12 @@ repos:
       - id: pip-tools-compile
         alias: compile-linux-py3.6-zmq-requirements
         name: Linux Py3.6 ZeroMQ Requirements
-        files: ^requirements/((base|zeromq|pytest)\.txt|static/(.*)\.in)$
+        files: ^requirements/((base|zeromq|pytest)\.txt|static/linux\.in)$
         exclude: ^requirements/static/(centos-6|amzn-2018\.03|lint|cloud|docs|osx|windows)\.in$
         args:
           - -v
           - --py-version=3.6
           - --platform=linux
-          - --out-prefix=zeromq
           - --include=requirements/base.txt
           - --include=requirements/zeromq.txt
           - --include=requirements/pytest.txt
@@ -175,7 +167,6 @@ repos:
           - -v
           - --py-version=3.6
           - --platform=darwin
-          - --out-prefix=zeromq
           - --include=pkg/osx/req.txt
           - --include=pkg/osx/req_ext.txt
           - --include=requirements/base.txt
@@ -191,7 +182,6 @@ repos:
           - -v
           - --py-version=3.6
           - --platform=windows
-          - --out-prefix=zeromq
           - --include=pkg/windows/req.txt
           - --include=pkg/windows/req_win.txt
           - --include=requirements/base.txt
@@ -219,13 +209,12 @@ repos:
       - id: pip-tools-compile
         alias: compile-linux-py3.7-zmq-requirements
         name: Linux Py3.7 ZeroMQ Requirements
-        files: ^requirements/((base|zeromq|pytest)\.txt|static/(.*)\.in)$
+        files: ^requirements/((base|zeromq|pytest)\.txt|static/linux\.in)$
         exclude: ^requirements/static/(centos-6|amzn-2018\.03|lint|cloud|docs|osx|windows)\.in$
         args:
           - -v
           - --py-version=3.7
           - --platform=linux
-          - --out-prefix=zeromq
           - --include=requirements/base.txt
           - --include=requirements/zeromq.txt
           - --include=requirements/pytest.txt
@@ -238,7 +227,6 @@ repos:
           - -v
           - --py-version=3.7
           - --platform=darwin
-          - --out-prefix=zeromq
           - --include=pkg/osx/req.txt
           - --include=pkg/osx/req_ext.txt
           - --include=requirements/base.txt
@@ -254,7 +242,6 @@ repos:
           - -v
           - --py-version=3.7
           - --platform=windows
-          - --out-prefix=zeromq
           - --include=pkg/windows/req.txt
           - --include=pkg/windows/req_win.txt
           - --include=requirements/base.txt

noxfile.py

@@ -175,9 +175,9 @@ def _install_system_packages(session):
                         shutil.copyfile(src, dst)
 
 
-def _install_requirements(session, transport, *extra_requirements):
+def _get_distro_pip_constraints(session, transport):
     # Install requirements
-    distro_requirements = None
+    distro_constraints = []
 
     if transport == 'tcp':
         # The TCP requirements are the exact same requirements as the ZeroMQ ones
@@ -186,51 +186,71 @@ def _install_requirements(session, transport, *extra_requirements):
     pydir = _get_pydir(session)
 
     if IS_WINDOWS:
-        _distro_requirements = os.path.join(REPO_ROOT,
-                                            'requirements',
-                                            'static',
-                                            pydir,
-                                            '{}-windows.txt'.format(transport))
-        if os.path.exists(_distro_requirements):
-            distro_requirements = _distro_requirements
+        _distro_constraints = os.path.join(REPO_ROOT,
+                                           'requirements',
+                                           'static',
+                                           pydir,
+                                           '{}-windows.txt'.format(transport))
+        if os.path.exists(_distro_constraints):
+            distro_constraints.append(_distro_constraints)
+        _distro_constraints = os.path.join(REPO_ROOT,
+                                           'requirements',
+                                           'static',
+                                           pydir,
+                                           'windows.txt')
+        if os.path.exists(_distro_constraints):
+            distro_constraints.append(_distro_constraints)
     else:
         _install_system_packages(session)
         distro = _get_distro_info(session)
         distro_keys = [
+            'linux',
             '{id}'.format(**distro),
             '{id}-{version}'.format(**distro),
             '{id}-{version_parts[major]}'.format(**distro)
         ]
         for distro_key in distro_keys:
-            _distro_requirements = os.path.join(REPO_ROOT,
-                                                'requirements',
-                                                'static',
-                                                pydir,
-                                                '{}-{}.txt'.format(transport, distro_key))
-            if os.path.exists(_distro_requirements):
-                distro_requirements = _distro_requirements
-                break
+            _distro_constraints = os.path.join(REPO_ROOT,
+                                               'requirements',
+                                               'static',
+                                               pydir,
+                                               '{}.txt'.format(distro_key))
+            if os.path.exists(_distro_constraints):
+                distro_constraints.append(_distro_constraints)
+            _distro_constraints = os.path.join(REPO_ROOT,
+                                               'requirements',
+                                               'static',
+                                               pydir,
+                                               '{}-{}.txt'.format(transport, distro_key))
+            if os.path.exists(_distro_constraints):
+                distro_constraints.append(_distro_constraints)
+    return distro_constraints
 
-    if distro_requirements is not None:
-        _requirements_files = [distro_requirements]
-        requirements_files = []
-    else:
-        _requirements_files = [
-            os.path.join(REPO_ROOT, 'requirements', 'pytest.txt')
+
+def _install_requirements(session, transport, *extra_requirements):
+    # Install requirements
+    distro_constraints = _get_distro_pip_constraints(session, transport)
+
+    _requirements_files = [
+        os.path.join(REPO_ROOT, 'requirements', 'base.txt'),
+        os.path.join(REPO_ROOT, 'requirements', 'zeromq.txt'),
+        os.path.join(REPO_ROOT, 'requirements', 'pytest.txt')
+    ]
+    if sys.platform.startswith('linux'):
+        requirements_files = [
+            os.path.join(REPO_ROOT, 'requirements', 'static', 'linux.in')
+        ]
+    elif sys.platform.startswith('win'):
+        requirements_files = [
+            os.path.join(REPO_ROOT, 'pkg', 'windows', 'req.txt'),
+            os.path.join(REPO_ROOT, 'requirements', 'static', 'windows.in')
+        ]
+    elif sys.platform.startswith('darwin'):
+        requirements_files = [
+            os.path.join(REPO_ROOT, 'pkg', 'osx', 'req.txt'),
+            os.path.join(REPO_ROOT, 'pkg', 'osx', 'req_ext.txt'),
+            os.path.join(REPO_ROOT, 'requirements', 'static', 'osx.in')
         ]
-        if sys.platform.startswith('linux'):
-            requirements_files = [
-                os.path.join(REPO_ROOT, 'requirements', 'tests.txt')
-            ]
-        elif sys.platform.startswith('win'):
-            requirements_files = [
-                os.path.join(REPO_ROOT, 'pkg', 'windows', 'req.txt'),
-            ]
-        elif sys.platform.startswith('darwin'):
-            requirements_files = [
-                os.path.join(REPO_ROOT, 'pkg', 'osx', 'req.txt'),
-                os.path.join(REPO_ROOT, 'pkg', 'osx', 'req_ext.txt'),
-            ]
 
     while True:
         if not requirements_files:
@@ -254,10 +274,25 @@ def _install_requirements(session, transport, *extra_requirements):
                     continue
 
     for requirements_file in _requirements_files:
-        session.install('--progress-bar=off', '-r', requirements_file, silent=PIP_INSTALL_SILENT)
+        install_command = [
+            '--progress-bar=off', '-r', requirements_file
+        ]
+        for distro_constraint in distro_constraints:
+            install_command.extend([
+                '--constraint', distro_constraint
+            ])
+        session.install(*install_command, silent=PIP_INSTALL_SILENT)
 
     if extra_requirements:
-        session.install('--progress-bar=off', *extra_requirements, silent=PIP_INSTALL_SILENT)
+        install_command = [
+            '--progress-bar=off',
+        ]
+        for distro_constraint in distro_constraints:
+            install_command.extend([
+                '--constraint', distro_constraint
+            ])
+        install_command += list(extra_requirements)
+        session.install(*install_command, silent=PIP_INSTALL_SILENT)
 
 
 def _run_with_coverage(session, *test_cmd):
@@ -371,7 +406,16 @@ def runtests_parametrized(session, coverage, transport, crypto):
             session.run('pip', 'uninstall', '-y', 'pycrypto', 'pycryptodome', 'pycryptodomex', silent=True)
         else:
             session.run('pip', 'uninstall', '-y', 'm2crypto', silent=True)
-        session.install('--progress-bar=off', crypto, silent=PIP_INSTALL_SILENT)
+        distro_constraints = _get_distro_pip_constraints(session, transport)
+        install_command = [
+            '--progress-bar=off',
+        ]
+        for distro_constraint in distro_constraints:
+            install_command.extend([
+                '--constraint', distro_constraint
+            ])
+        install_command.append(crypto)
+        session.install(*install_command, silent=PIP_INSTALL_SILENT)
 
     cmd_args = [
         '--tests-logfile={}'.format(
@@ -557,7 +601,16 @@ def pytest_parametrized(session, coverage, transport, crypto):
             session.run('pip', 'uninstall', '-y', 'pycrypto', 'pycryptodome', 'pycryptodomex', silent=True)
         else:
             session.run('pip', 'uninstall', '-y', 'm2crypto', silent=True)
-        session.install('--progress-bar=off', crypto, silent=PIP_INSTALL_SILENT)
+        distro_constraints = _get_distro_pip_constraints(session, transport)
+        install_command = [
+            '--progress-bar=off',
+        ]
+        for distro_constraint in distro_constraints:
+            install_command.extend([
+                '--constraint', distro_constraint
+            ])
+        install_command.append(crypto)
+        session.install(*install_command, silent=PIP_INSTALL_SILENT)
 
     cmd_args = [
         '--rootdir', REPO_ROOT,

pkg/osx/req.txt

@@ -24,7 +24,7 @@ pyobjc==5.1.2
 pyopenssl
 python-dateutil==2.8.0
 python-gnupg==0.4.4
-pyyaml==3.13
+pyyaml==5.1.2
 pyzmq==18.0.1
 requests==2.21.0
 setproctitle

pkg/windows/req.txt

@@ -28,7 +28,7 @@ pyopenssl==19.0.0
 python-dateutil==2.8.0
 python-gnupg==0.4.4
 pythonnet==2.3.0
-pyyaml==3.13
+pyyaml==5.1.2
 pyzmq==18.0.1
 requests==2.21.0
 setproctitle

requirements/base.txt

@@ -2,7 +2,7 @@ Jinja2
 # This should be changed to msgpack-python for Packages
 # msgpack-python>0.3,!=0.5.5
 msgpack>=0.5,!=0.5.5
-PyYAML<5.1
+PyYAML
 MarkupSafe
 requests>=1.0.0
 tornado>=4.2.1,<6.0; python_version < '3'