Fix dependency conflicts

CHANGELOG.rst

@@ -22,6 +22,8 @@ Changed
 
   Latest version of mongoengine should show some performance improvements (5-20%) when
   writing very large executions (executions with large results) to the database. #4767
+* Improved development instructions in requirements.txt and dist_utils.py comment headers
+  (improvement) #4774
 
 Fixed
 ~~~~~
@@ -42,6 +44,8 @@ Fixed
   Contributed by JP Bourget (@punkrokk Syncurity) #4732
 * Update ``dist_utils`` module which is bundled with ``st2client`` and other Python packages so it
   doesn't depend on internal pip API and so it works with latest pip version. (bug fix) #4750
+* Fix dependency conflicts in pack CI runs: downgrade requests dependency back to 0.21.0, update
+  internal dependencies and test expectations (amqp, pyyaml, prance, six) (bugfix) #4774
 
 3.1.0 - June 27, 2019
 ---------------------

Makefile

@@ -455,12 +455,14 @@ requirements: virtualenv .sdist-requirements install-runners
 	done
 
 	# Fix for Travis CI race
-	$(VIRTUALENV_DIR)/bin/pip install "six==1.11.0"
+	$(VIRTUALENV_DIR)/bin/pip install "six==1.12.0"
 
 	# Fix for Travis CI caching issue
-	$(VIRTUALENV_DIR)/bin/pip uninstall -y "pytz" || echo "not installed"
-	$(VIRTUALENV_DIR)/bin/pip uninstall -y "python-dateutil" || echo "not installed"
-	$(VIRTUALENV_DIR)/bin/pip uninstall -y "orquesta" || echo "not installed"
+	if [[ "$(TRAVIS_EVENT_TYPE)" != "" ]]; then\
+		$(VIRTUALENV_DIR)/bin/pip uninstall -y "pytz" || echo "not installed"; \
+		$(VIRTUALENV_DIR)/bin/pip uninstall -y "python-dateutil" || echo "not installed"; \
+		$(VIRTUALENV_DIR)/bin/pip uninstall -y "orquesta" || echo "not installed"; \
+	fi
 
 	# Install requirements
 	#
@@ -478,7 +480,7 @@ requirements: virtualenv .sdist-requirements install-runners
 	# Note: We install prance here and not as part of any component
 	# requirements.txt because it has a conflict with our dependency (requires
 	# new version of requests) which we cant resolve at this moment
-	$(VIRTUALENV_DIR)/bin/pip install "prance==0.6.1"
+	$(VIRTUALENV_DIR)/bin/pip install "prance==0.15.0"
 
 	# Install st2common to register metrics drivers
 	# NOTE: We pass --no-deps to the script so we don't install all the
@@ -920,7 +922,7 @@ debs:
 	# Copy over shared dist utils module which is needed by setup.py
 	@for component in $(COMPONENTS_WITH_RUNNERS); do\
 		cp -f ./scripts/dist_utils.py $$component/dist_utils.py;\
-		sed -i -e '1s;^;# NOTE: This file is auto-generated - DO NOT EDIT MANUALLY\n;' $$component/dist_utils.py;\
+		scripts/write-headers.sh $$component/dist_utils.py || break;\
 	done
 
 	# Copy over CHANGELOG.RST, CONTRIBUTING.RST and LICENSE file to each component directory

contrib/runners/action_chain_runner/dist_utils.py

@@ -1,4 +1,6 @@
 # NOTE: This file is auto-generated - DO NOT EDIT MANUALLY
+#       Instead modify scripts/dist_utils.py and run 'make .sdist-requirements' to
+#       update dist_utils.py files for all components
 # -*- coding: utf-8 -*-
 # Copyright 2019 Extreme Networks, Inc.
 #

contrib/runners/action_chain_runner/requirements.txt

@@ -1,2 +1,8 @@
 # Don't edit this file. It's generated automatically!
+# If you want to update global dependencies, modify fixed-requirements.txt
+# and then run 'make requirements' to update requirements.txt for all
+# components.
+# If you want to update depdencies for a single component, modify the
+# in-requirements.txt for that component and then run 'make requirements' to
+# update the component requirements.txt
 

contrib/runners/announcement_runner/dist_utils.py

@@ -1,4 +1,6 @@
 # NOTE: This file is auto-generated - DO NOT EDIT MANUALLY
+#       Instead modify scripts/dist_utils.py and run 'make .sdist-requirements' to
+#       update dist_utils.py files for all components
 # -*- coding: utf-8 -*-
 # Copyright 2019 Extreme Networks, Inc.
 #

contrib/runners/announcement_runner/requirements.txt

@@ -1,2 +1,8 @@
 # Don't edit this file. It's generated automatically!
+# If you want to update global dependencies, modify fixed-requirements.txt
+# and then run 'make requirements' to update requirements.txt for all
+# components.
+# If you want to update depdencies for a single component, modify the
+# in-requirements.txt for that component and then run 'make requirements' to
+# update the component requirements.txt
 

contrib/runners/http_runner/dist_utils.py

@@ -1,4 +1,6 @@
 # NOTE: This file is auto-generated - DO NOT EDIT MANUALLY
+#       Instead modify scripts/dist_utils.py and run 'make .sdist-requirements' to
+#       update dist_utils.py files for all components
 # -*- coding: utf-8 -*-
 # Copyright 2019 Extreme Networks, Inc.
 #

contrib/runners/http_runner/requirements.txt

@@ -1,2 +1,8 @@
 # Don't edit this file. It's generated automatically!
+# If you want to update global dependencies, modify fixed-requirements.txt
+# and then run 'make requirements' to update requirements.txt for all
+# components.
+# If you want to update depdencies for a single component, modify the
+# in-requirements.txt for that component and then run 'make requirements' to
+# update the component requirements.txt
 

contrib/runners/inquirer_runner/dist_utils.py

@@ -1,4 +1,6 @@
 # NOTE: This file is auto-generated - DO NOT EDIT MANUALLY
+#       Instead modify scripts/dist_utils.py and run 'make .sdist-requirements' to
+#       update dist_utils.py files for all components
 # -*- coding: utf-8 -*-
 # Copyright 2019 Extreme Networks, Inc.
 #

contrib/runners/inquirer_runner/requirements.txt

@@ -1,2 +1,8 @@
 # Don't edit this file. It's generated automatically!
+# If you want to update global dependencies, modify fixed-requirements.txt
+# and then run 'make requirements' to update requirements.txt for all
+# components.
+# If you want to update depdencies for a single component, modify the
+# in-requirements.txt for that component and then run 'make requirements' to
+# update the component requirements.txt
 

contrib/runners/local_runner/dist_utils.py

@@ -1,4 +1,6 @@
 # NOTE: This file is auto-generated - DO NOT EDIT MANUALLY
+#       Instead modify scripts/dist_utils.py and run 'make .sdist-requirements' to
+#       update dist_utils.py files for all components
 # -*- coding: utf-8 -*-
 # Copyright 2019 Extreme Networks, Inc.
 #

contrib/runners/local_runner/requirements.txt

@@ -1,2 +1,8 @@
 # Don't edit this file. It's generated automatically!
+# If you want to update global dependencies, modify fixed-requirements.txt
+# and then run 'make requirements' to update requirements.txt for all
+# components.
+# If you want to update depdencies for a single component, modify the
+# in-requirements.txt for that component and then run 'make requirements' to
+# update the component requirements.txt
 

contrib/runners/mistral_v2/dist_utils.py

@@ -1,4 +1,6 @@
 # NOTE: This file is auto-generated - DO NOT EDIT MANUALLY
+#       Instead modify scripts/dist_utils.py and run 'make .sdist-requirements' to
+#       update dist_utils.py files for all components
 # -*- coding: utf-8 -*-
 # Copyright 2019 Extreme Networks, Inc.
 #

contrib/runners/mistral_v2/requirements.txt

@@ -1,2 +1,8 @@
 # Don't edit this file. It's generated automatically!
+# If you want to update global dependencies, modify fixed-requirements.txt
+# and then run 'make requirements' to update requirements.txt for all
+# components.
+# If you want to update depdencies for a single component, modify the
+# in-requirements.txt for that component and then run 'make requirements' to
+# update the component requirements.txt
 

contrib/runners/noop_runner/dist_utils.py

@@ -1,4 +1,6 @@
 # NOTE: This file is auto-generated - DO NOT EDIT MANUALLY
+#       Instead modify scripts/dist_utils.py and run 'make .sdist-requirements' to
+#       update dist_utils.py files for all components
 # -*- coding: utf-8 -*-
 # Copyright 2019 Extreme Networks, Inc.
 #

contrib/runners/noop_runner/requirements.txt

@@ -1,2 +1,8 @@
 # Don't edit this file. It's generated automatically!
+# If you want to update global dependencies, modify fixed-requirements.txt
+# and then run 'make requirements' to update requirements.txt for all
+# components.
+# If you want to update depdencies for a single component, modify the
+# in-requirements.txt for that component and then run 'make requirements' to
+# update the component requirements.txt
 

contrib/runners/orquesta_runner/dist_utils.py

@@ -1,4 +1,6 @@
 # NOTE: This file is auto-generated - DO NOT EDIT MANUALLY
+#       Instead modify scripts/dist_utils.py and run 'make .sdist-requirements' to
+#       update dist_utils.py files for all components
 # -*- coding: utf-8 -*-
 # Copyright 2019 Extreme Networks, Inc.
 #

contrib/runners/orquesta_runner/requirements.txt

@@ -1,2 +1,8 @@
 # Don't edit this file. It's generated automatically!
+# If you want to update global dependencies, modify fixed-requirements.txt
+# and then run 'make requirements' to update requirements.txt for all
+# components.
+# If you want to update depdencies for a single component, modify the
+# in-requirements.txt for that component and then run 'make requirements' to
+# update the component requirements.txt
 git+https://github.com/StackStorm/orquesta.git@e6ebbbeb2c661486067e659dc7552f0a986603a6#egg=orquesta

contrib/runners/python_runner/dist_utils.py

@@ -1,4 +1,6 @@
 # NOTE: This file is auto-generated - DO NOT EDIT MANUALLY
+#       Instead modify scripts/dist_utils.py and run 'make .sdist-requirements' to
+#       update dist_utils.py files for all components
 # -*- coding: utf-8 -*-
 # Copyright 2019 Extreme Networks, Inc.
 #

contrib/runners/python_runner/requirements.txt

@@ -1,2 +1,8 @@
 # Don't edit this file. It's generated automatically!
+# If you want to update global dependencies, modify fixed-requirements.txt
+# and then run 'make requirements' to update requirements.txt for all
+# components.
+# If you want to update depdencies for a single component, modify the
+# in-requirements.txt for that component and then run 'make requirements' to
+# update the component requirements.txt
 

contrib/runners/remote_runner/dist_utils.py

@@ -1,4 +1,6 @@
 # NOTE: This file is auto-generated - DO NOT EDIT MANUALLY
+#       Instead modify scripts/dist_utils.py and run 'make .sdist-requirements' to
+#       update dist_utils.py files for all components
 # -*- coding: utf-8 -*-
 # Copyright 2019 Extreme Networks, Inc.
 #

contrib/runners/remote_runner/requirements.txt

@@ -1,2 +1,8 @@
 # Don't edit this file. It's generated automatically!
+# If you want to update global dependencies, modify fixed-requirements.txt
+# and then run 'make requirements' to update requirements.txt for all
+# components.
+# If you want to update depdencies for a single component, modify the
+# in-requirements.txt for that component and then run 'make requirements' to
+# update the component requirements.txt
 

contrib/runners/winrm_runner/dist_utils.py

@@ -1,4 +1,6 @@
 # NOTE: This file is auto-generated - DO NOT EDIT MANUALLY
+#       Instead modify scripts/dist_utils.py and run 'make .sdist-requirements' to
+#       update dist_utils.py files for all components
 # -*- coding: utf-8 -*-
 # Copyright 2019 Extreme Networks, Inc.
 #

contrib/runners/winrm_runner/requirements.txt

@@ -1,2 +1,8 @@
 # Don't edit this file. It's generated automatically!
+# If you want to update global dependencies, modify fixed-requirements.txt
+# and then run 'make requirements' to update requirements.txt for all
+# components.
+# If you want to update depdencies for a single component, modify the
+# in-requirements.txt for that component and then run 'make requirements' to
+# update the component requirements.txt
 pywinrm==0.3.0

fixed-requirements.txt

@@ -7,14 +7,14 @@ eventlet==0.25.0
 gunicorn==19.9.0
 kombu==4.6.4
 # Note: amqp is used by kombu
-amqp==2.5.0
+amqp==2.5.1
 # NOTE: Recent version substantially affect the performance and add big import time overhead
 # See https://github.com/StackStorm/st2/issues/4160#issuecomment-394386433 for details
 oslo.config>=1.12.1,<1.13
 oslo.utils>=3.36.2,<=3.37.0
 six==1.12.0
 pyyaml==5.1.2
-requests[security]>=2.22.0,<2.23.0
+requests[security]>=2.21.0,<2.22.0
 apscheduler==3.6.1
 gitpython==2.1.11
 jsonschema==2.6.0

requirements.txt

@@ -1,6 +1,12 @@
 # Don't edit this file. It's generated automatically!
+# If you want to update global dependencies, modify fixed-requirements.txt
+# and then run 'make requirements' to update requirements.txt for all
+# components.
+# If you want to update depdencies for a single component, modify the
+# in-requirements.txt for that component and then run 'make requirements' to
+# update the component requirements.txt
 RandomWords
-amqp==2.5.0
+amqp==2.5.1
 apscheduler==3.6.1
 argcomplete
 bcrypt==3.1.7
@@ -45,7 +51,7 @@ pytz==2019.1
 pywinrm==0.3.0
 pyyaml==5.1.2
 rednose
-requests[security]<2.23.0,>=2.22.0
+requests[security]<2.22.0,>=2.21.0
 retrying==1.3.3
 routes==2.4.1
 semver==2.8.1

scripts/fixate-requirements.py

@@ -183,6 +183,12 @@ def write_requirements(sources=None, fixed_requirements=None, output_file=None,
     data = '\n'.join(lines_to_write) + '\n'
     with open(output_file, 'w') as fp:
         fp.write('# Don\'t edit this file. It\'s generated automatically!\n')
+        fp.write('# If you want to update global dependencies, modify fixed-requirements.txt\n')
+        fp.write('# and then run \'make requirements\' to update requirements.txt for all\n')
+        fp.write('# components.\n')
+        fp.write('# If you want to update depdencies for a single component, modify the\n')
+        fp.write('# in-requirements.txt for that component and then run \'make requirements\' to\n')
+        fp.write('# update the component requirements.txt\n')
         fp.write(data)
 
     print('Requirements written to: {0}'.format(output_file))

scripts/write-headers.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+HEADER="# NOTE: This file is auto-generated - DO NOT EDIT MANUALLY\\
+#       Instead modify scripts/dist_utils.py and run 'make .sdist-requirements' to\\
+#       update dist_utils.py files for all components\\
+"
+
+if [[ $(uname) == "Linux" ]]; then
+	sed -i -e "1s;^;$HEADER;" $* || exit -1
+elif [[ $(uname) == "Darwin" ]]; then
+	sed -i '' -e "1s;^;$HEADER;" $* || exit -1
+else
+	echo >&2 "Unknown OS"
+	exit -1
+fi
+

st2actions/dist_utils.py

@@ -1,4 +1,6 @@
 # NOTE: This file is auto-generated - DO NOT EDIT MANUALLY
+#       Instead modify scripts/dist_utils.py and run 'make .sdist-requirements' to
+#       update dist_utils.py files for all components
 # -*- coding: utf-8 -*-
 # Copyright 2019 Extreme Networks, Inc.
 #

st2actions/requirements.txt

@@ -1,4 +1,10 @@
 # Don't edit this file. It's generated automatically!
+# If you want to update global dependencies, modify fixed-requirements.txt
+# and then run 'make requirements' to update requirements.txt for all
+# components.
+# If you want to update depdencies for a single component, modify the
+# in-requirements.txt for that component and then run 'make requirements' to
+# update the component requirements.txt
 apscheduler==3.6.1
 eventlet==0.25.0
 git+https://github.com/Kami/logshipper.git@stackstorm_patched#egg=logshipper
@@ -13,5 +19,5 @@ pyinotify==0.9.6
 python-dateutil==2.8.0
 python-json-logger
 pyyaml==5.1.2
-requests[security]<2.23.0,>=2.22.0
+requests[security]<2.22.0,>=2.21.0
 six==1.12.0

st2api/dist_utils.py

@@ -1,4 +1,6 @@
 # NOTE: This file is auto-generated - DO NOT EDIT MANUALLY
+#       Instead modify scripts/dist_utils.py and run 'make .sdist-requirements' to
+#       update dist_utils.py files for all components
 # -*- coding: utf-8 -*-
 # Copyright 2019 Extreme Networks, Inc.
 #

st2api/requirements.txt

@@ -1,4 +1,10 @@
 # Don't edit this file. It's generated automatically!
+# If you want to update global dependencies, modify fixed-requirements.txt
+# and then run 'make requirements' to update requirements.txt for all
+# components.
+# If you want to update depdencies for a single component, modify the
+# in-requirements.txt for that component and then run 'make requirements' to
+# update the component requirements.txt
 eventlet==0.25.0
 git+https://github.com/StackStorm/python-mistralclient#egg=python-mistralclient
 gunicorn==19.9.0

st2auth/dist_utils.py

@@ -1,4 +1,6 @@
 # NOTE: This file is auto-generated - DO NOT EDIT MANUALLY
+#       Instead modify scripts/dist_utils.py and run 'make .sdist-requirements' to
+#       update dist_utils.py files for all components
 # -*- coding: utf-8 -*-
 # Copyright 2019 Extreme Networks, Inc.
 #

st2auth/requirements.txt

@@ -1,4 +1,10 @@
 # Don't edit this file. It's generated automatically!
+# If you want to update global dependencies, modify fixed-requirements.txt
+# and then run 'make requirements' to update requirements.txt for all
+# components.
+# If you want to update depdencies for a single component, modify the
+# in-requirements.txt for that component and then run 'make requirements' to
+# update the component requirements.txt
 bcrypt==3.1.7
 eventlet==0.25.0
 git+https://github.com/StackStorm/st2-auth-backend-flat-file.git@master#egg=st2-auth-backend-flat-file

st2client/dist_utils.py

@@ -1,4 +1,6 @@
 # NOTE: This file is auto-generated - DO NOT EDIT MANUALLY
+#       Instead modify scripts/dist_utils.py and run 'make .sdist-requirements' to
+#       update dist_utils.py files for all components
 # -*- coding: utf-8 -*-
 # Copyright 2019 Extreme Networks, Inc.
 #

st2client/requirements.txt

@@ -1,4 +1,10 @@
 # Don't edit this file. It's generated automatically!
+# If you want to update global dependencies, modify fixed-requirements.txt
+# and then run 'make requirements' to update requirements.txt for all
+# components.
+# If you want to update depdencies for a single component, modify the
+# in-requirements.txt for that component and then run 'make requirements' to
+# update the component requirements.txt
 argcomplete
 cryptography==2.7
 jsonpath-rw==1.4.0
@@ -9,6 +15,6 @@ python-dateutil==2.8.0
 python-editor==1.0.4
 pytz==2019.1
 pyyaml==5.1.2
-requests[security]<2.23.0,>=2.22.0
+requests[security]<2.22.0,>=2.21.0
 six==1.12.0
 sseclient-py==1.7

st2common/dist_utils.py

@@ -1,4 +1,6 @@
 # NOTE: This file is auto-generated - DO NOT EDIT MANUALLY
+#       Instead modify scripts/dist_utils.py and run 'make .sdist-requirements' to
+#       update dist_utils.py files for all components
 # -*- coding: utf-8 -*-
 # Copyright 2019 Extreme Networks, Inc.
 #