From 136e69d1f77c00797ecc1aa5c2c2f7a3d24a3d4c Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Fri, 4 Mar 2016 12:27:22 -0800 Subject: [PATCH 1/5] Updating requirements. --- container_engine/django_tutorial/requirements.txt | 2 +- managed_vms/datastore/requirements.txt | 2 +- managed_vms/django_cloudsql/requirements.txt | 2 +- managed_vms/hello_world_django/requirements.txt | 2 +- managed_vms/kinto/requirements.txt | 2 +- managed_vms/pubsub/requirements.txt | 2 +- managed_vms/sendgrid/requirements.txt | 2 +- requirements-py27-dev.txt | 8 ++++---- requirements-py34-dev.txt | 8 ++++---- 9 files changed, 15 insertions(+), 15 deletions(-) diff --git a/container_engine/django_tutorial/requirements.txt b/container_engine/django_tutorial/requirements.txt index 377a14aab47b..6d724096fce8 100644 --- a/container_engine/django_tutorial/requirements.txt +++ b/container_engine/django_tutorial/requirements.txt @@ -1,4 +1,4 @@ -Django==1.9.2 +Django==1.9.3 mysqlclient==1.3.7 wheel==0.29.0 gunicorn==19.4.5 diff --git a/managed_vms/datastore/requirements.txt b/managed_vms/datastore/requirements.txt index 6a6bfb7c53f9..ae27a8070a21 100644 --- a/managed_vms/datastore/requirements.txt +++ b/managed_vms/datastore/requirements.txt @@ -1,4 +1,4 @@ Flask==0.10.1 gcloud==0.10.1 gunicorn==19.4.5 -oauth2client==2.0.0.post1 +oauth2client==2.0.1 diff --git a/managed_vms/django_cloudsql/requirements.txt b/managed_vms/django_cloudsql/requirements.txt index 377a14aab47b..6d724096fce8 100644 --- a/managed_vms/django_cloudsql/requirements.txt +++ b/managed_vms/django_cloudsql/requirements.txt @@ -1,4 +1,4 @@ -Django==1.9.2 +Django==1.9.3 mysqlclient==1.3.7 wheel==0.29.0 gunicorn==19.4.5 diff --git a/managed_vms/hello_world_django/requirements.txt b/managed_vms/hello_world_django/requirements.txt index b68903a987a7..55fd25b8803c 100644 --- a/managed_vms/hello_world_django/requirements.txt +++ b/managed_vms/hello_world_django/requirements.txt @@ -1,2 +1,2 @@ -Django==1.9.2 +Django==1.9.3 gunicorn==19.4.5 diff --git a/managed_vms/kinto/requirements.txt b/managed_vms/kinto/requirements.txt index 3acab72551f3..b4e154f0ff5b 100644 --- a/managed_vms/kinto/requirements.txt +++ b/managed_vms/kinto/requirements.txt @@ -1,3 +1,3 @@ gunicorn==19.4.5 -oauth2client==2.0.0.post1 +oauth2client==2.0.1 kinto==1.11.2 diff --git a/managed_vms/pubsub/requirements.txt b/managed_vms/pubsub/requirements.txt index 6a6bfb7c53f9..ae27a8070a21 100644 --- a/managed_vms/pubsub/requirements.txt +++ b/managed_vms/pubsub/requirements.txt @@ -1,4 +1,4 @@ Flask==0.10.1 gcloud==0.10.1 gunicorn==19.4.5 -oauth2client==2.0.0.post1 +oauth2client==2.0.1 diff --git a/managed_vms/sendgrid/requirements.txt b/managed_vms/sendgrid/requirements.txt index a2d9f40f85a4..e8e67f01aa1f 100644 --- a/managed_vms/sendgrid/requirements.txt +++ b/managed_vms/sendgrid/requirements.txt @@ -1,3 +1,3 @@ Flask==0.10.1 -sendgrid==1.6.22 +sendgrid==2.2.1 gunicorn==19.4.5 diff --git a/requirements-py27-dev.txt b/requirements-py27-dev.txt index c8e31cca20b8..f551457c1d9a 100644 --- a/requirements-py27-dev.txt +++ b/requirements-py27-dev.txt @@ -1,6 +1,6 @@ gcloud==0.10.1 google-api-python-client==1.5.0 -oauth2client==2.0.0.post1 +oauth2client==2.0.1 requests[security]==2.9.1 beautifulsoup4==4.4.1 coverage==4.1b2 @@ -21,10 +21,10 @@ PyMySQL==0.7.2 pymemcache==1.3.5 PyCrypto==2.6.1 flaky==3.1.0 -Django==1.9.2 +Django==1.9.3 twilio==6.3.dev0 -sendgrid==1.6.22 +sendgrid==2.2.1 Flask-Sockets==0.2.0 mysql-python==1.2.5 -pytest==2.8.7 +pytest==2.9.0 pytest-cov==2.2.1 diff --git a/requirements-py34-dev.txt b/requirements-py34-dev.txt index 968df9921a93..2c490ffe5ec3 100644 --- a/requirements-py34-dev.txt +++ b/requirements-py34-dev.txt @@ -1,6 +1,6 @@ gcloud==0.10.1 google-api-python-client==1.5.0 -oauth2client==2.0.0.post1 +oauth2client==2.0.1 requests[security]==2.9.1 beautifulsoup4==4.4.1 coverage==4.1b2 @@ -21,8 +21,8 @@ PyMySQL==0.7.2 pymemcache==1.3.5 PyCrypto==2.6.1 flaky==3.1.0 -Django==1.9.2 +Django==1.9.3 twilio==6.3.dev0 -sendgrid==1.6.22 -pytest==2.8.7 +sendgrid==2.2.1 +pytest==2.9.0 pytest-cov==2.2.1 From b928bb159ab93c6a2732fe941cd4b28bd8cdea95 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Fri, 25 Mar 2016 13:35:31 -0700 Subject: [PATCH 2/5] Adding nox configuration for the standard (non-gae) tests --- .gitignore | 1 + .travis.yml | 6 +- nox.py | 83 +++++++++++++++++++ ...-dev.txt => requirements-python2.7-dev.txt | 0 ...-dev.txt => requirements-python3.4-dev.txt | 0 tox.ini | 7 -- 6 files changed, 88 insertions(+), 9 deletions(-) create mode 100644 nox.py rename requirements-py27-dev.txt => requirements-python2.7-dev.txt (100%) rename requirements-py34-dev.txt => requirements-python3.4-dev.txt (100%) diff --git a/.gitignore b/.gitignore index 850c3a9fab06..176a48ea6e78 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ secrets.tar .cache junit.xml credentials.dat +.nox diff --git a/.travis.yml b/.travis.yml index 4a147705cd5d..b02e7129092c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,11 +18,13 @@ env: - GAE_ROOT=${HOME}/.cache/ - secure: Orp9Et2TIwCG/Hf59aa0NUDF1pNcwcS4TFulXX175918cFREOzf/cNZNg+Ui585ZRFjbifZdc858tVuCVd8XlxQPXQgp7bwB7nXs3lby3LYg4+HD83Gaz7KOWxRLWVor6IVn8OxeCzwl6fJkdmffsTTO9csC4yZ7izHr+u7hiO4= before_install: +- pip install --upgrade pip wheel virtualenv - openssl aes-256-cbc -k "$secrets_password" -in secrets.tar.enc -out secrets.tar -d - tar xvf secrets.tar install: -- pip install tox coverage +- pip install nox-automation coverage script: - source ${TRAVIS_BUILD_DIR}/testing/resources/test-env.sh -- tox +- nox --stop-on-first-error -s lint travis +after_script: - coverage report diff --git a/nox.py b/nox.py new file mode 100644 index 000000000000..14603778c786 --- /dev/null +++ b/nox.py @@ -0,0 +1,83 @@ +import fnmatch +import os + +import nox + +REPO_TOOLS_REQ =\ + 'git+https://github.com/GoogleCloudPlatform/python-repo-tools.git' + + +def session_lint(session): + session.install('flake8', 'flake8-import-order') + session.run( + 'flake8', '--builtin=gettext', '--max-complexity=10', + '--import-order-style=google', + '--exclude', + 'container_engine/django_tutorial/polls/migrations/*,.nox,.cache,env', + *(session.posargs or ['.'])) + + +def list_files(folder, pattern): + for root, folders, files in os.walk(folder): + for filename in files: + if fnmatch.fnmatch(filename, pattern): + yield os.path.join(root, filename) + + +def session_reqcheck(session): + session.install(REPO_TOOLS_REQ) + + if 'update' in session.posargs: + command = 'update-requirements' + else: + command = 'check-requirements' + + for reqfile in list_files('.', 'requirements*.txt'): + session.run('gcprepotools', command, reqfile) + + +COMMON_PYTEST_ARGS = [ + '-x', '--no-success-flaky-report', '--cov', '--cov-config', + '.coveragerc', '--cov-append', '--cov-report='] + +SAMPLES = [ + 'bigquery/api', + 'blog/introduction_to_data_models_in_cloud_datastore', + 'cloud_logging/api', + 'compute/api', + 'compute/autoscaler/demo', + 'datastore/api', + 'managed_vms/cloudsql', + 'managed_vms/datastore', + 'managed_vms/disk', + 'managed_vms/extending_runtime', + 'managed_vms/hello_world', + 'managed_vms/hello_world_compat', + 'managed_vms/memcache', + 'managed_vms/pubsub', + 'managed_vms/static_files', + 'managed_vms/storage', + 'monitoring/api', + 'storage/api', +] + + +@nox.parametrize('interpreter', ['python2.7', 'python3.4']) +def session_tests(session, interpreter, extra_pytest_args=None): + session.interpreter = interpreter + session.install(REPO_TOOLS_REQ) + session.install('-r', 'requirements-{}-dev.txt'.format(interpreter)) + + pytest_args = COMMON_PYTEST_ARGS + (extra_pytest_args or []) + + for sample in (session.posargs or SAMPLES): + session.run( + 'py.test', sample, + *pytest_args, + success_codes=[0, 5]) # Treat no test collected as success. + + +def session_travis(session): + """On travis, just run with python3.4 and don't run slow or flaky tests.""" + session_tests( + session, 'python3.4', extra_pytest_args=['-m not slow and not flaky']) diff --git a/requirements-py27-dev.txt b/requirements-python2.7-dev.txt similarity index 100% rename from requirements-py27-dev.txt rename to requirements-python2.7-dev.txt diff --git a/requirements-py34-dev.txt b/requirements-python3.4-dev.txt similarity index 100% rename from requirements-py34-dev.txt rename to requirements-python3.4-dev.txt diff --git a/tox.ini b/tox.ini index 5e8939b42b42..acdce8ac6753 100644 --- a/tox.ini +++ b/tox.ini @@ -133,10 +133,3 @@ whitelist_externals = bash find xargs - -[flake8] -exclude=container_engine/django_tutorial/polls/migrations/* - -[pytest] -addopts = -x --no-success-flaky-report --cov --cov-config .coveragerc --cov-append --cov-report= - From a5f2daf331bff12735326f06d36d6e46a38725cf Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Mon, 28 Mar 2016 11:07:25 -0700 Subject: [PATCH 3/5] Adding gae test session --- .travis.yml | 1 - nox.py | 96 ++++++++++++++++++++++++++++++++++++++--------------- 2 files changed, 70 insertions(+), 27 deletions(-) diff --git a/.travis.yml b/.travis.yml index b02e7129092c..8b43097f2c0b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,6 @@ env: - PATH=${PATH}:${HOME}/gcloud/google-cloud-sdk/bin - GOOGLE_APPLICATION_CREDENTIALS=${TRAVIS_BUILD_DIR}/testing/resources/service-account.json - GOOGLE_CLIENT_SECRETS=${TRAVIS_BUILD_DIR}/testing/resources/client-secrets.json - - GAE_PYTHONPATH=${HOME}/.cache/google_appengine - GAE_ROOT=${HOME}/.cache/ - secure: Orp9Et2TIwCG/Hf59aa0NUDF1pNcwcS4TFulXX175918cFREOzf/cNZNg+Ui585ZRFjbifZdc858tVuCVd8XlxQPXQgp7bwB7nXs3lby3LYg4+HD83Gaz7KOWxRLWVor6IVn8OxeCzwl6fJkdmffsTTO9csC4yZ7izHr+u7hiO4= before_install: diff --git a/nox.py b/nox.py index 14603778c786..04e37b889ab6 100644 --- a/nox.py +++ b/nox.py @@ -1,11 +1,55 @@ import fnmatch import os +import tempfile import nox REPO_TOOLS_REQ =\ 'git+https://github.com/GoogleCloudPlatform/python-repo-tools.git' +COMMON_PYTEST_ARGS = [ + '-x', '--no-success-flaky-report', '--cov', '--cov-config', + '.coveragerc', '--cov-append', '--cov-report='] + +SAMPLES = [ + 'bigquery/api', + 'blog/introduction_to_data_models_in_cloud_datastore', + 'cloud_logging/api', + 'compute/api', + 'compute/autoscaler/demo', + 'datastore/api', + 'managed_vms/cloudsql', + 'managed_vms/datastore', + 'managed_vms/disk', + 'managed_vms/extending_runtime', + 'managed_vms/hello_world', + 'managed_vms/hello_world_compat', + 'managed_vms/memcache', + 'managed_vms/pubsub', + 'managed_vms/static_files', + 'managed_vms/storage', + 'monitoring/api', + 'storage/api', +] + +GAE_SAMPLES = [ + 'appengine/app_identity/signing', + 'appengine/bigquery', + 'appengine/blobstore', + 'appengine/cloudsql', + 'appengine/images', + 'appengine/localtesting', + 'appengine/logging/reading_logs', + 'appengine/logging/writing_logs', + 'appengine/mailgun', + 'appengine/memcache/guestbook', + 'appengine/multitenancy', + 'appengine/ndb/modeling', + 'appengine/ndb/overview', + 'appengine/ndb/transactions', + 'appengine/storage', +] + def session_lint(session): session.install('flake8', 'flake8-import-order') @@ -36,32 +80,6 @@ def session_reqcheck(session): session.run('gcprepotools', command, reqfile) -COMMON_PYTEST_ARGS = [ - '-x', '--no-success-flaky-report', '--cov', '--cov-config', - '.coveragerc', '--cov-append', '--cov-report='] - -SAMPLES = [ - 'bigquery/api', - 'blog/introduction_to_data_models_in_cloud_datastore', - 'cloud_logging/api', - 'compute/api', - 'compute/autoscaler/demo', - 'datastore/api', - 'managed_vms/cloudsql', - 'managed_vms/datastore', - 'managed_vms/disk', - 'managed_vms/extending_runtime', - 'managed_vms/hello_world', - 'managed_vms/hello_world_compat', - 'managed_vms/memcache', - 'managed_vms/pubsub', - 'managed_vms/static_files', - 'managed_vms/storage', - 'monitoring/api', - 'storage/api', -] - - @nox.parametrize('interpreter', ['python2.7', 'python3.4']) def session_tests(session, interpreter, extra_pytest_args=None): session.interpreter = interpreter @@ -77,7 +95,33 @@ def session_tests(session, interpreter, extra_pytest_args=None): success_codes=[0, 5]) # Treat no test collected as success. +def session_gae(session, extra_pytest_args=None): + session.interpreter = 'python2.7' + session.install(REPO_TOOLS_REQ) + session.install('-r', 'requirements-python2.7-dev.txt') + + # Install the app engine sdk and setup import paths. + gae_root = os.environ.get('GAE_ROOT', tempfile.gettempdir()) + session.env['PYTHONPATH'] = os.path.join(gae_root, 'google_appengine') + session.run('gcprepotools', 'download-appengine-sdk', gae_root) + + # Create a lib directory to prevent the GAE vendor library from + # complaining. + if not os.path.exists('lib'): + os.makedirs('lib') + + pytest_args = COMMON_PYTEST_ARGS + (extra_pytest_args or []) + + for sample in (session.posargs or GAE_SAMPLES): + session.run( + 'py.test', sample, + *pytest_args, + success_codes=[0, 5]) # Treat no test collected as success. + + def session_travis(session): """On travis, just run with python3.4 and don't run slow or flaky tests.""" session_tests( session, 'python3.4', extra_pytest_args=['-m not slow and not flaky']) + session_gae( + session, extra_pytest_args=['-m not slow and not flaky']) From 41034ff489f8ecdf00ecd0ddd51654dabd56c1cd Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Mon, 28 Mar 2016 11:07:41 -0700 Subject: [PATCH 4/5] Removing tox.ini --- tox.ini | 135 -------------------------------------------------------- 1 file changed, 135 deletions(-) delete mode 100644 tox.ini diff --git a/tox.ini b/tox.ini deleted file mode 100644 index acdce8ac6753..000000000000 --- a/tox.ini +++ /dev/null @@ -1,135 +0,0 @@ -[tox] -skipsdist = True -envlist = pep8, py27, py34, gae - -[testenv] -passenv = * -basepython = python2.7 -examples = - bigquery/api - blog/introduction_to_data_models_in_cloud_datastore - cloud_logging/api - compute/api - compute/autoscaler/demo - datastore/api - managed_vms/cloudsql - managed_vms/datastore - managed_vms/disk - managed_vms/extending_runtime - managed_vms/hello_world - managed_vms/hello_world_compat - managed_vms/memcache - managed_vms/pubsub - managed_vms/static_files - managed_vms/storage - monitoring/api - storage/api -setenv = - PYTHONPATH=. - -[testenv:pep8] -deps = - flake8 - flake8-import-order -commands = - flake8 --builtin=gettext --max-complexity=10 --import-order-style=google {posargs} - -[testenv:py27] -deps = - git+https://github.com/GoogleCloudPlatform/python-repo-tools - -rrequirements-py27-dev.txt -commands = - python scripts/run-tests.py {posargs:{[testenv]examples}} - -[testenv:py27-all] -deps = - {[testenv:py27]deps} -commands = - python scripts/run-tests.py \ - --junit \ - --run-slow \ - {posargs:{[testenv]examples}} - -[testenv:py34] -basepython = python3.4 -deps = - git+https://github.com/GoogleCloudPlatform/python-repo-tools - -rrequirements-py34-dev.txt -commands = - python scripts/run-tests.py {posargs:{[testenv]examples}} - -[testenv:py34-all] -basepython = python3.4 -deps = - {[testenv:py34]deps} -commands = - python scripts/run-tests.py \ - --junit \ - --run-slow \ - {posargs:{[testenv]examples}} - -[testenv:gae] -deps = - git+https://github.com/GoogleCloudPlatform/python-repo-tools - -rrequirements-py27-dev.txt -examples = - appengine/app_identity/signing - appengine/bigquery - appengine/blobstore - appengine/cloudsql - appengine/images - appengine/localtesting - appengine/logging/reading_logs - appengine/logging/writing_logs - appengine/mailgun - appengine/memcache/guestbook - appengine/multitenancy - appengine/ndb/modeling - appengine/ndb/overview - appengine/ndb/transactions - appengine/storage -commands = - gcprepotools download-appengine-sdk {env:GAE_ROOT} - # Create a lib directory, otherwise, the vendor library will explode. - mkdir -p lib - python scripts/run-tests.py {posargs:{[testenv:gae]examples}} -setenv = - PYTHONPATH=.:{env:GAE_ROOT:}/google_appengine -whitelist_externals = mkdir - -[testenv:gae-all] -deps = - {[testenv:gae]deps} -commands = - gcprepotools download-appengine-sdk {env:GAE_ROOT} - # Create a lib directory, otherwise, the vendor library will explode. - mkdir -p lib - python scripts/run-tests.py \ - --junit \ - --run-slow \ - {posargs:{[testenv:gae]examples}} -setenv = - PYTHONPATH=.:{env:GAE_ROOT:}/google_appengine -whitelist_externals = mkdir - -[testenv:reqcheck] -deps= - git+https://github.com/GoogleCloudPlatform/python-repo-tools -commands = - bash -c "find . -name requirements\*.txt |\ - xargs -P 4 -n 1 gcprepotools check-requirements" -whitelist_externals = - bash - find - xargs - -[testenv:requpdate] -deps= - git+https://github.com/GoogleCloudPlatform/python-repo-tools -commands = - bash -c "find . -name requirements\*.txt |\ - xargs -P 4 -n 1 gcprepotools update-requirements" -whitelist_externals = - bash - find - xargs From 879176b1a11ee330e0a461f804b49620d7704abe Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Tue, 29 Mar 2016 12:27:45 -0700 Subject: [PATCH 5/5] Adding some helpful comments to the noxfile. --- conftest.py | 14 ++++++++++++++ nox.py | 18 ++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/conftest.py b/conftest.py index 92a808d07b0c..8bf0e8d2dcf4 100644 --- a/conftest.py +++ b/conftest.py @@ -1,3 +1,17 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os import pytest diff --git a/nox.py b/nox.py index 04e37b889ab6..a3d3f0a0e15c 100644 --- a/nox.py +++ b/nox.py @@ -1,3 +1,17 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import fnmatch import os import tempfile @@ -86,8 +100,12 @@ def session_tests(session, interpreter, extra_pytest_args=None): session.install(REPO_TOOLS_REQ) session.install('-r', 'requirements-{}-dev.txt'.format(interpreter)) + # extra_pytest_args can be send by another session calling this session, + # see session_travis. pytest_args = COMMON_PYTEST_ARGS + (extra_pytest_args or []) + # session.posargs is any leftover arguments from the command line, which + # allows users to run a particular test instead of all of them. for sample in (session.posargs or SAMPLES): session.run( 'py.test', sample,