From 2cae67905da7a36d390c75ad297ba4335f2c7e1e Mon Sep 17 00:00:00 2001 From: Humza Iqbal Date: Thu, 2 Dec 2021 20:42:13 -0800 Subject: [PATCH] Migrating travis -> circle (#1685) ## Description of proposed changes Replacing the CI system from Travis to Circle as we have seen issues running on Travis ## Related issue(s) N/A Fixes # (issue) N/A ## Test plan Having the CI tests run be Circle instead of Travis should test this ## Checklist Need help on these? Just ask! * [x] I have read the **CONTRIBUTING** document. * [x] I have updated the documentation accordingly. * [x] I have added tests to cover my changes. * [x] I have run `tox -e complex` and/or `tox -e spark` if appropriate. * [x] All new and existing tests passed. --- .circleci/config.yml | 154 +++++++++++++++++++++++++++++++++++++++++++ .travis.yml | 70 -------------------- CONTRIBUTING.md | 4 +- 3 files changed, 156 insertions(+), 72 deletions(-) create mode 100644 .circleci/config.yml delete mode 100644 .travis.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 000000000..d04e2458e --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,154 @@ +# Use the latest 2.1 version of CircleCI pipeline process engine. +version: 2.1 + +orbs: + python: circleci/python@1.2 + +commands: + setup_dependencies: + description: "Install depenencies" + parameters: + after-deps: + description: "Install dependenceis" + type: steps + default: [] + steps: + - run: + name: "Install open JDK" + command: sudo add-apt-repository -y ppa:openjdk-r/ppa + - run: + name: "Install qq" + command: sudo apt-get -qq update + - run: + name: "No install recommends for JDK" + command: sudo apt-get install -y openjdk-8-jdk --no-install-recommends + - run: + name: "Run Java Alternatives install for JDK" + command: sudo update-java-alternatives -s java-1.8.0-openjdk-amd64 + - run: + name: "Run pip install setup tools and wheel" + command: pip install -U pip setuptools wheel + - run: + name: "Install Tox" + command: pip install -U tox==3.12.0 + - run: + name: "Install Code Cov" + command: pip install -U codecov + - steps: << parameters.after-deps >> + +# We want to make sure we run this only on master branch, release, or when we make tags +run_complex: &run_complex + filters: + branches: + only: + - (master|release-v.*) + tags: + only: /.*/ + +jobs: + Python38-Unit-Tests: + docker: + - image: cimg/python:3.8 + environment: + TOXENV: coverage,doctest,type,check + TOX_INSTALL_DIR: .env + JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + + steps: + - checkout + - setup_dependencies + - run: + name: "Run Tox" + command: tox + + Python37-Unit-Tests: + docker: + - image: cimg/python:3.7 + environment: + TOXENV: coverage,doctest,type,check + TOX_INSTALL_DIR: .env + JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + + steps: + - checkout + - setup_dependencies + - run: + name: "Run Tox" + command: tox + + Python36-Unit-Tests: + docker: + - image: cimg/python:3.6 + environment: + TOXENV: coverage,doctest,type,check + TOX_INSTALL_DIR: .env + JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + + steps: + - checkout + - setup_dependencies + - run: + name: "Run Tox" + command: tox + Python38-Integration-Tests: + docker: + - image: cimg/python:3.8 + environment: + TOXENV: complex,type,check + TOX_INSTALL_DIR: .env + JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + + steps: + - checkout + - setup_dependencies + - run: + name: "Run Tox" + command: tox + + Python37-Integration-Tests: + docker: + - image: cimg/python:3.7 + environment: + TOXENV: complex,spark,type,check + TOX_INSTALL_DIR: .env + JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + + steps: + - checkout + - setup_dependencies + - run: + name: "Run Tox" + command: tox + + Python36-Integration-Tests: + docker: + - image: cimg/python:3.6 + environment: + TOXENV: coverage,complex,spark,doctest,type,check + TOX_INSTALL_DIR: .env + JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + + steps: + - checkout + - setup_dependencies + - run: + name: "Run Tox" + command: tox + +workflows: + version: 2 + + Interation-Tests: + jobs: + - Python38-Integration-Tests: + <<: *run_complex + - Python37-Integration-Tests: + <<: *run_complex + - Python36-Integration-Tests: + <<: *run_complex + Unit-Tests: + jobs: + - Python37-Unit-Tests + - Python36-Unit-Tests + - Python38-Unit-Tests + diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index c0844705c..000000000 --- a/.travis.yml +++ /dev/null @@ -1,70 +0,0 @@ -# Only run CI on master, release branches, tags, and PRs -if: tag IS present OR type = pull_request OR ((branch = master OR branch =~ release-*) AND type = push) - -# Main dist is Python -language: python - -# Cache package wheels -cache: pip - -# python3.7 only available on xenial -dist: xenial - -jobs: - include: - - if: type = pull_request - name: "Python 3.6: unit tests" - python: 3.6 - env: TOXENV=coverage,doctest,type,check TOX_INSTALL_DIR=.env JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 - - if: type = pull_request - name: "Python 3.7: unit tests" - python: 3.7 - env: TOXENV=type,check TOX_INSTALL_DIR=.env JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 - - if: type = pull_request - name: "Python 3.8: unit tests" - python: 3.8 - env: TOXENV=type,check TOX_INSTALL_DIR=.env JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 - - if: type != pull_request - name: "Python 3.6: unit + integration tests" - python: 3.6 - env: TOXENV=coverage,complex,spark,doctest,type,check TOX_INSTALL_DIR=.env JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 - - if: type != pull_request - name: "Python 3.7: unit + integration tests" - python: 3.7 - env: TOXENV=complex,spark,type,check TOX_INSTALL_DIR=.env JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 - - if: type != pull_request - name: "Python 3.8: unit + integration tests" - python: 3.8 - env: TOXENV=complex,type,check TOX_INSTALL_DIR=.env JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 - -# Install JDK8 for PySpark tests -before_install: - - sudo add-apt-repository -y ppa:openjdk-r/ppa - - sudo apt-get -qq update - - sudo apt-get install -y openjdk-8-jdk --no-install-recommends - - sudo update-java-alternatives -s java-1.8.0-openjdk-amd64 - -install: - - pip install -U pip setuptools wheel - - pip install -U tox==3.12.0 - - pip install -U codecov - - travis_wait 15 tox --notest - -script: - - tox - -after_success: - - travis_retry codecov - -notifications: - email: - recipients: - secure: "d91inMv9lhRuSDEbT+aRtsrGC0M8VSvloPSoDt1k9k1y1EuduRF3+QuE3kd1T2sz0ARypWvDj1M89Gyfnbdrg/rphmBIIOaLQdjA+zueid6Dpc/6Xxe2nnXCjS+7PL7CkKjHo7iE3XW97Hep7WUEv3A60bD5O9kx0mS9qbKeO5KJTmb7zctVF6s1jz8VGewS+XybDW5ZJ0KDVhvlvunJt+PCDXfGAFxdEuXGF6dOtyAXzZ5h7aRooj06QQPyDO8sDmpbwritR2SXw3OBZi3xO4qZOfLyFWtuo6VvMRTaNa4UNnyP0QGZbHuzZe6/SAqPrU417XwuklVHJQchRFPFrQxcRgNKjOXgD8KM5cuQJJEJIIpppEp0fO902F13Kci9D4Fysd2rCFZWzStTyeyDWYMPtkiNZfWlBVxPUFQcXRAXyz3e7kz6kIVUGnBiuruYrmBb5iuz9dlB8vJfeIml4iAoPnUs1J+NNiirqqRPt629Y3LaISWxcIN/v+oQp+W1XxmwjSf/FhpTDsqG1if7ks4VMjdoFgR76bZ8qBiKBLsdggJPyTljoOs+Tnhe18P2szfNyL47u+/QaZtliR2+bS7dmr9Sd5ol4cfb71xgGJBcJGf3e5OcgAgy3vZdshEGTGHxN+ykRRK4iOcFPjnyLwJmUUEbbn6jpFE0H3N1r2s=" - on_success: change # default: change [change, always, never] - on_failure: change # default: always - slack: - on_success: change # default: change - on_failure: change # default: always - on_pull_requests: false - room: - secure: "jUQz1DEG+BfCpVYOOolZgyhMddv4arkx0/VY5AQveST1oo3jVsFkHhLzwFTjDnh+YSEVReHGV5BrLmf2Va5jq03AKUUA3CAvcHlZVQElbo06NE1zbtmTXlllSc8+34Fn3SjfcKs565jHTP2YxpK4cEB0rUN2fBimKqGHdzEqCwBJ++6PiWFgudnoXafTTkfKU2DFiyE5V+PpNiLEdwtxDwJr4HZnbBdkd+yXLyrzVsCaQp0D0lPQZyB0pMJBk/awnkQg9OVsM+9VXZtPOeDd0oPAXxp6m+pnVR2dNRZoKlxEEWEzobkMj44amE0PPnE/YfsBE1T9MytJJ9XsXDwswrAmuHGoEKp/lUySlfhkCWfi6/JXy0NFAXTzWbau4Tlr3bqA1vKieDLy6OV4GbOK3BlDIdhk+toHU4zCWLtn1PT9+Hey6ajdNrpwUKkbM8wBuccqSw6bukFl1E2/UW8hqG1iYCMq86+I1UKalcd/HCSrPraioFic6rPs0f5YvQePjd2o/c00n+fbDgmj2oCrXZpnyc4LYbRO4c8pT7F0ozwhRNA/LtpoKDcZCE3MTBtDR/Nh6JZaLsjpSEE1MUsAUIEd17zfkWlig/mWD9IepMgTsnNARmXOeBM+Ih/VoRECXmlcb/GQcRIc8/8FiNW2j+V6q686nK701WYh5hQqVpU=" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bc8168e9a..9c532c9d8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -112,7 +112,7 @@ Any test that runs longer than half a second should be marked with the `@pytest.mark.complex` decorator. Typically, these will be integration tests or tests that verify complex properties like model convergence. -We exclude long-running tests from the default `tox` and Travis builds +We exclude long-running tests from the default `tox` and Circle CI builds on non-master and non-release branches to keep things moving fast. If you're touching areas of the code that could break a long-running test, you should include the results of `tox -e complex` in the PR's test plan. @@ -124,7 +124,7 @@ To see the durations of the 10 longest-running tests, run PySpark tests are invoked separately from the rest since they require installing Java and the large PySpark package. -They are executed on Travis, but not by default for a local `tox` command. +They are executed on Circle CI, but not by default for a local `tox` command. If you're making changes to Spark-based operators, make sure you have Java 8 installed locally and then run `tox -e spark`. If you add a test that imports PySpark mark it with the