azure-pipelines-dependency-graph-testing.yml

# This pipeline is meant to run the GE test suite with an experimental test runner strategy.
# The significant changes between this YAML and the primary azure-pipelines.yml file are:
#  - Utilizes a custom script to filter the test files selected and passed on to pytest
#  - Removes stages that aren't relevant to testing this strategy (i.e. usage stats)
#  - Only tests Python 3.8 to improve performance

trigger:
  branches:
    include:
    - pre_pr-*
    - develop
    - main

resources:
  containers:
  - container: postgres
    image: postgres:11
    ports:
    - 5432:5432
    env:
      POSTGRES_DB: "test_ci"
      POSTGRES_HOST_AUTH_METHOD: "trust"
  - container: mysql
    image: mysql:8.0.20
    ports:
      - 3306:3306
    env:
      MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
      MYSQL_DATABASE: test_ci
  - container: mssql
    image: mcr.microsoft.com/mssql/server:2019-latest
    env:
      ACCEPT_EULA: Y
      MSSQL_SA_PASSWORD: ReallyStrongPwd1234%^&*
      MSSQL_DB: test_ci
      MSSQL_PID: Developer
    ports:
      - 1433:1433

variables:
  isMain: $[eq(variables['Build.SourceBranch'], 'refs/heads/main')]
  isDevelop: $[eq(variables['Build.SourceBranch'], 'refs/heads/develop')]
  GE_USAGE_STATISTICS_URL: "https://qa.stats.greatexpectations.io/great_expectations/v1/usage_statistics"

stages:
  - stage: scope_check
    pool:
      vmImage: 'ubuntu-20.04'
    jobs:
      - job: changes
        steps:
          - task: ChangedFiles@1
            name: CheckChanges
            inputs:
              verbose: true
              rules: |
                [ContribChanged]
                contrib/**

                [ExperimentalChanged]
                contrib/experimental/**

                [DocsChanged]
                docs/**
                tests/integration/docusaurus/**
                tests/integration/fixtures/**
                tests/test_sets/**

                [GEChanged]
                great_expectations/**
                tests/**
                /*.txt
                /*.yml

  - stage: lint
    dependsOn: scope_check
    pool:
      vmImage: 'ubuntu-latest'

    jobs:
      - job: lint
        condition: or(eq(stageDependencies.scope_check.changes.outputs['CheckChanges.GEChanged'], true), eq(variables.isMain, true))
        steps:
          - task: UsePythonVersion@0
            inputs:
              versionSpec: 3.7
            displayName: 'Use Python 3.7'

          - script: |
              pip install isort[requirements]==5.4.2 flake8==3.8.3 black==21.8b0 pyupgrade==2.7.2
              EXIT_STATUS=0
              isort . --check-only --skip docs/ || EXIT_STATUS=$?
              black --check --exclude docs/ . || EXIT_STATUS=$?
              flake8 great_expectations/core || EXIT_STATUS=$?
              pyupgrade --py3-plus || EXIT_STATUS=$?
              exit $EXIT_STATUS

  - stage: required
    dependsOn: [scope_check, lint]
    pool:
      vmImage: 'ubuntu-18.04'

    jobs:
      - job: compatibility_matrix
        condition: or(eq(stageDependencies.scope_check.changes.outputs['CheckChanges.GEChanged'], true), eq(variables.isMain, true))
        variables:
          GE_pytest_opts: '--no-sqlalchemy --no-spark'
        strategy:
          matrix:
            Python38-PandasLatest:
              python.version: '3.8'
              numpy.version: 'latest'
              pandas.version: 'latest'
              scipy.version: 'latest'
              GE_pytest_pip_opts: '--requirement requirements-dev-base.txt --constraint constraints-dev.txt'

        steps:
          - task: UsePythonVersion@0
            inputs:
              versionSpec: '$(python.version)'
            displayName: 'Use Python $(python.version)'

          - bash: python -m pip install --upgrade pip  # --upgrade pip==20.2.4
            displayName: 'Update pip'

          - bash: pip install numpy
            condition: eq(variables['scipy.version'], 'numpy')
            displayName: 'Install numpy latest'

          - bash: pip install pandas
            condition: eq(variables['pandas.version'], 'latest')
            displayName: 'Install pandas latest'

          - bash: pip install scipy
            condition: eq(variables['scipy.version'], 'latest')
            displayName: 'Install scipy latest'

          - bash: pip install pandas==$(pandas.version)
            condition: ne(variables['pandas.version'], 'latest')
            displayName: 'Install pandas - $(pandas.version)'

          - script: |
              pip install $(GE_pytest_pip_opts)
              pip install  --requirement requirements.txt
              # Consider fragmenting *all* integration tests into separate folder and run
              pip install  .
            displayName: 'Install dependencies'

          - script: |
              # Install dependencies
              pip install pytest pytest-cov pytest-azurepipelines

              # Run script and save output to a temp file
              echo "Tests identified by dependency graph test runner [dgtest]:"
              python scripts/determine_tests_to_run.py --depth 3 --ignore 'tests/cli' 'tests/integration/usage_statistics' | tee .dgtest

              # If the script found any matches, feed them into pytest
              EXIT_STATUS=0
              if [ -s .dgtest ]; then
                cat .dgtest | xargs pytest $(GE_pytest_opts) --napoleon-docstrings --junitxml=junit/test-results.xml --cov=. --cov-report=xml --cov-report=html
                EXIT_STATUS=$?
              fi

              # Clean up
              rm .dgtest
              exit $EXIT_STATUS
            displayName: 'pytest'

          - task: PublishTestResults@2
            condition: succeededOrFailed()
            inputs:
              testResultsFiles: '**/test-*.xml'
              testRunTitle: 'Publish test results for Python $(python.version)'

          - task: PublishCodeCoverageResults@1
            inputs:
              codeCoverageTool: Cobertura
              summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml'
              reportDirectory: '$(System.DefaultWorkingDirectory)/**/htmlcov'

      - job: comprehensive
        condition: or(eq(stageDependencies.scope_check.changes.outputs['CheckChanges.GEChanged'], true), eq(variables.isMain, true))

        services:
          postgres: postgres

        variables:
          GE_pytest_opts: ''

        strategy:
          matrix:
            Python38:
              python.version: '3.8'
              pandas.version: 'latest'
              GE_pytest_pip_opts: '--requirement requirements-dev.txt --constraint constraints-dev.txt'

        steps:
          - task: UsePythonVersion@0
            inputs:
              versionSpec: '$(python.version)'
            displayName: 'Use Python $(python.version)'

          - bash: python -m pip install --upgrade pip  # pip==20.2.4
            displayName: 'Update pip'

          - script: |
              sudo apt-get install -y pandoc
              pip install pypandoc
            displayName: 'Install pandoc'

          - bash: pip install pandas
            condition: eq(variables['pandas.version'], 'latest')
            displayName: 'Install pandas latest'

          - bash: pip install pandas==$(pandas.version)
            condition: ne(variables['pandas.version'], 'latest')
            displayName: 'Install pandas - $(pandas.version)'

          - script: |
              pip install --requirement requirements.txt
              echo "about to run pip install $(GE_pytest_pip_opts)"
              pip install $(GE_pytest_pip_opts)
              pip install  .
            displayName: 'Install dependencies'

          - script: |
              # Install dependencies
              pip install pytest pytest-cov pytest-azurepipelines

              # Run script and save output to a temp file
              echo "Tests identified by dependency graph test runner [dgtest]:"
              python scripts/determine_tests_to_run.py --depth 3 --ignore 'tests/cli' 'tests/integration/usage_statistics' | tee .dgtest

              # If the script found any matches, feed them into pytest
              EXIT_STATUS=0
              if [ -s .dgtest ]; then
                cat .dgtest | xargs pytest $(GE_pytest_opts) --napoleon-docstrings --junitxml=junit/test-results.xml --cov=. --cov-report=xml --cov-report=html
                EXIT_STATUS=$?
              fi

              # Clean up
              rm .dgtest
              exit $EXIT_STATUS
            displayName: 'pytest'

          - task: PublishTestResults@2
            condition: succeededOrFailed()
            inputs:
              testResultsFiles: '**/test-*.xml'
              testRunTitle: 'Publish test results for Python $(python.version)'

          - task: PublishCodeCoverageResults@1
            inputs:
              codeCoverageTool: Cobertura
              summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml'
              reportDirectory: '$(System.DefaultWorkingDirectory)/**/htmlcov'

  - stage: db_integration
    pool:
      vmImage: 'ubuntu-latest'

    dependsOn: [scope_check, lint]

    jobs:
      - job: mysql
        condition: or(eq(stageDependencies.scope_check.changes.outputs['CheckChanges.GEChanged'], true), eq(variables.isMain, true))

        services:
          mysql: mysql

        variables:
          python.version: '3.8'

        steps:
          - task: UsePythonVersion@0
            inputs:
              versionSpec: '$(python.version)'
            displayName: 'Use Python $(python.version)'

          - bash: python -m pip install --upgrade pip  # pip==20.2.4
            displayName: 'Update pip'

          - script: |
              printf 'Waiting for MySQL database to accept connections'
              until mysql --host=localhost --protocol=TCP --port=3306 --user=root --password='' --execute "SHOW DATABASES"; do
                printf '.'
                sleep 1;
              done;
            displayName: Wait for database to initialise

          - script: |
              echo "SET GLOBAL sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY',''));" > mysql_setup_script.sql
              mysql --host=localhost --protocol=TCP --port=3306 --user=root --password='' --reconnect < mysql_setup_script.sql
            displayName: 'Configure mysql'

          - script: |
              pip install --requirement requirements-dev-base.txt --requirement requirements-dev-sqlalchemy.txt --constraint constraints-dev.txt
              pip install --requirement requirements.txt
              pip install .
            displayName: 'Install dependencies'

          - script: |
              # Install dependencies
              pip install --requirement requirements.txt
              pip install pytest pytest-cov pytest-azurepipelines

              # Run script and save output to a temp file
              echo "Tests identified by dependency graph test runner [dgtest]:"
              python scripts/determine_tests_to_run.py --depth 3 --ignore 'tests/cli' 'tests/integration/usage_statistics' | tee .dgtest

              # If the script found any matches, feed them into pytest
              EXIT_STATUS=0
              if [ -s .dgtest ]; then
                cat .dgtest | xargs pytest --mysql --no-postgresql --no-spark --napoleon-docstrings --junitxml=junit/test-results.xml --cov=. --cov-report=xml --cov-report=html
                EXIT_STATUS=$?
              fi

              # Clean up
              rm .dgtest
              exit $EXIT_STATUS
            displayName: 'pytest'

      - job: mssql
        condition: or(eq(stageDependencies.scope_check.changes.outputs['CheckChanges.GEChanged'], true), eq(variables.isMain, true))

        services:
          mssql: mssql

        variables:
          python.version: '3.8'

        steps:
          - task: UsePythonVersion@0
            inputs:
              versionSpec: '$(python.version)'
            displayName: 'Use Python $(python.version)'

          - bash: python -m pip install --upgrade pip  # pip==20.2.4
            displayName: 'Update pip'

          - script: |
              sqlcmd -U sa -P "ReallyStrongPwd1234%^&*" -Q "CREATE DATABASE test_ci;" -o create_db_output.txt

          - script: |
              pip install --requirement requirements-dev-base.txt --requirement requirements-dev-sqlalchemy.txt --constraint constraints-dev.txt
              pip install --requirement requirements.txt
              pip install .
            displayName: 'Install dependencies'

          - script: |
              # Install dependencies
              pip install pytest pytest-cov pytest-azurepipelines

              # Run script and save output to a temp file
              echo "Tests identified by dependency graph test runner [dgtest]:"
              python scripts/determine_tests_to_run.py --depth 3 --ignore 'tests/cli' 'tests/integration/usage_statistics' | tee .dgtest

              # If the script found any matches, feed them into pytest
              EXIT_STATUS=0
              if [ -s .dgtest ]; then
                cat .dgtest | xargs pytest --mssql --no-postgresql --no-spark --napoleon-docstrings --junitxml=junit/test-results.xml --cov=. --cov-report=xml --cov-report=html
                EXIT_STATUS=$?
              fi

              # Clean up
              rm .dgtest
              exit $EXIT_STATUS
            displayName: 'pytest'

  - stage: cli_integration
    dependsOn: [scope_check, lint, required, db_integration]
    pool:
      vmImage: 'ubuntu-latest'

    jobs:
      - job: test_cli
        condition: or(eq(stageDependencies.scope_check.changes.outputs['CheckChanges.GEChanged'], true), eq(variables.isMain, true))

        services:
          postgres: postgres

        variables:
          python.version: '3.8'

        steps:
          - task: UsePythonVersion@0
            inputs:
              versionSpec: '$(python.version)'
            displayName: 'Use Python $(python.version)'

          - bash: python -m pip install --upgrade pip  # pip==20.2.4
            displayName: 'Update pip'

          - script: |
              sudo apt-get install -y pandoc
              pip install pypandoc
            displayName: 'Install pandoc'

          - script: |
              pip install --requirement requirements-dev-base.txt --requirement requirements-dev-spark.txt --requirement requirements-dev-sqlalchemy.txt --constraint constraints-dev.txt
              pip install --requirement requirements.txt
              pip install .
            displayName: 'Install dependencies'

          - script: |
              # Install dependencies
              pip install pytest pytest-azurepipelines

              # Run script and save output to a temp file
              echo "Tests identified by dependency graph test runner [dgtest]:"
              python scripts/determine_tests_to_run.py --filter 'tests/cli' --depth 3 | tee .dgtest

              # If the script found any matches, feed them into pytest
              EXIT_STATUS=0
              if [ -s .dgtest ]; then
                cat .dgtest | xargs pytest --aws-integration -v
                EXIT_STATUS=$?
              fi

              # Clean up
              rm .dgtest
              exit $EXIT_STATUS
            displayName: 'pytest'