azure-pipelines.yml

trigger:
  # start a new build for every push
  batch: False
  branches:
    include:
      - main
      - maintenance/*


pr:
  branches:
    include:
    - '*'  # must quote since "*" is a YAML reserved character; we want a string


stages:

- stage: Check
  jobs:
    - job: Skip
      pool:
        vmImage: 'ubuntu-20.04'
      variables:
        DECODE_PERCENTS: 'false'
        RET: 'true'
      steps:
      - bash: |
          git_log=`git log --max-count=1 --skip=1 --pretty=format:"%B" | tr "\n" " "`
          echo "##vso[task.setvariable variable=log]$git_log"
      - bash: echo "##vso[task.setvariable variable=RET]false"
        condition: or(contains(variables.log, '[skip azp]'), contains(variables.log, '[azp skip]'), contains(variables.log, '[skip ci]'), contains(variables.log, '[ci skip]'))
      - bash: echo "##vso[task.setvariable variable=start_main;isOutput=true]$RET"
        name: result

- stage: InitialTests
  condition: and(succeeded(), eq(dependencies.Check.outputs['Skip.result.start_main'], 'true'))
  dependsOn: Check
  jobs:

  # Native build is based on gcc flag `-march=native`
  - job: Linux_baseline_native
    pool:
      vmImage: 'ubuntu-20.04'
    steps:
    - script: |
        git submodule update --init
      displayName: 'Fetch submodules'
    - script: |
            if ! `gcc 2>/dev/null`; then
                sudo apt install gcc
            fi
            sudo apt install python3
            sudo apt install python3-dev
            # python3 has no setuptools, so install one to get us going
            python3 -m pip install --user --upgrade pip 'setuptools<49.2.0'
            python3 -m pip install --user -r test_requirements.txt
      displayName: 'install python/requirements'
    - script: |
            python3 runtests.py --show-build-log --cpu-baseline=native --cpu-dispatch=none \
            --debug-info --mode=full -- -rsx --junitxml=junit/test-results.xml
      displayName: 'Run native baseline Build / Tests'
    - task: PublishTestResults@2
      condition: succeededOrFailed()
      inputs:
        testResultsFiles: '**/test-*.xml'
        failTaskOnFailedTests: true
        testRunTitle: 'Publish test results for baseline/native'

- stage: ComprehensiveTests
  condition: and(succeeded(), eq(dependencies.Check.outputs['Skip.result.start_main'], 'true'))
  dependsOn: Check
  jobs:

  - job: Lint
    condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest'))
    pool:
      vmImage: 'ubuntu-20.04'
    steps:
    - task: UsePythonVersion@0
      inputs:
        versionSpec: '3.8'
        addToPath: true
        architecture: 'x64'
    - script: >-
        python -m pip install -r linter_requirements.txt
      displayName: 'Install tools'
      # pip 21.1 emits a pile of garbage messages to annoy users :)
      #      failOnStderr: true
    - script: |
        python tools/linter.py --branch origin/$(System.PullRequest.TargetBranch)
      displayName: 'Run Lint Checks'
      failOnStderr: true

  - job: Linux_Python_38_32bit_full_with_asserts
    pool:
      vmImage: 'ubuntu-20.04'
    steps:
    - script: |
        git submodule update --init
      displayName: 'Fetch submodules'
    - script: |
            docker run -v $(pwd):/numpy -e CFLAGS="-msse2 -std=c99 -UNDEBUG" \
            -e F77=gfortran-5 -e F90=gfortran-5 quay.io/pypa/manylinux2014_i686 \
            /bin/bash -xc " \
            git config --global --add safe.directory /numpy && \
            cd /numpy && \
            /opt/python/cp38-cp38/bin/python -mvenv venv && \
            source venv/bin/activate && \
            target=\$(python3 tools/openblas_support.py) && \
            cp -r \$target/lib/* /usr/lib && \
            cp \$target/include/* /usr/include && \
            python3 -m pip install -r test_requirements.txt && \
            echo CFLAGS \$CFLAGS && \
            python3 -m pip install -v . && \
            python3 runtests.py -n --debug-info --mode=full -- -rsx --junitxml=junit/test-results.xml && \
            python3 tools/openblas_support.py --check_version"
      displayName: 'Run 32-bit manylinux2014 Docker Build / Tests'
    - task: PublishTestResults@2
      condition: succeededOrFailed()
      inputs:
        testResultsFiles: '**/test-*.xml'
        failTaskOnFailedTests: true
        testRunTitle: 'Publish test results for Python 3.8-32 bit full Linux'


  - job: macOS
    pool:
      vmImage: 'macOS-11'
    strategy:
      maxParallel: 3
      matrix:
          Python38:
            PYTHON_VERSION: '3.8'
            USE_OPENBLAS: '1'
          Python38-ILP64:
            PYTHON_VERSION: '3.8'
            NPY_USE_BLAS_ILP64: '1'
            USE_OPENBLAS: '1'
    steps:
    # the @0 refers to the (major) version of the *task* on Microsoft's
    # end, not the order in the build matrix nor anything to do
    # with version of Python selected
    - task: UsePythonVersion@0
      inputs:
        versionSpec: $(PYTHON_VERSION)
        addToPath: true
        architecture: 'x64'
    - script: |
        set -xe
        [ -n "$USE_XCODE_10" ] && /bin/bash -c "sudo xcode-select -s /Applications/Xcode_10.app/Contents/Developer"
        clang --version
      displayName: 'report clang version'
    # NOTE: might be better if we could avoid installing
    # two C compilers, but with homebrew looks like we're
    # now stuck getting the full gcc toolchain instead of
    # just pulling in gfortran
    - script: |
        set -xe
        # same version of gfortran as the open-libs and numpy-wheel builds
        curl -L https://github.com/MacPython/gfortran-install/raw/master/archives/gfortran-4.9.0-Mavericks.dmg -o gfortran.dmg
        GFORTRAN_SHA256=$(shasum -a 256 gfortran.dmg)
        KNOWN_SHA256="d2d5ca5ba8332d63bbe23a07201c4a0a5d7e09ee56f0298a96775f928c3c4b30  gfortran.dmg"
        if [ "$GFORTRAN_SHA256" != "$KNOWN_SHA256" ]; then
            echo sha256 mismatch
            exit 1
        fi
        hdiutil attach -mountpoint /Volumes/gfortran gfortran.dmg
        sudo installer -pkg /Volumes/gfortran/gfortran.pkg -target /
        otool -L /usr/local/gfortran/lib/libgfortran.3.dylib
        # Manually symlink gfortran-4.9 to plain gfortran for f2py.
        # No longer needed after Feb 13 2020 as gfortran is already present
        # and the attempted link errors. Keep this for future reference.
        # ln -s /usr/local/bin/gfortran-4.9 /usr/local/bin/gfortran
      displayName: 'make libgfortran available on mac os for openblas'
    # use the pre-built openblas binary that most closely
    # matches our MacOS wheel builds -- currently based
    # primarily on file size / name details
    - script: |
        set -xe
        target=$(python tools/openblas_support.py)
        ls -lR $target
        # manually link to appropriate system paths
        cp $target/lib/lib* /usr/local/lib/
        cp $target/include/* /usr/local/include/
        otool -L /usr/local/lib/libopenblas*
      displayName: 'install pre-built openblas'
      condition: eq(variables['USE_OPENBLAS'], '1')
    - script: python -m pip install --upgrade pip 'setuptools<49.2.0' wheel
      displayName: 'Install tools'
    - script: |
        python -m pip install -r test_requirements.txt
        # Don't use doc_requirements.txt since that messes up tests
        python -m pip install vulture sphinx==4.3.0 numpydoc==1.4.0
      displayName: 'Install dependencies; some are optional to avoid test skips'
    - script: /bin/bash -c "! vulture . --min-confidence 100 --exclude doc/,numpy/distutils/ | grep 'unreachable'"
      displayName: 'Check for unreachable code paths in Python modules'
    # prefer usage of clang over gcc proper
    # to match likely scenario on many user mac machines
    - script: python setup.py build -j 4 build_src --verbose-cfg install
      displayName: 'Build NumPy'
      env:
        BLAS: None
        LAPACK: None
        ATLAS: None
        CC: /usr/bin/clang
      condition: eq(variables['USE_OPENBLAS'], '1')
    - script: python setup.py build -j 4 build_ext --inplace install
      displayName: 'Build NumPy without OpenBLAS and new casting'
      env:
        BLAS: None
        LAPACK: None
        ATLAS: None
        CC: /usr/bin/clang
      condition: eq(variables['USE_OPENBLAS'], '0')
    # wait until after dev build of NumPy to pip
    # install matplotlib to avoid pip install of older numpy
    - script: python -m pip install matplotlib
      displayName: 'Install matplotlib before refguide run'
    - script: python runtests.py -g --refguide-check
      displayName: 'Run Refguide Check'
      condition: eq(variables['USE_OPENBLAS'], '1')
    - script: |
        echo LIBRARY_PATH ${LIBRARY_PATH}
        python runtests.py -n --mode=full -- -rsx --junitxml=junit/test-results.xml
      displayName: 'Run Full NumPy Test Suite'
      condition: eq(variables['USE_OPENBLAS'], '1')
      env:
        # gfortran installed above adds -lSystem, so this is needed to find it (gh-22043)
        LIBRARY_PATH: /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib
    - bash: python tools/openblas_support.py --check_version
      displayName: 'Verify OpenBLAS version'
      condition: eq(variables['USE_OPENBLAS'], '1')
    # import doesn't work when in numpy src directory , so do a pip dev install of build lib to test
    - script: |
        #!/bin/bash -v
        set +e
        python -c "import numpy as np" > test_output.log 2>&1
        check_output_code=$?
        cat test_output.log
        grep "buggy Accelerate backend"  test_output.log
        check_message=$?
        if [ $check_output_code == 1 ] && [ $check_message == 0 ]; then exit 0; else exit 1;fi
      displayName: "Check if numpy import fails with accelerate"
      condition: eq(variables['USE_OPENBLAS'], '0')
    - task: PublishTestResults@2
      condition: succeededOrFailed()
      inputs:
        testResultsFiles: '**/test-*.xml'
        failTaskOnFailedTests: true
        testRunTitle: 'Publish test results for Python 3.8 64-bit full Mac OS'


  - job: Windows
    pool:
      vmImage: 'windows-2019'
    strategy:
      maxParallel: 6
      matrix:
          Python38-32bit-fast:
            PYTHON_VERSION: '3.8'
            PYTHON_ARCH: 'x86'
            TEST_MODE: fast
            BITS: 32
          Python38-64bit-full:
            PYTHON_VERSION: '3.8'
            PYTHON_ARCH: 'x64'
            TEST_MODE: full
            BITS: 64
          Python39-32bit-fast:
            PYTHON_VERSION: '3.9'
            PYTHON_ARCH: 'x86'
            TEST_MODE: fast
            BITS: 32
          Python39-64bit-full:
            PYTHON_VERSION: '3.9'
            PYTHON_ARCH: 'x64'
            TEST_MODE: full
            BITS: 64
            NPY_USE_BLAS_ILP64: '1'
          PyPy38-64bit-fast:
            PYTHON_VERSION: 'PyPy'
            PYTHON_ARCH: 'x64'
            TEST_MODE: fast
            BITS: 64
            NPY_USE_BLAS_ILP64: '1'

    steps:
    - template: azure-steps-windows.yml


  - job: Linux_conda
    pool:
      vmImage: 'ubuntu-20.04'
    steps:
    - script: |
        git submodule update --init
      displayName: 'Fetch submodules'
    - script: |
            # create and activate conda environment
            conda env create -f environment.yml
      displayName: 'Create conda environment.'
    - script: |
            # >>> conda initialize >>>
            # !! Contents within this block are 'conda init' !!
            # see https://github.com/conda/conda/issues/7980
            __conda_setup="$('conda' 'shell.bash' 'hook' 2> /dev/null)"
            eval "$__conda_setup"
            unset __conda_setup
            # <<< conda initialize <<<
            conda activate numpy-dev
            # Run native baseline Build / Tests
            python runtests.py --show-build-log --cpu-baseline=native --cpu-dispatch=none \
            --debug-info --mode=full -- -rsx --junitxml=junit/test-results.xml
      displayName: 'Run native baseline Build / Tests in conda.'
    - task: PublishTestResults@2
      condition: succeededOrFailed()
      inputs:
        testResultsFiles: '**/test-*.xml'
        failTaskOnFailedTests: true
        testRunTitle: 'Publish test results for conda installation'