build.yml

name: Build

on: [push, pull_request, workflow_dispatch]

jobs:
  rat:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v2

      - name: rat
        run: make rat clean

  linux-build-docker:
    strategy:
      matrix:
        distro: [ubuntu, centos, debian, i386/ubuntu]
        exclude:
          - distro: ubuntu
          - distro: centos
          - distro: debian
          - distro: i386/ubuntu
        include:
          - distro: ubuntu
            ver: 22.04
            codecov: no
            btype: release
            buser: qfsbuild
          - distro: ubuntu
            ver: 20.04
            codecov: no
            btype: release
            buser: qfsbuild
          - distro: ubuntu
            ver: 18.04
            codecov: no
            btype: release
            buser: qfsbuild
          - distro: ubuntu
            ver: 18.04
            codecov: yes
            btype: release
            buser: qfsbuild
          - distro: ubuntu
            ver: 18.04
            codecov: no
            btype: debug
            buser: qfsbuild
          - distro: ubuntu
            ver: 18.04
            codecov: no
            btype: release
            buser: root
          - distro: ubuntu
            ver: 16.04
            codecov: no
            btype: release
            buser: qfsbuild
          - distro: ubuntu
            ver: 14.04
            codecov: no
            btype: release
            buser: qfsbuild
          - distro: debian
            ver: 10
            codecov: no
            btype: release
            buser: qfsbuild
          - distro: centos
            ver: 6
            codecov: no
            btype: release
            buser: qfsbuild
          - distro: centos
            ver: 7
            codecov: no
            btype: release
            buser: qfsbuild
          - distro: centos
            ver: 8
            codecov: no
            btype: release
            buser: qfsbuild
          - distro: i386/ubuntu
            ver: 18.04
            codecov: no
            btype: release
            buser: qfsbuild
    runs-on: ubuntu-latest
    env:
      BUILD_OS_NAME: linux
      BUILD_RUN_DOCKER: "yes"
      DISTRO: ${{ matrix.distro }}
      VER: ${{ matrix.ver }}
      CODECOV: ${{ matrix.codecov }}
      BTYPE: ${{ matrix.btype }}
      BUSER: ${{ matrix.buser }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v2

      - name: Linux docker build
        run: |
          travis/script.sh
          sudo chown -R "$USER" .git || true

      - name: Upload tarball
        if: |
          ('refs/heads/master' == github.ref
            || startsWith(github.ref, 'refs/tags/'))
          && 'release' == matrix.btype
          && 'qfsbuild' == matrix.buser
          && 'yes' != matrix.codecov
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          AWS_DEFAULT_REGION: us-west-2
        run: aws s3 cp build/qfs-*.tgz s3://quantcast-qfs

  osx-build:
    runs-on: macos-latest
    env:
      BUILD_OS_NAME: osx
    steps:
      - name: Brew install
        run: |
          brew install boost || true
          brew install macfuse || true
          brew install go || true

      - name: Checkout code
        uses: actions/checkout@v2

      - name: MacOS build
        run: travis/script.sh

      - name: Upload tarball
        if: |
          'refs/heads/master' == github.ref
          || startsWith(github.ref, 'refs/tags/')
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          AWS_DEFAULT_REGION: us-west-2
        run: aws s3 cp build/qfs-*.tgz s3://quantcast-qfs

  cygwin-build:
    if: ${{ false }}  # turn off for now
    runs-on: windows-latest
    env:
      CYG_MIRROR: http://cygwin.mirror.constant.com
      DEPENDENCIES: gcc-g++,autoconf,automake,make,cmake,libboost-devel,libssl-devel,bzip2,python,git,libstdc++6-devel,libkrb5-devel,zlib-devel
    steps:
      - name: Set git to use LF
        run: |
          git config --global core.autocrlf false
          git config --global core.eol lf

      - name: Checkout code
        uses: actions/checkout@v2

      - name: Install Hadoop tools, Cygwin, and do build
        shell: pwsh
        run: |
          $cur = (Get-Item -Path "." -Verbose).FullName
          $tools = "$cur\wtools"
          if (!(Test-Path $tools)) {
              New-Item -ItemType Directory -Force -Path $tools | Out-Null
          }
          $hadoopVer = "2.6.0"
          $hadoopPath = "$tools\hadoop"
          if (!(Test-Path $hadoopPath)) {
              New-Item -ItemType Directory -Force -Path $hadoopPath | Out-Null
          }
          Push-Location $hadoopPath
          Invoke-WebRequest `
            -Uri "https://github.com/steveloughran/winutils/archive/master.zip" `
            -OutFile "winutils-master.zip"
          # extract
          Invoke-Expression "7z.exe x winutils-master.zip"
          # add hadoop bin to environment variables
          $env:HADOOP_HOME = "$hadoopPath/winutils-master/hadoop-$hadoopVer"
          Pop-Location

          # Download and install Cygwin
          Push-Location $tools
          $cygwinPath = "$tools\cygwin"
          if (!(Test-Path $cygwinPath)) {
              New-Item -ItemType Directory -Force -Path $cygwinPath | Out-Null
          }
          Invoke-WebRequest -Uri "http://cygwin.com/setup-x86.exe" `
            -OutFile "setup-x86.exe"
          $psi = new-object "Diagnostics.ProcessStartInfo"
          $psi.FileName = "$tools\setup-x86.exe"
          $psi.Arguments = '--quiet-mode --no-shortcuts --only-site --root "'
          $psi.Arguments += "$cygwinPath"
          $psi.Arguments += '" --site "'
          $psi.Arguments += "$env:CYG_MIRROR"
          $psi.Arguments += '" --local-package-dir "'
          $psi.Arguments += "$cygwinPath\var\cache\setup"
          $psi.Arguments += '" --packages "'
          $psi.Arguments += "$env:DEPENDENCIES"
          $psi.Arguments += '"'
          $psi.WorkingDirectory = "$tools"
          $proc = [Diagnostics.Process]::Start($psi)
          $proc.WaitForExit()
          Pop-Location
          # To make "$tools\setup-x86.exe" work, add localhost to /etc/hosts as
          # cygwin has no /etc/resolv.conf, an alternative is to create
          # resolv.conf
          # $hostsEdit = "grep -E '^[^#]+localhost' /etc/hosts || {"
          # $hostsEdit += " chmod +w /etc/hosts"
          # $hostsEdit += "; printf '\r\n127.0.0.1 localhost\r\n' >> /etc/hosts"
          # $hostsEdit += "; grep -E '^[^#]+localhost' /etc/hosts; }"
          # & "$cygwinPath\bin\bash" -lc "$hostsEdit"
          # Check if Cygwin bash works
          & "$cygwinPath\bin\bash" -lc `
            "cygcheck -dc cygwin && cmake --version && mvn --version"
          # Build
          # Debug build for now, as otherwise gcc produces unusable meta server
          # executable even with -O1.
          $mko = "CMAKE_OPTIONS='-D CMAKE_BUILD_TYPE=Debug"
          $mko += " -D QFS_EXTRA_C_OPTIONS=-flax-vector-conversions'"
          $mko += ' -j2 --no-print-directory'
          & "$cygwinPath\bin\bash" -lc `
            "cd '$cur' ; df -h . ; make $mko tarball"

      - name: Upload tarball
        if: |
          'refs/heads/master' == github.ref
          || startsWith(github.ref, 'refs/tags/')
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          AWS_DEFAULT_REGION: us-west-2
        shell: pwsh
        run: |
          aws s3 cp (Get-ChildItem "build\qfs-*.tgz").FullName `
            s3://quantcast-qfs