Test packages #11
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # | |
| # This workflow is run either by a manual workflow dispatch, or | |
| # as part of a pull request check. | |
| # | |
| # It builds a version of GAP (which one depends on its inputs), then builds | |
| # all packages (resp. all matching some glob), then runs the tests of all | |
| # packages (resp. all matching some glob) with a testfile. The various | |
| # test results are aggregated into a test-status JSON file and is uploaded | |
| # as an artifact. Finally, the test results are compared against the latest | |
| # "official" test run. A human-readable comparison report is generated as a | |
| # MARKDOWN file and a summary of the changes as a test-status-diff JSON file | |
| # which other workflows can process. Both files are uploaded as artifacts. | |
| # | |
| name: "Test packages" | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| which-gap: | |
| description: 'Either a GAP branch name or a GAP version' | |
| required: true | |
| type: string | |
| default: master # or 4.11.1 or ... | |
| which-gap-repo: | |
| description: 'URL of any fork of the GAP repository' | |
| required: false | |
| type: string | |
| default: https://github.com/gap-system/gap.git | |
| pkg-build-glob: | |
| description: 'Only build packages matching the given glob' | |
| required: false | |
| type: string | |
| default: "*" | |
| pkg-test-glob: | |
| description: 'Only test packages matching the given glob' | |
| required: false | |
| type: string | |
| default: "*" | |
| # see https://github.com/marketplace/actions/debugging-with-tmate | |
| #debug_enabled: | |
| # description: 'Run the build with tmate debugging enabled' | |
| # type: boolean | |
| # default: false | |
| workflow_call: | |
| inputs: | |
| which-gap: | |
| description: 'Either a GAP branch name or a GAP version' | |
| required: true | |
| type: string | |
| default: master # or 4.11.1 or ... | |
| which-gap-repo: | |
| description: 'URL of any fork of the GAP repository' | |
| required: false | |
| type: string | |
| default: https://github.com/gap-system/gap.git | |
| pkg-build-glob: | |
| description: 'Only build packages matching the given glob' | |
| required: false | |
| type: string | |
| default: "*" | |
| pkg-test-glob: | |
| description: 'Only test packages matching the given glob' | |
| required: false | |
| type: string | |
| default: "*" | |
| jobs: | |
| build: | |
| # If you change this name, you must adjust the corresponding name | |
| # in tools/generate_test_status.py | |
| name: "Build GAP and packages" | |
| runs-on: ubuntu-20.04 | |
| outputs: | |
| matrix: ${{ steps.get-names.outputs.matrix }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| # Cache the package archives we download across runs of this job | |
| # to speed up things considerably | |
| - name: "Cache archives" | |
| uses: actions/cache@v3 | |
| with: | |
| path: _archives | |
| key: archives-${{ hashFiles('packages/*/meta.json') }} | |
| restore-keys: | | |
| archives- | |
| - name: "Install package distribution tools" | |
| run: python -m pip install -r tools/requirements.txt | |
| # TOOD: dependencies should come from a container | |
| - name: "Install binary package dependencies" | |
| run: | | |
| sudo apt-get update | |
| deps=$(tools/gather_dependencies.py packages/*/meta.json) | |
| if [[ -n "$deps" ]]; then | |
| echo "Installing required binary depedencies: $deps" | |
| sudo apt-get install --no-install-recommends $deps | |
| else | |
| echo "No required binary depedencies to be installed" | |
| fi | |
| - name: "Download packages" | |
| run: tools/download_packages.py packages/*/meta.json | |
| - name: "Cleanup archives" | |
| run: tools/cleanup_archives.py | |
| # the following step is useful for debugging | |
| - name: "Determine system architecture for -march=native" | |
| run: | | |
| echo "cc -march=native: $(tools/compiler_arch.sh cc)" | |
| echo "c++ -march=native: $(tools/compiler_arch.sh c++)" | |
| # Setup ccache, to speed up repeated compilation of the same binaries | |
| # (i.e., GAP and the packages) | |
| - name: "Setup ccache" | |
| uses: Chocobo1/setup-ccache-action@v1 | |
| with: | |
| update_packager_index: false | |
| - name: "Install GAP" | |
| run: | | |
| echo "::group::fetch" | |
| whichgap="${{ github.event.inputs.which-gap || inputs.which-gap }}" | |
| whichgaprepo="${{ github.event.inputs.which-gap-repo || inputs.which-gap-repo }}" | |
| if [[ "${whichgap}" == 4.* ]]; then | |
| # assume it is a tag | |
| wget --quiet https://github.com/gap-system/gap/releases/download/v${whichgap}/gap-${whichgap}-core.tar.gz | |
| tar xf gap-${whichgap}-core.tar.gz | |
| rm gap-${whichgap}-core.tar.gz | |
| mv gap-${whichgap} $HOME/gap | |
| else | |
| git clone --depth=2 -b ${whichgap} ${whichgaprepo} $HOME/gap | |
| fi | |
| cd $HOME/gap | |
| echo "::endgroup::" | |
| echo "::group::autogen" | |
| ./autogen.sh | |
| echo "::endgroup::" | |
| echo "::group::configure" | |
| ./configure | |
| echo "::endgroup::" | |
| echo "::group::make" | |
| make -j4 | |
| echo "::endgroup::" | |
| # put GAP into PATH | |
| ln -s $HOME/gap/gap /usr/local/bin/gap | |
| - name: "Extract packages" | |
| run: | | |
| mkdir -p $HOME/gap/pkg | |
| cd $HOME/gap/pkg | |
| for f in $GITHUB_WORKSPACE/_archives/* ; do | |
| echo "::group::$(basename $f)" | |
| ls -l $f | |
| case "$f" in | |
| *.tar.*) | |
| echo "Extracting $f" | |
| tar -xvf "$f" | |
| ;; | |
| *.zip) | |
| echo "Extracting $f" | |
| unzip "$f" | |
| ;; | |
| *) | |
| echo "Skipping $f" | |
| ;; | |
| esac | |
| echo "::endgroup::" | |
| done | |
| - name: "Build packages" | |
| run: | | |
| cd $HOME/gap/pkg | |
| # skip xgap: no X11 headers, and no means to test it | |
| rm -rf xgap* | |
| MAKEFLAGS=-j3 ../bin/BuildPackages.sh --strict ${{ github.event.inputs.pkg-build-glob || inputs.pkg-build-glob }} | |
| - name: "Test LoadAllPackages + GAP test suite" | |
| run: | | |
| gap -A --quitonbreak -r -c " | |
| SetInfoLevel(InfoPackageLoading, PACKAGE_DEBUG); | |
| LoadAllPackages(); | |
| SetInfoLevel(InfoPackageLoading, PACKAGE_ERROR); | |
| ReadGapRoot(\"tst/testinstall.g\"); | |
| QUIT; | |
| " | |
| - name: "Create tarball" | |
| run: | | |
| cd $HOME | |
| tar --exclude-vcs --exclude=build --exclude=.libs -cf gap.tar.zst gap | |
| - name: "Upload GAP with packages as artifact" | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: gap-${{ github.event.inputs.which-gap || inputs.which-gap }} | |
| path: /home/runner/gap.tar.zst | |
| - name: "Create jobs for matching packages" | |
| id: get-names | |
| run: | | |
| MATRIX="{\"package\":[" | |
| SKIPPED="" | |
| for PKG in packages/${{ github.event.inputs.pkg-test-glob || inputs.pkg-test-glob }}/meta.json; do | |
| PKG=${PKG%"/meta.json"} | |
| PKG=${PKG#"packages/"} | |
| if ! jq -e -r '.TestFile' < packages/${PKG}/meta.json > /dev/null ; then | |
| echo "Skip ${PKG}: no TestFile" | |
| SKIPPED="${SKIPPED}${PKG} " | |
| elif [[ ${PKG} == xgap ]]; then | |
| echo "Skip xgap: no X11 headers, and no means to test it" | |
| SKIPPED="${SKIPPED}${PKG} " | |
| elif [[ ${PKG} == polycyclic ]]; then | |
| # HACK FIXME TODO: skip polycyclic for now | |
| echo "Skip polycyclic tests for now, as they run in an infinite (?) loop" | |
| echo "Re-enable them once there is a new polycyclic release" | |
| SKIPPED="${SKIPPED}${PKG} " | |
| else | |
| MATRIX="${MATRIX}\"${PKG}\"," | |
| fi | |
| done | |
| MATRIX="${MATRIX}]}" | |
| echo "matrix=$MATRIX" >> $GITHUB_OUTPUT | |
| test-package: | |
| # If you change this name, you must adjust the corresponding name | |
| # in tools/generate_test_status.py | |
| name: "${{ matrix.package }}" | |
| if: ${{ needs.build.outputs.matrix != '{"package":[]}' }} | |
| needs: build | |
| runs-on: ubuntu-20.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.build.outputs.matrix) }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: "Install package distribution tools" | |
| run: python -m pip install -r tools/requirements.txt | |
| # the following step is useful for debugging | |
| - name: "Determine system architecture for -march=native" | |
| run: | | |
| echo "cc -march=native: $(tools/compiler_arch.sh cc)" | |
| echo "c++ -march=native: $(tools/compiler_arch.sh c++)" | |
| - name: "Download GAP from previous job" | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: gap-${{ github.event.inputs.which-gap || inputs.which-gap }} | |
| - name: "Extract GAP artifact" | |
| run: | | |
| cd $HOME | |
| tar xvf $GITHUB_WORKSPACE/gap.tar.zst | |
| ln -s $HOME/gap/gap /usr/local/bin/gap | |
| cd $GITHUB_WORKSPACE | |
| - name: "Install binary package dependencies" | |
| run: | | |
| sudo apt-get update | |
| deps=$(tools/gather_dependencies.py ${{ matrix.package }}) | |
| if [[ -n "$deps" ]]; then | |
| echo "Installing required binary depedencies: $deps" | |
| sudo apt-get install --no-install-recommends $deps | |
| else | |
| echo "No required binary depedencies to be installed" | |
| fi | |
| #- name: "Setup tmate session" | |
| # uses: mxschmitt/action-tmate@v3 | |
| # if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled }} | |
| - name: "Run tests" | |
| timeout-minutes: 10 | |
| id: tests-default | |
| run: | | |
| PKG=${{ matrix.package }} | |
| gap --quitonbreak -r -c " | |
| pkgname := \"$PKG\"; | |
| SetInfoLevel(InfoPackageLoading, PACKAGE_DEBUG); | |
| LoadPackage(pkgname); | |
| SetInfoLevel(InfoPackageLoading, PACKAGE_ERROR); | |
| res:=TestPackage(pkgname); | |
| FORCE_QUIT_GAP(res); | |
| " | |
| - name: "Run tests with OnlyNeeded" | |
| timeout-minutes: 10 | |
| if: always() | |
| id: tests-only-needed | |
| run: | | |
| PKG=${{ matrix.package }} | |
| gap -A --quitonbreak -r -c " | |
| pkgname := \"$PKG\"; | |
| # WORKAROUNDS for various packages which need additional packages | |
| if pkgname = \"agt\" then | |
| LoadPackage(\"grape\" : OnlyNeeded); | |
| elif pkgname = \"ctbllib\" then | |
| LoadPackage(\"browse\" : OnlyNeeded); | |
| LoadPackage(\"tomlib\" : OnlyNeeded); | |
| LoadPackage(\"spinsym\" : OnlyNeeded); | |
| elif pkgname = \"gapdoc\" then | |
| LoadPackage(\"browse\" : OnlyNeeded); | |
| elif pkgname = \"guarana\" then | |
| LoadPackage(\"nq\" : OnlyNeeded); | |
| elif pkgname = \"hap\" then | |
| # hap really needs all (?) its optional dependencies in its tests | |
| # so we cheat and load it early | |
| SetInfoLevel(InfoPackageLoading, PACKAGE_DEBUG); | |
| LoadPackage(\"hap\"); | |
| elif pkgname = \"packagemanager\" then | |
| QUIT_GAP(true); # HACK: workaround for weird failures | |
| elif pkgname = \"sglppow\" then | |
| LoadPackage(\"liepring\" : OnlyNeeded); | |
| elif pkgname = \"standardff\" then | |
| LoadPackage(\"factint\" : OnlyNeeded); | |
| elif pkgname = \"ugaly\" then | |
| LoadPackage(\"fga\" : OnlyNeeded); | |
| fi; | |
| SetInfoLevel(InfoPackageLoading, PACKAGE_DEBUG); | |
| LoadPackage(pkgname : OnlyNeeded); | |
| SetInfoLevel(InfoPackageLoading, PACKAGE_ERROR); | |
| res:=TestPackage(pkgname); | |
| FORCE_QUIT_GAP(res); | |
| " | |
| report: | |
| name: "Generate report" | |
| needs: test-package | |
| if: always() | |
| runs-on: ubuntu-20.04 | |
| outputs: | |
| test-status: ${{ steps.test-status.outputs.test-status }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: "Install package distribution tools" | |
| run: python -m pip install -r tools/requirements.txt | |
| - name: "Generate test-status.json" | |
| id: test-status | |
| run: | | |
| # Is this a workflow_call? | |
| if [ "${{inputs.which-gap}}" != "" ]; then | |
| JOB_NAME_PREFIX="${{inputs.which-gap}} / " | |
| else | |
| JOB_NAME_PREFIX="" | |
| fi | |
| ROOT='data/reports' | |
| # relative path (with respect to ROOT) to generated test-status.json, i.e. the "id" entry of the json-file. | |
| DIR_REL=$(tools/generate_test_status.py ${{secrets.GITHUB_TOKEN}} ${{ github.repository }} "$GITHUB_RUN_ID" "$GITHUB_SHA" ${{ github.event.inputs.which-gap || inputs.which-gap }} "$JOB_NAME_PREFIX") | |
| DIR="${ROOT}/${DIR_REL}" | |
| echo "dir=${DIR}" >> $GITHUB_OUTPUT | |
| echo "dir-rel=${DIR_REL}" >> $GITHUB_OUTPUT | |
| - name: "Download latest report" | |
| id: download-latest-report | |
| run: | | |
| ROOT="data/reports" | |
| DIR_SYM_REL=latest-${{ github.event.inputs.which-gap || inputs.which-gap }} | |
| DIR_SYM="${ROOT}/${DIR_SYM_REL}" | |
| URL_SYM="https://raw.githubusercontent.com/${{ github.repository }}/${DIR_SYM}" | |
| # Check if file at url exists, | |
| # so we do not run into errors for the first run of the script | |
| # (when the first report is created and thus no latest report is available) | |
| if wget --spider "${URL_SYM}"; then | |
| # wget downloads the "symbolic link" as a plain file | |
| # containing as content the path that the symlink points to, | |
| # so we need to convert this into a real symbolic link. | |
| DIR_REL=$(wget -O - ${URL_SYM}) | |
| DIR="${ROOT}/${DIR_REL}" | |
| ln -s ${DIR_REL} ${DIR_SYM} | |
| URL="https://raw.githubusercontent.com/${{ github.repository }}/${DIR}/test-status.json" | |
| wget -P ${DIR} ${URL} | |
| fi | |
| - name: "Generate report" | |
| id: report | |
| run: tools/generate_report.py ${{ steps.test-status.outputs.dir-rel }} latest-${{ github.event.inputs.which-gap || inputs.which-gap }} | |
| - name: "Upload report as artifact" | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: "report-${{ github.event.inputs.which-gap || inputs.which-gap }}" | |
| path: "${{ steps.test-status.outputs.dir }}" |