From f51a365c6404e3654a9d6a1e358623f8675a81c1 Mon Sep 17 00:00:00 2001 From: Markus Pfeiffer Date: Mon, 16 Jan 2017 16:30:24 +0000 Subject: [PATCH] Make manual examples a testsuite * Extracts the examples from the reference manual chapter by chapter into separate `.tst` files * Runs each of the created `.tst` files in a separate GAP process * Creates coverage reports, which are uploaded to codecov.io. --- .gitignore | 3 ++ .travis.yml | 7 +++ etc/ci.sh | 106 ++++++++++++++++++++++++++++++------------- tst/extractmanuals.g | 60 ++++++++++++++++++++++++ tst/testmanuals.g | 60 ++++++++++++++++++++++++ 5 files changed, 205 insertions(+), 31 deletions(-) create mode 100644 tst/extractmanuals.g create mode 100644 tst/testmanuals.g diff --git a/.gitignore b/.gitignore index 8e5a802bfe..f33cf08aba 100644 --- a/.gitignore +++ b/.gitignore @@ -63,3 +63,6 @@ /tags /src/TAGS + +# Tests +/tst/testmanuals/*.tst diff --git a/.travis.yml b/.travis.yml index d52e80d293..87a68e507d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,6 +22,13 @@ matrix: apt_packages: - libgmp-dev + - os: linux + env: TEST_SUITE=testmanuals + compiler: gcc + addons: + apt_packages: + - libgmp-dev + # OS X builds: since those are slow and limited on Travis, # we only run testinstall for now - os: osx diff --git a/etc/ci.sh b/etc/ci.sh index f548445a56..cb745f48fc 100644 --- a/etc/ci.sh +++ b/etc/ci.sh @@ -8,26 +8,17 @@ set -ex -if [[ $TEST_SUITE = 'makemanuals' && $TRAVIS_OS_NAME = 'linux' ]] -then - make manuals - cat doc/*/make_manuals.out - if [ `cat doc/*/make_manuals.out | grep -c "manual.lab written"` != '3' ] - then - echo "Build failed" - exit 1 - fi -else - if [ ! -f tst/${TEST_SUITE}.g ] - then - echo "Could not read test suite tst/${TEST_SUITE}.g" - exit 1 - fi - - if [[ x"$ABI" == "x32" ]] - then - sh bin/gap.sh tst/${TEST_SUITE}.g - else +case $TEST_SUITE in + makemanuals) + make manuals + cat doc/*/make_manuals.out + if [ `cat doc/*/make_manuals.out | grep -c "manual.lab written"` != '3' ] + then + echo "Build failed" + exit 1 + fi + ;; + testmanuals) cd pkg/io* ./configure make @@ -37,18 +28,71 @@ else make cd ../.. - sh bin/gap.sh --cover coverage tst/${TEST_SUITE}.g + sh bin/gap.sh -q tst/extractmanuals.g + COVDIR=`mktemp -d` - # generate coverage report sh bin/gap.sh -q < true then - Print("ERROR: could not load profiling package"); - FORCE_QUIT_GAP(1); - fi; - OutputJsonCoverage("coverage", "coverage.json"); + Read("tst/testmanuals.g"); + SaveWorkspace("testmanuals.wsp"); + QUIT_GAP(0); +GAPInput + + for ch in tst/testmanuals/*.tst + do + COVNAME="coverage.`basename $ch .tst`" + sh bin/gap.sh -q -L testmanuals.wsp --cover $COVDIR/$COVNAME < true then + Print("ERROR: could not load profiling package"); + FORCE_QUIT_GAP(1); + fi; + + for f in DirectoryContents("$COVDIR") do + if not (f in [".", ".."]) then + OutputJsonCoverage( Filename(Directory("$COVDIR"), f) + , Concatenation(f, ".json")); + fi; + od; + QUIT_GAP(0); +GAPInput + ;; + *) + if [ ! -f tst/${TEST_SUITE}.g ] + then + echo "Could not read test suite tst/${TEST_SUITE}.g" + exit 1 + fi + + if [[ x"$ABI" == "x32" ]] + then + sh bin/gap.sh tst/${TEST_SUITE}.g + else + cd pkg/io* + ./configure + make + cd ../.. + cd pkg/profiling* + ./configure + make + cd ../.. + + sh bin/gap.sh --cover coverage tst/${TEST_SUITE}.g + + # generate coverage report + sh bin/gap.sh -q < true then + Print("ERROR: could not load profiling package"); + FORCE_QUIT_GAP(1); + fi; + OutputJsonCoverage("coverage", "coverage.json"); + QUIT_GAP(0); +GAPInput + fi +esac; +cd bin/x86* ; gcov -o . ../../src/* +cd ../.. diff --git a/tst/extractmanuals.g b/tst/extractmanuals.g new file mode 100644 index 0000000000..8b09b5e7d2 --- /dev/null +++ b/tst/extractmanuals.g @@ -0,0 +1,60 @@ +############################################################################# +## +#W testmanuals.g +## +## <#GAPDoc Label="[1]{testmanuals.g}"> +## <#/GAPDoc> +## + +# This code extracts the examples from manuals chapter-wise and +# stores them in a file that can be passed to the Test function + +pathtodoc := DirectoriesLibrary("doc/ref"); +Read(Filename(pathtodoc, "makedocreldata.g")); +GAPInfo.ManualDataRef.pathtodoc := DirectoriesLibrary("doc/ref"); +GAPInfo.ManualDataRef.pathtoroot := DirectoriesLibrary(""); + +exsref := ExtractExamples( + GAPInfo.ManualDataRef.pathtodoc, + GAPInfo.ManualDataRef.main, + GAPInfo.ManualDataRef.files, + "Chapter" ); + +WriteExamplesTst := function(directory) + local ch, chname, chapterfiles, i, a, output; + chapterfiles := []; + directory := Directory(directory); + for i in [1..Length(exsref)] do + ch := exsref[i]; + if Length(ch) > 0 then + chname := STRINGIFY("chapter", i, ".tst"); + Add(chapterfiles, chname); + + # Note that the following truncates the testfile. + output := OutputTextFile( Filename(directory, chname), false ); + SetPrintFormattingStatus( output, false ); + + AppendTo(output, "#### Reference manual, Chapter ",i," ####\n", + "gap> START_TEST(\"", chname, "\");\n"); + for a in ch do + AppendTo(output, "\n#LOC# ", a[2], a[1]); + if a[1][Length(a[1])] <> '\n' then + AppendTo(output, "\n"); + fi; + od; + AppendTo(output, "\n\n\ngap> STOP_TEST(\"", chname, "\", 0);"); + fi; + od; + return chapterfiles; +end; + +testdir := Filename(DirectoriesLibrary("tst")[1], "testmanuals"); +CreateDir(testdir); +Print("Extracting manual examples to ", testdir, "...\n"); +WriteExamplesTst( testdir ); +QUIT_GAP(0); + +############################################################################# +## +#E + diff --git a/tst/testmanuals.g b/tst/testmanuals.g new file mode 100644 index 0000000000..93cc9ad2ae --- /dev/null +++ b/tst/testmanuals.g @@ -0,0 +1,60 @@ +############################################################################# +## +#W testmanuals.g +## +## <#GAPDoc Label="[1]{testmanuals.g}"> +## <#/GAPDoc> +## + +# This code extracts the examples from manuals chapter-wise and +# stores them in a file that can be passed to the Test function + +ExamplesReportDiff := function(inp, expout, found, fnam, line, time) + local tstf, i, loc, res; + + Print("########> Diff in "); + if IsStream(fnam) then + Print("test stream, line ",line,"\n"); + else + tstf := SplitString(StringFile(fnam), "\n"); + i := line; + # Look for location marker + while (i > 0) and + ((Length(tstf[i]) < 5) or (tstf[i]{[1..5]} <> "#LOC#")) do + i := i - 1; + od; + # Found a location marker + if i > 0 then + loc := InputTextString(Concatenation(tstf[i]{[6..Length(tstf[i])]}, ";")); + res := READ_COMMAND_REAL(loc, false); + if res[1] = true then + Print(res[2][1],":",res[2][2]); + fi; + Print(" (", fnam,":",line,")\n"); + else # did not find a location marker + Print(fnam,":",line,"\n"); + fi; + fi; + Print("# Input is:\n", inp); + Print("# Expected output:\n", expout); + Print("# But found:\n", found); + Print("########\n"); +end; + +TestManualChapter := function(filename) + local testResult; + + GAP_EXIT_CODE(1); + testResult := Test(filename, rec( width := 72, + compareFunction := "uptowhitespace", + reportDiff := ExamplesReportDiff ) ); + if not(testResult) then + QUIT_GAP(1); + fi; + QUIT_GAP(0); +end; + +############################################################################# +## +#E +