Skip to content

Commit

Permalink
Make manual examples a testsuite
Browse files Browse the repository at this point in the history
* Extracts the examples from the reference manual chapter by chapter
  into separate `.tst` files
* Runs each of the created `.tst` files in a separate GAP process
* Creates coverage reports, which are uploaded to codecov.io.
  • Loading branch information
markuspf authored and fingolfin committed Feb 6, 2017
1 parent 20169c7 commit f51a365
Show file tree
Hide file tree
Showing 5 changed files with 205 additions and 31 deletions.
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -63,3 +63,6 @@

/tags
/src/TAGS

# Tests
/tst/testmanuals/*.tst
7 changes: 7 additions & 0 deletions .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,13 @@ matrix:
apt_packages:
- libgmp-dev

- os: linux
env: TEST_SUITE=testmanuals
compiler: gcc
addons:
apt_packages:
- libgmp-dev

# OS X builds: since those are slow and limited on Travis,
# we only run testinstall for now
- os: osx
Expand Down
106 changes: 75 additions & 31 deletions etc/ci.sh
Original file line number Diff line number Diff line change
Expand Up @@ -8,26 +8,17 @@

set -ex

if [[ $TEST_SUITE = 'makemanuals' && $TRAVIS_OS_NAME = 'linux' ]]
then
make manuals
cat doc/*/make_manuals.out
if [ `cat doc/*/make_manuals.out | grep -c "manual.lab written"` != '3' ]
then
echo "Build failed"
exit 1
fi
else
if [ ! -f tst/${TEST_SUITE}.g ]
then
echo "Could not read test suite tst/${TEST_SUITE}.g"
exit 1
fi

if [[ x"$ABI" == "x32" ]]
then
sh bin/gap.sh tst/${TEST_SUITE}.g
else
case $TEST_SUITE in
makemanuals)
make manuals
cat doc/*/make_manuals.out
if [ `cat doc/*/make_manuals.out | grep -c "manual.lab written"` != '3' ]
then
echo "Build failed"
exit 1
fi
;;
testmanuals)
cd pkg/io*
./configure
make
Expand All @@ -37,18 +28,71 @@ else
make
cd ../..

sh bin/gap.sh --cover coverage tst/${TEST_SUITE}.g
sh bin/gap.sh -q tst/extractmanuals.g
COVDIR=`mktemp -d`

# generate coverage report
sh bin/gap.sh -q <<GAPInput
if LoadPackage("profiling") <> true then
Print("ERROR: could not load profiling package");
FORCE_QUIT_GAP(1);
fi;
OutputJsonCoverage("coverage", "coverage.json");
Read("tst/testmanuals.g");
SaveWorkspace("testmanuals.wsp");
QUIT_GAP(0);
GAPInput

for ch in tst/testmanuals/*.tst
do
COVNAME="coverage.`basename $ch .tst`"
sh bin/gap.sh -q -L testmanuals.wsp --cover $COVDIR/$COVNAME <<GAPInput
TestManualChapter("$ch");
QUIT_GAP(0);
GAPInput
fi
cd bin/x86* ; gcov -o . ../../src/*
cd ../..
fi;
done

sh bin/gap.sh -q <<GAPInput
if LoadPackage("profiling") <> true then
Print("ERROR: could not load profiling package");
FORCE_QUIT_GAP(1);
fi;
for f in DirectoryContents("$COVDIR") do
if not (f in [".", ".."]) then
OutputJsonCoverage( Filename(Directory("$COVDIR"), f)
, Concatenation(f, ".json"));
fi;
od;
QUIT_GAP(0);
GAPInput
;;
*)
if [ ! -f tst/${TEST_SUITE}.g ]
then
echo "Could not read test suite tst/${TEST_SUITE}.g"
exit 1
fi

if [[ x"$ABI" == "x32" ]]
then
sh bin/gap.sh tst/${TEST_SUITE}.g
else
cd pkg/io*
./configure
make
cd ../..
cd pkg/profiling*
./configure
make
cd ../..

sh bin/gap.sh --cover coverage tst/${TEST_SUITE}.g

# generate coverage report
sh bin/gap.sh -q <<GAPInput
if LoadPackage("profiling") <> true then
Print("ERROR: could not load profiling package");
FORCE_QUIT_GAP(1);
fi;
OutputJsonCoverage("coverage", "coverage.json");
QUIT_GAP(0);
GAPInput
fi
esac;
cd bin/x86* ; gcov -o . ../../src/*
cd ../..
60 changes: 60 additions & 0 deletions tst/extractmanuals.g
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
#############################################################################
##
#W testmanuals.g
##
## <#GAPDoc Label="[1]{testmanuals.g}">
## <#/GAPDoc>
##

# This code extracts the examples from manuals chapter-wise and
# stores them in a file that can be passed to the Test function

pathtodoc := DirectoriesLibrary("doc/ref");
Read(Filename(pathtodoc, "makedocreldata.g"));
GAPInfo.ManualDataRef.pathtodoc := DirectoriesLibrary("doc/ref");
GAPInfo.ManualDataRef.pathtoroot := DirectoriesLibrary("");

exsref := ExtractExamples(
GAPInfo.ManualDataRef.pathtodoc,
GAPInfo.ManualDataRef.main,
GAPInfo.ManualDataRef.files,
"Chapter" );

WriteExamplesTst := function(directory)
local ch, chname, chapterfiles, i, a, output;
chapterfiles := [];
directory := Directory(directory);
for i in [1..Length(exsref)] do
ch := exsref[i];
if Length(ch) > 0 then
chname := STRINGIFY("chapter", i, ".tst");
Add(chapterfiles, chname);

# Note that the following truncates the testfile.
output := OutputTextFile( Filename(directory, chname), false );
SetPrintFormattingStatus( output, false );

AppendTo(output, "#### Reference manual, Chapter ",i," ####\n",
"gap> START_TEST(\"", chname, "\");\n");
for a in ch do
AppendTo(output, "\n#LOC# ", a[2], a[1]);
if a[1][Length(a[1])] <> '\n' then
AppendTo(output, "\n");
fi;
od;
AppendTo(output, "\n\n\ngap> STOP_TEST(\"", chname, "\", 0);");
fi;
od;
return chapterfiles;
end;

testdir := Filename(DirectoriesLibrary("tst")[1], "testmanuals");
CreateDir(testdir);
Print("Extracting manual examples to ", testdir, "...\n");
WriteExamplesTst( testdir );
QUIT_GAP(0);

#############################################################################
##
#E

60 changes: 60 additions & 0 deletions tst/testmanuals.g
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
#############################################################################
##
#W testmanuals.g
##
## <#GAPDoc Label="[1]{testmanuals.g}">
## <#/GAPDoc>
##

# This code extracts the examples from manuals chapter-wise and
# stores them in a file that can be passed to the Test function

ExamplesReportDiff := function(inp, expout, found, fnam, line, time)
local tstf, i, loc, res;

Print("########> Diff in ");
if IsStream(fnam) then
Print("test stream, line ",line,"\n");
else
tstf := SplitString(StringFile(fnam), "\n");
i := line;
# Look for location marker
while (i > 0) and
((Length(tstf[i]) < 5) or (tstf[i]{[1..5]} <> "#LOC#")) do
i := i - 1;
od;
# Found a location marker
if i > 0 then
loc := InputTextString(Concatenation(tstf[i]{[6..Length(tstf[i])]}, ";"));
res := READ_COMMAND_REAL(loc, false);
if res[1] = true then
Print(res[2][1],":",res[2][2]);
fi;
Print(" (", fnam,":",line,")\n");
else # did not find a location marker
Print(fnam,":",line,"\n");
fi;
fi;
Print("# Input is:\n", inp);
Print("# Expected output:\n", expout);
Print("# But found:\n", found);
Print("########\n");
end;

TestManualChapter := function(filename)
local testResult;

GAP_EXIT_CODE(1);
testResult := Test(filename, rec( width := 72,
compareFunction := "uptowhitespace",
reportDiff := ExamplesReportDiff ) );
if not(testResult) then
QUIT_GAP(1);
fi;
QUIT_GAP(0);
end;

#############################################################################
##
#E

0 comments on commit f51a365

Please sign in to comment.