From 0d4423118785b2744d0391db8eeb51910330693e Mon Sep 17 00:00:00 2001 From: Steve Linton Date: Tue, 24 Oct 2017 12:12:14 +0100 Subject: [PATCH 1/5] Fix documentation of GasmanStatistics --- lib/gasman.gd | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/gasman.gd b/lib/gasman.gd index d64405aa01..ecaa22bc6f 100644 --- a/lib/gasman.gd +++ b/lib/gasman.gd @@ -27,7 +27,7 @@ ##

## The full component will be present if a full garbage collection ## has taken place since ⪆ started. It contains information about -## the most recent full garbage collection. It is a record, with six +## the most recent full garbage collection. It is a record, with eight ## components: livebags contains the number of bags which survived ## the garbage collection; livekb contains the total number of ## kilobytes occupied by those bags; deadbags contains the total @@ -36,7 +36,10 @@ ## previous full garbage collection; deadkb contains the total ## number of kilobytes occupied by those bags; freekb reports the ## total number of kilobytes available in the ⪆ workspace for new -## objects and totalkb the actual size of the workspace. +## objects; totalkb reports the actual size of the workspace; +## time reports the CPU time in milliseconds spent on the last garbage +## collection and cumulative the total CPU time in milliseconds spent +## on that type of garbage collection since ⪆ started. ##

## These figures should be viewed with some caution. They are ## stored internally in fixed length integer formats, and deadkb From 9784e606a5fbe3c0cbbaee6b96cdc3bea024c819 Mon Sep 17 00:00:00 2001 From: Steve Linton Date: Tue, 24 Oct 2017 12:13:16 +0100 Subject: [PATCH 2/5] Fix counting of SizeAllBags in ResizeBag also use memmove in one place --- src/gasman.c | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/src/gasman.c b/src/gasman.c index cd6df5f3dc..00cdd166ed 100644 --- a/src/gasman.c +++ b/src/gasman.c @@ -1251,9 +1251,9 @@ UInt ResizeBag ( #ifdef COUNT_BAGS /* update the statistics */ InfoBags[type].sizeLive += new_size - old_size; - InfoBags[type].sizeAll += new_size - old_size; + // InfoBags[type].sizeAll += new_size - old_size; #endif - SizeAllBags += new_size - old_size; + // SizeAllBags += new_size - old_size; const Int diff = WORDS_BAG(new_size) - WORDS_BAG(old_size); @@ -1303,6 +1303,13 @@ UInt ResizeBag ( YoungBags += diff; AllocBags += diff; + /* In this case we increase the total amount allocated by the + difference */ +#ifdef COUNT_BAGS + InfoBags[type].sizeAll += new_size - old_size; +#endif + SizeAllBags += new_size - old_size; + ADD_CANARY(); header->size = new_size; @@ -1335,6 +1342,12 @@ UInt ResizeBag ( newHeader->flags = flags; newHeader->size = new_size; +#ifdef COUNT_BAGS + InfoBags[type].sizeAll += new_size; +#endif + SizeAllBags += new_size; + + CANARY_DISABLE_VALGRIND(); /* if the bag is already on the changed bags list, keep it there */ if ( header->link != bag ) { @@ -1354,15 +1367,12 @@ UInt ResizeBag ( CANARY_ENABLE_VALGRIND(); /* set the masterpointer */ - Bag * src = DATA(header); - Bag * end = src + WORDS_BAG(old_size); Bag * dst = DATA(newHeader); SET_PTR_BAG(bag, dst); /* copy the contents of the bag */ - while ( src < end ) - *dst++ = *src++; - + memmove((void *)dst, (void *)DATA(header), + sizeof(Obj) * WORDS_BAG(old_size)); } /* return success */ From c3b66cf4db7cbc6ef52975a255771a42019b6fa3 Mon Sep 17 00:00:00 2001 From: Steve Linton Date: Tue, 24 Oct 2017 12:15:18 +0100 Subject: [PATCH 3/5] Add TotalMemoryAllocated() and memory_allocated. These are analogs of Runtime() and time, but for allocated memory. --- doc/ref/debug.xml | 28 ++++++++++++++++++++++++++-- doc/ref/mloop.xml | 8 ++++++-- src/gap.c | 36 +++++++++++++++++++++++++++++++++--- 3 files changed, 65 insertions(+), 7 deletions(-) diff --git a/doc/ref/debug.xml b/doc/ref/debug.xml index a13a747892..5d61e419e4 100644 --- a/doc/ref/debug.xml +++ b/doc/ref/debug.xml @@ -423,11 +423,14 @@ should be treated as an estimate. - + + In the read-eval-print loop, - stores the number of milliseconds the last command took. + stores the number of milliseconds the last command +took. and stored the number of bytes of +memory it allocated. @@ -444,6 +447,27 @@ in nanoseconds. + +

+ Tracking Memory Usage + + + + returns the total amount of memory + in bytes allocated by the &GAP; memory manager since &GAP; started. + + + + + + +In the read-eval-print loop, + stores the number of bytes of +memoryallocated by the last completed statement. + + + +
diff --git a/doc/ref/mloop.xml b/doc/ref/mloop.xml index 4eab302b96..c694421b27 100644 --- a/doc/ref/mloop.xml +++ b/doc/ref/mloop.xml @@ -27,6 +27,8 @@ interactive environment in which you use &GAP;. last last2 last3 +time +memory_allocated previous result The normal interaction with &GAP; happens in the so-called read-eval-print loop. @@ -154,8 +156,10 @@ gap> last3 + last2 * last; ]]>

Also in each statement the time spent by the last statement, whether it -produced a value or not, is available in the variable . -This is an integer that holds the number of milliseconds. +produced a value or not, is available in the variable . This is an integer that holds the number of +milliseconds. Similarly the amount of memory allocated during that +statement (in bytes) is stored in the variable .

diff --git a/src/gap.c b/src/gap.c index 8d8e29bce4..842c2b6749 100644 --- a/src/gap.c +++ b/src/gap.c @@ -148,6 +148,16 @@ UInt Last3; */ UInt Time; +/**************************************************************************** +** +*V MemoryAllocated. . . . . . . . . . . global variable 'memory_allocated' +** +** 'MemoryAllocated' is the global variable 'memory_allocated', +** which is automatically assigned the amount of memory allocated while +** executing the last command. +*/ +UInt MemoryAllocated; + /**************************************************************************** ** @@ -226,6 +236,7 @@ Obj Shell ( Obj context, Char *outFile) { UInt time = 0; + UInt mem = 0; UInt status; Obj evalResult; UInt dualSemicolon; @@ -261,8 +272,10 @@ Obj Shell ( Obj context, while ( 1 ) { /* start the stopwatch */ - if (setTime) - time = SyTime(); + if (setTime) { + time = SyTime(); + mem = SizeAllBags; + } /* read and evaluate one command */ STATE(Prompt) = prompt; @@ -339,8 +352,14 @@ Obj Shell ( Obj context, } /* stop the stopwatch */ - if (setTime) + if (setTime) { AssGVar( Time, INTOBJ_INT( SyTime() - time ) ); + /* This should be correct for small allocated totals at least, + even on 32 bits, but this functionality will never be very useful + on 32 bits */ + AssGVar(MemoryAllocated, + INTOBJ_INT((SizeAllBags - mem) % (1L << NR_SMALL_INT_BITS))); + } if (STATE(UserHasQuit)) { @@ -1944,6 +1963,14 @@ Obj FuncSHALLOW_SIZE ( return ObjInt_UInt( SIZE_BAG( obj ) ); } +/**************************************************************************** +** +*F FuncTotalMemoryAllocated( ) .expert function 'TotalMemoryAllocated' +*/ + +Obj FuncTotalMemoryAllocated( Obj self ) { + return INTOBJ_INT(SizeAllBags); +} /**************************************************************************** ** @@ -2871,6 +2898,7 @@ static StructGVarFunc GVarFuncs [] = { GVAR_FUNC(GASMAN_MESSAGE_STATUS, 0, ""), GVAR_FUNC(GASMAN_LIMITS, 0, ""), GVAR_FUNC(SHALLOW_SIZE, 1, "object"), + GVAR_FUNC(TotalMemoryAllocated, 0, ""), GVAR_FUNC(TNUM_OBJ, 1, "object"), GVAR_FUNC(TNUM_OBJ_INT, 1, "object"), GVAR_FUNC(OBJ_HANDLE, 1, "object"), @@ -2969,7 +2997,9 @@ static Int PostRestore ( Last2 = GVarName( "last2" ); Last3 = GVarName( "last3" ); Time = GVarName( "time" ); + MemoryAllocated = GVarName( "memory_allocated" ); AssGVar(Time, INTOBJ_INT(0)); + AssGVar(MemoryAllocated, INTOBJ_INT(0)); QUITTINGGVar = GVarName( "QUITTING" ); /* return success */ From dc5b403838b39c6805978e9763c73c0d425855e6 Mon Sep 17 00:00:00 2001 From: Steve Linton Date: Tue, 24 Oct 2017 15:44:12 +0100 Subject: [PATCH 4/5] Add StringOfMemoryAmount a function for printing numbers of bytes in a human-friendly format. --- doc/ref/string.xml | 1 + lib/string.gd | 27 +++++++++++++++++++++++++++ lib/string.gi | 31 +++++++++++++++++++++++++++++++ lib/test.gi | 7 +++++-- 4 files changed, 64 insertions(+), 2 deletions(-) diff --git a/doc/ref/string.xml b/doc/ref/string.xml index 4ce4cb42b9..021055ccfd 100644 --- a/doc/ref/string.xml +++ b/doc/ref/string.xml @@ -521,6 +521,7 @@ vice versa. They are useful for examples of text encryption. <#Include Label="NumbersString"> <#Include Label="StringNumbers"> +<#Include Label="StringOfMemoryAmount"> diff --git a/lib/string.gd b/lib/string.gd index 0d3816966d..dca87faaa2 100644 --- a/lib/string.gd +++ b/lib/string.gd @@ -834,6 +834,33 @@ DeclareGlobalFunction("LaTeXTable"); BindGlobal("BHINT", MakeImm("\>\<")); +############################################################################# +## +#F StringOfMemoryAmount( ) returns an appropriate human-readable string +## representation of bytes +## +## +## <#GAPDoc Label="StringOfMemoryAmount"> +## +## +## +## +## This function returns a human-readable string representing +## numbytes of memory. It is used in printing amounts of memory +## allocated by tests and benchmarks. Binary prefixes (representing powers of +## 1024) are used. +## +## +## StringOfMemoryAmount(123456789); +## "117MB" +## ]]> +## <#/GAPDoc> + + +DeclareGlobalFunction("StringOfMemoryAmount"); + + ############################################################################# ## #E diff --git a/lib/string.gi b/lib/string.gi index 5e9ef17a00..8d3c89df1b 100644 --- a/lib/string.gi +++ b/lib/string.gi @@ -1202,6 +1202,37 @@ function(a,b) Error("concatenating strings via + is not supported, use Concatenation(,) instead"); end); +############################################################################# +## +#F StringOfMemoryAmount( ) returns an appropriate human-readable string +## representation of bytes +## + +InstallGlobalFunction(StringOfMemoryAmount, function(m) + local whole, frac, shift, s, units; + whole := m; + frac := 0; + shift := 0; + while whole >= 1024 do + frac := whole mod 1024; + whole := Int(whole / 1024); + shift := shift+1; + od; + s := ShallowCopy(String(whole)); + if whole < 100 then + Append(s,"."); + Append(s,String(Int(frac/102.4))); + if whole < 10 then + Append(s, String(Int(frac/10.24) mod 10)); + fi; + fi; + units := ["B","KB","MB","GB","TB","PB","EB","YB","ZB"]; + Append(s, units[shift+1]); + return s; +end); + + + ############################################################################# ## diff --git a/lib/test.gi b/lib/test.gi index cf628a9c1f..8809699831 100644 --- a/lib/test.gi +++ b/lib/test.gi @@ -677,7 +677,9 @@ InstallGlobalFunction( "TestDirectory", function(arg) totalTime := totalTime + time; if opts.showProgress then - Print( String( time, 8 ), " msec for ", files[i].shortName, "\n" ); + Print( String( time, 8 ), " msec (",String(gctime),"ms GC) and ", + StringOfMemoryAmount( mem ), + " allocated for ", files[i].shortName, "\n" ); fi; od; @@ -685,7 +687,8 @@ InstallGlobalFunction( "TestDirectory", function(arg) Print("-----------------------------------\n"); Print( "total", - String( totalTime, 10 ), " msec\n\n" ); + String( totalTime, 10 ), " msec (",String( totalGcTime ),"ms GC) and ", + StringOfMemoryAmount( totalMem )," allocated\n\n" ); if not opts.suppressStatusMessage then if testTotal then From a896c2089ecd970c6ac63b04de05f9b8a3f3c040 Mon Sep 17 00:00:00 2001 From: Steve Linton Date: Tue, 24 Oct 2017 15:45:05 +0100 Subject: [PATCH 5/5] TestDirectory: print amounts of memory allocated and gc time --- lib/test.gi | 28 ++++++++++++++++++++++++---- src/gap.c | 2 +- src/gasman.c | 5 +---- 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/lib/test.gi b/lib/test.gi index 8809699831..076735a38e 100644 --- a/lib/test.gi +++ b/lib/test.gi @@ -567,14 +567,24 @@ end); ## InstallGlobalFunction( "TestDirectory", function(arg) - local basedirs, nopts, opts, files, newfiles, filetimes, - f, c, i, recurseFiles, - startTime, time, testResult, testTotal, - totalTime, STOP_TEST_CPY; + local testTotal, totalTime, totalMem, STOP_TEST_CPY, + basedirs, nopts, opts, testOptions, earlyStop, + showProgress, suppressStatusMessage, exitGAP, c, files, + filetimes, filemems, recurseFiles, f, i, startTime, + startMem, testResult, time, mem, startGcTime, gctime, + totalGcTime, filegctimes, GcTime; testTotal := true; totalTime := 0; + totalMem := 0; + totalGcTime := 0; + GcTime := function() + local g; + g := GASMAN_STATS(); + return g[1][8] + g[2][8]; + end; + STOP_TEST_CPY := STOP_TEST; STOP_TEST := function(arg) end; @@ -609,6 +619,8 @@ InstallGlobalFunction( "TestDirectory", function(arg) files := []; filetimes := []; + filemems := []; + filegctimes := []; recurseFiles := function(dirs, prefix) local dircontents, testfiles, t, testrecs, shortName, recursedirs, d, subdirs; @@ -658,6 +670,8 @@ InstallGlobalFunction( "TestDirectory", function(arg) fi; startTime := Runtime(); + startMem := TotalMemoryAllocated(); + startGcTime := GcTime(); testResult := Test(files[i].name, opts.testOptions); if not(testResult) and opts.earlyStop then STOP_TEST := STOP_TEST_CPY; @@ -673,8 +687,14 @@ InstallGlobalFunction( "TestDirectory", function(arg) testTotal := testTotal and testResult; time := Runtime() - startTime; + mem := TotalMemoryAllocated() - startMem; + gctime := GcTime() - startGcTime; filetimes[i] := time; + filemems[i] := mem; + filegctimes[i] := gctime; totalTime := totalTime + time; + totalMem := totalMem + mem; + totalGcTime := totalGcTime + gctime; if opts.showProgress then Print( String( time, 8 ), " msec (",String(gctime),"ms GC) and ", diff --git a/src/gap.c b/src/gap.c index 842c2b6749..e3ab4af497 100644 --- a/src/gap.c +++ b/src/gap.c @@ -150,7 +150,7 @@ UInt Time; /**************************************************************************** ** -*V MemoryAllocated. . . . . . . . . . . global variable 'memory_allocated' +*V MemoryAllocated . . . . . . . . . . . global variable 'memory_allocated' ** ** 'MemoryAllocated' is the global variable 'memory_allocated', ** which is automatically assigned the amount of memory allocated while diff --git a/src/gasman.c b/src/gasman.c index 00cdd166ed..832d2a7871 100644 --- a/src/gasman.c +++ b/src/gasman.c @@ -1251,9 +1251,7 @@ UInt ResizeBag ( #ifdef COUNT_BAGS /* update the statistics */ InfoBags[type].sizeLive += new_size - old_size; - // InfoBags[type].sizeAll += new_size - old_size; #endif - // SizeAllBags += new_size - old_size; const Int diff = WORDS_BAG(new_size) - WORDS_BAG(old_size); @@ -1303,8 +1301,7 @@ UInt ResizeBag ( YoungBags += diff; AllocBags += diff; - /* In this case we increase the total amount allocated by the - difference */ + // and increase the total amount allocated by the difference #ifdef COUNT_BAGS InfoBags[type].sizeAll += new_size - old_size; #endif