kernel: make memory usage more dynamic

This commit combines all the changes by @DanielPointon, with a few
cleanups by me on top. The aim of this commit it to make the memory
allocation more dynamic in the kernel extension of Digraphs for
computing homomorphisms and cliques. Namely, instead of allocating
memory for a bunch of fixed sized (512) arrays we only allocate the
memory required to perform any requested computations, and we
reuse that if possible for subsequent computations, and only allocate
more memory if there isn't enough allocated previously. This commit also
exposes two functions DIGRAPHS_FREE_* that can be used to release the
allocated memory for computing homomorphisms or cliques.

There might be a small performance penalty for these changes, for
example, with these changes:

gap> Test("~/digraphs/tst/extreme/cliques.tst");
Digraphs package: standard/cliques.tst
msecs: 2848
true
gap> Test("~/digraphs/tst/extreme/cliques.tst");
Digraphs package: standard/cliques.tst
msecs: 2923
true
gap> Test("~/digraphs/tst/extreme/grahom.tst");
Digraphs package: extreme/grahom.tst
msecs: 65177
true
gap> Test("~/digraphs/tst/extreme/grahom.tst");
Digraphs package: extreme/grahom.tst
msecs: 65956
true

and before:

gap> Test("~/digraphs/tst/extreme/cliques.tst");
Digraphs package: standard/cliques.tst
msecs: 2861
true
gap> Test("~/digraphs/tst/extreme/cliques.tst");
Digraphs package: standard/cliques.tst
msecs: 2928
true
gap> Test("~/digraphs/tst/extreme/grahom.tst");
Digraphs package: extreme/grahom.tst
msecs: 64881
true
gap> Test("~/digraphs/tst/extreme/grahom.tst");
Digraphs package: extreme/grahom.tst
msecs: 64997
true

doc/oper.xml

@@ -2432,4 +2432,39 @@ false
 ]]></Example>
   </Description>
 </ManSection>
+
+<#GAPDoc Label="DIGRAPHS_FREE_HOMOS_DATA">
+  <Oper Name="DIGRAPHS_FREE_HOMOS_DATA"/>
+  <Returns> This function does not return a value. </Returns>
+  <Description>
+  Releases the memory used by the internal data structures for homomorphism
+  finding. This will not have any noticeable impact on your &GAP; session, but
+  will reduce memory usage of &Digraphs;. After <C>DIGRAPHS_FREE_HOMOS_DATA</C>
+  is called, any subsequent calls to <Ref Func="HomomorphismDigraphsFinder"/>
+  will have to reallocate memory for the internal data structures used by the
+  homomorphism finding code in the kernel extension, and, depending on the
+  amount of memory required, there might be a one-off time penalty for doing
+  this.
+  </Description>
+  <Example><![CDATA[
+  gap> DIGRAPHS_FREE_HOMOS_DATA();
+  ]]></Example>
+<#/GAPDoc>
+
+<#GAPDoc Label="DIGRAPHS_FREE_CLIQUES_DATA">
+  <Oper Name="DIGRAPHS_FREE_CLIQUES_DATA"/>
+  <Returns> This function does not return a value. </Returns>
+  <Description>
+  Releases the memory used by the internal data structures for clique
+  finding. This will not have any noticeable impact on your &GAP; session, but
+  will reduce memory usage of &Digraphs;. After <C>DIGRAPHS_FREE_CLIQUES_DATA</C>
+  is called, any subsequent calls to <Ref Func="DigraphsCliquesFinder"/>
+  will have to reallocate memory for the internal data structures used by the
+  clique finding code in the kernel extension, and, depending on the
+  amount of memory required, there might be a one-off time penalty for doing
+  this.
+  </Description>
+  <Example><![CDATA[
+  gap> DIGRAPHS_FREE_CLIQUES_DATA();
+  ]]></Example>
 <#/GAPDoc>

gap/utils.gi

@@ -30,6 +30,10 @@ InstallGlobalFunction(DIGRAPHS_StopTest,
 function()
   SetInfoLevel(InfoWarning, DIGRAPHS_TestRec.InfoLevelInfoWarning);
   SetInfoLevel(InfoDigraphs, DIGRAPHS_TestRec.InfoLevelInfoDigraphs);
+
+  # Wipe internal structures for homos and cliques
+  DIGRAPHS_FREE_HOMOS_DATA();
+  DIGRAPHS_FREE_CLIQUES_DATA();
   return;
 end);
 

src/bitarray.c

@@ -1,42 +1,129 @@
 /********************************************************************************
 **
-*A  bitarray.c               bit arrays                           J. D. Mitchell
+**  bit_array.c               bit arrays                       J. D. Mitchell
 **
-**  Copyright (C) 2019 - J. D. Mitchell
+**  Copyright (C) 2019-2024
 **
 **  This file is free software, see the digraphs/LICENSE.
 **
 ********************************************************************************/
 
 #include "bitarray.h"
 
+// C headers
+#include <stdbool.h>  // for true and false
+#include <stddef.h>   // for size_t, NULL
+#include <stdlib.h>   // for free, calloc, malloc
+
 // GAP headers
 #include "gap-includes.h"  // for Obj, ELM_LIST, ISB_LIST, Fail
 
 // Digraphs headers
 #include "digraphs-debug.h"  // for DIGRAPHS_ASSERT
-#include "safemalloc.h"
+#include "safemalloc.h"      // for safe_malloc
+
+////////////////////////////////////////////////////////////////////////
+// Macros
+////////////////////////////////////////////////////////////////////////
+
+#define NR_BITS_IN_SIZE_T (sizeof(size_t) * 8)
+
+////////////////////////////////////////////////////////////////////////
+// Global variables
+////////////////////////////////////////////////////////////////////////
+
+bool     LOOKUPS_INITIALISED = false;
+size_t*  NR_BLOCKS_LOOKUP   = NULL;
+size_t*  QUOTIENT_LOOKUP     = NULL;
+size_t*  REMAINDER_LOOKUP    = NULL;
+Block*   MASK_LOOKUP         = NULL;
+uint16_t LOOKUP_SIZE         = 65535;
+
+////////////////////////////////////////////////////////////////////////
+// Static functions
+////////////////////////////////////////////////////////////////////////
+
+static size_t calculate_quotient(size_t N) {
+  return (size_t) N / NR_BITS_IN_SIZE_T;
+}
+
+static size_t calculate_number_of_blocks(size_t N) {
+  return (N + NR_BITS_IN_SIZE_T - 1) / NR_BITS_IN_SIZE_T;
+}
+
+static size_t calculate_remainder(size_t N) {
+  return (size_t) N % NR_BITS_IN_SIZE_T;
+}
+
+static Block calculate_mask(size_t N) {
+  return (Block) 1 << N;
+}
+
+static void allocate_num_blocks_lookup(uint16_t new_lookup_size) {
+  NR_BLOCKS_LOOKUP = (size_t*) safe_calloc(new_lookup_size, sizeof(size_t));
+
+  for (uint16_t i = 0; i < new_lookup_size; i++) {
+    NR_BLOCKS_LOOKUP[i] = calculate_number_of_blocks(i);
+  }
+}
+
+static void allocate_quotient_lookup(uint16_t new_lookup_size) {
+  QUOTIENT_LOOKUP = (size_t*) safe_calloc(new_lookup_size, sizeof(size_t));
+
+  for (uint16_t i = 0; i < new_lookup_size; i++) {
+    QUOTIENT_LOOKUP[i] = calculate_quotient(i);
+  }
+}
+
+static void allocate_remainder_lookup(uint16_t new_lookup_size) {
+  REMAINDER_LOOKUP = (size_t*) safe_calloc(new_lookup_size, sizeof(size_t));
+
+  for (uint16_t i = 0; i < new_lookup_size; i++) {
+    REMAINDER_LOOKUP[i] = calculate_remainder(i);
+  }
+}
+
+static void allocate_mask_lookup(uint16_t new_lookup_size) {
+  MASK_LOOKUP = (Block*) safe_calloc(new_lookup_size, sizeof(Block));
+
+  for (uint16_t i = 0; i < new_lookup_size; i++) {
+    MASK_LOOKUP[i] = calculate_mask(i);
+  }
+}
+
+static void initialize_bit_array_lookups(void) {
+  if (!LOOKUPS_INITIALISED) {
+    allocate_num_blocks_lookup(LOOKUP_SIZE);
+    allocate_quotient_lookup(LOOKUP_SIZE);
+    allocate_remainder_lookup(LOOKUP_SIZE);
+    allocate_mask_lookup(NR_BITS_PER_BLOCK);
+
+    LOOKUPS_INITIALISED = true;
+  }
+}
+
+////////////////////////////////////////////////////////////////////////
+// Non-static functions
+////////////////////////////////////////////////////////////////////////
 
 BitArray* new_bit_array(uint16_t const nr_bits) {
-  BitArray* bit_array  = safe_malloc(sizeof(BitArray));
-  bit_array->nr_bits   = nr_bits;
-  bit_array->nr_blocks = ((nr_bits % NUMBER_BITS_PER_BLOCK) == 0
-                              ? nr_bits / NUMBER_BITS_PER_BLOCK
-                              : nr_bits / NUMBER_BITS_PER_BLOCK + 1);
-  // The previous line is not tested since all the bit arrays we use are
-  // currently of length MAXVERTS = 512.
-  bit_array->blocks = safe_calloc(bit_array->nr_blocks, NUMBER_BITS_PER_BLOCK);
+  initialize_bit_array_lookups();
+  BitArray* bit_array = safe_malloc(sizeof(BitArray));
+
+  bit_array->nr_bits = nr_bits;
+  bit_array->nr_blocks =
+      ((nr_bits % NR_BITS_PER_BLOCK) == 0 ? nr_bits / NR_BITS_PER_BLOCK
+                                           : nr_bits / NR_BITS_PER_BLOCK + 1);
+  bit_array->blocks = safe_calloc(bit_array->nr_blocks, NR_BITS_PER_BLOCK);
+
   return bit_array;
 }
 
-// free_bit_array is not currently used, but kept in case it is required in the
-// future. JDM 2019
-
-// void free_bit_array(BitArray* const bit_array) {
-//   DIGRAPHS_ASSERT(bit_array != NULL);
-//   free(bit_array->blocks);
-//   free(bit_array);
-// }
+void free_bit_array(BitArray* const bit_array) {
+  DIGRAPHS_ASSERT(bit_array != NULL);
+  free(bit_array->blocks);
+  free(bit_array);
+}
 
 void set_bit_array_from_gap_int(BitArray* const bit_array, Obj o) {
   DIGRAPHS_ASSERT(bit_array != NULL);