Ensure all lists of lists are marked as rectangular tables when appro…

…priate (#5450)

src/plist.c

@@ -286,21 +286,23 @@ static Int KTNumPlist(Obj list, Obj * famfirst)
             ktnumFirst = 0;
         }
         isHom   = 1;
-        areMut  = IS_MUTABLE_OBJ(elm);
-        if ( ktnumFirst >= T_PLIST_HOM ||
-             ( ktnumFirst == 0 && IS_HOMOG_LIST( elm) )) {
+        areMut = IS_MUTABLE_OBJ(elm);
 
-          // entry is a homogeneous list, so this might be a table
-          isTable = 1;
-
-          // also check for rectangularity, unless this would be expensive
-          if (IS_PLIST(elm))
-            {
-              isRect = 1;
-              len = LEN_PLIST(elm);
+        // if entry is a homogeneous list this might be a table or list
+        if (ktnumFirst >= T_PLIST_HOM) {
+            isTable = 1;
+            isRect = 1;
+            len = LEN_PLIST(elm);
+        }
+        else if (ktnumFirst == 0 && IS_HOMOG_LIST(elm)) {
+            isTable = 1;
+            // only handle small lists as rectangular
+            if (IS_SMALL_LIST(elm)) {
+                isRect = 1;
+                len = LEN_LIST(elm);
             }
-
         }
+
         if (!testing) CLEAR_OBJ_FLAG(list, OBJ_FLAG_TESTING);
     }
 
@@ -342,16 +344,21 @@ static Int KTNumPlist(Obj list, Obj * famfirst)
                     isRect = 0;
                 }
                 if ( isTable ) {
-                    // IS_PLIST first, as it is much cheaper
-                    if (!(IS_PLIST(elm) || IS_LIST(elm))) {
-                        isTable = 0;
-                        isRect = 0;
+                    // check IS_PLIST first, as it is much cheaper
+                    if (IS_PLIST(elm)) {
+                        if (isRect && LEN_PLIST(elm) != len) {
+                            isRect = 0;
+                        }
                     }
-                    if ( isRect ) {
-                        if ( !(IS_PLIST(elm) && LEN_PLIST(elm) == len) ) {
+                    else if (IS_SMALL_LIST(elm)) {
+                        if (isRect && LEN_LIST(elm) != len) {
                             isRect = 0;
                         }
                     }
+                    else {
+                        isTable = 0;
+                        isRect = 0;
+                    }
                 }
             }
             areMut  = (areMut || IS_MUTABLE_OBJ(elm));

tst/testinstall/table.tst

@@ -0,0 +1,31 @@
+#@local u,v, check
+gap> START_TEST("table.tst");
+gap> check := {m} -> [IsTable(m), IsRectangularTable(m), HasIsRectangularTable(m) ];
+function( m ) ... end
+gap> check( [ ]);
+[ false, false, false ]
+gap> check( [ [] ]);
+[ false, false, false ]
+gap> check( [ [], [] ]);
+[ false, false, false ]
+gap> check([ [1,2], [3,4] ]);
+[ true, true, true ]
+gap> check( [ [1,2], [3]]);
+[ true, false, false ]
+gap> check( [ "ab", "cd" ]);
+[ true, true, true ]
+gap> check( [ ['a', 'b'], "cd"]);
+[ true, true, true ]
+gap> check( [ "ab", ['a', 'b']]);
+[ true, true, true ]
+gap> check( [ [1,2], "cd"]);
+[ false, false, false ]
+gap> check( [ [1,2], 3]);
+[ false, false, false ]
+gap> check(InfiniteListOfNames("a"));
+[ false, false, true ]
+gap> check([InfiniteListOfNames("a")]);
+[ true, true, true ]
+gap> check([ ["a","b"], InfiniteListOfNames("a")]);
+[ false, false, false ]
+gap> STOP_TEST( "table.tst", 1);