Decrease recursion depth limit to 3 + smarter check for recursion

src/compiler/checker.ts

@@ -20355,27 +20355,25 @@ namespace ts {
         }
 
         // Return true if the given type is deeply nested. We consider this to be the case when structural type comparisons
-        // for 5 or more occurrences or instantiations of the type have been recorded on the given stack. It is possible,
+        // for maxDepth or more occurrences or instantiations of the type have been recorded on the given stack. It is possible,
         // though highly unlikely, for this test to be true in a situation where a chain of instantiations is not infinitely
-        // expanding. Effectively, we will generate a false positive when two types are structurally equal to at least 5
+        // expanding. Effectively, we will generate a false positive when two types are structurally equal to at least maxDepth
         // levels, but unequal at some level beyond that.
-        // In addition, this will also detect when an indexed access has been chained off of 5 or more times (which is essentially
-        // the dual of the structural comparison), and likewise mark the type as deeply nested, potentially adding false positives
-        // for finite but deeply expanding indexed accesses (eg, for `Q[P1][P2][P3][P4][P5]`).
-        // It also detects when a recursive type reference has expanded 5 or more times, eg, if the true branch of
-        // `type A<T> = null extends T ? [A<NonNullable<T>>] : [T]`
-        // has expanded into `[A<NonNullable<NonNullable<NonNullable<NonNullable<NonNullable<T>>>>>>]`
-        // in such cases we need to terminate the expansion, and we do so here.
-        function isDeeplyNestedType(type: Type, stack: Type[], depth: number, maxDepth = 5): boolean {
+        function isDeeplyNestedType(type: Type, stack: Type[], depth: number, maxDepth = 3): boolean {
             if (depth >= maxDepth) {
                 const identity = getRecursionIdentity(type);
                 let count = 0;
+                let lastTypeId = 0;
                 for (let i = 0; i < depth; i++) {
-                    if (getRecursionIdentity(stack[i]) === identity) {
+                    const t = stack[i];
+                    // We only count occurrences with higher type ids than the previous occurrences, since higher
+                    // type ids are an indicator of newer instantiations caused by recursion.
+                    if (getRecursionIdentity(t) === identity && t.id >= lastTypeId) {
                         count++;
                         if (count >= maxDepth) {
                             return true;
                         }
+                        lastTypeId = t.id;
                     }
                 }
             }
@@ -20410,13 +20408,6 @@ namespace ts {
             if (type.flags & TypeFlags.TypeParameter) {
                 return type.symbol;
             }
-            if (type.flags & TypeFlags.IndexedAccess) {
-                // Identity is the leftmost object type in a chain of indexed accesses, eg, in A[P][Q] it is A
-                do {
-                    type = (type as IndexedAccessType).objectType;
-                } while (type.flags & TypeFlags.IndexedAccess);
-                return type;
-            }
             if (type.flags & TypeFlags.Conditional) {
                 // The root object represents the origin of the conditional type
                 return (type as ConditionalType).root;