Show all the items in tuples and lists. Fixes #1056

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_constants.py

@@ -266,6 +266,25 @@ def as_int_in_env(env_key, default):
 # If not specified, uses default heuristic to determine if it should be loaded.
 PYDEVD_USE_FRAME_EVAL = os.getenv('PYDEVD_USE_FRAME_EVAL', '').lower()
 
+# Values used to determine how much container items will be shown.
+# PYDEVD_CONTAINER_INITIAL_EXPANDED_ITEMS:
+#     - Defines how many items will appear initially expanded after which a 'more...' will appear.
+#
+# PYDEVD_CONTAINER_BUCKET_SIZE
+#    - Defines the size of each bucket inside the 'more...' item
+#        i.e.: a bucket with size == 2 would show items such as:
+#            - [2:4]
+#            - [4:6]
+#            ...
+#
+# PYDEVD_CONTAINER_RANDOM_ACCESS_MAX_ITEMS
+#    - Defines the maximum number of items for dicts and sets.
+#
+PYDEVD_CONTAINER_INITIAL_EXPANDED_ITEMS = as_int_in_env('PYDEVD_CONTAINER_INITIAL_EXPANDED_ITEMS', 100)
+PYDEVD_CONTAINER_BUCKET_SIZE = as_int_in_env('PYDEVD_CONTAINER_BUCKET_SIZE', 1000)
+PYDEVD_CONTAINER_RANDOM_ACCESS_MAX_ITEMS = as_int_in_env('PYDEVD_CONTAINER_RANDOM_ACCESS_MAX_ITEMS', 500)
+PYDEVD_CONTAINER_NUMPY_MAX_ITEMS = as_int_in_env('PYDEVD_CONTAINER_NUMPY_MAX_ITEMS', 500)
+
 PYDEVD_IPYTHON_COMPATIBLE_DEBUGGING = is_true_in_env('PYDEVD_IPYTHON_COMPATIBLE_DEBUGGING')
 
 # If specified in PYDEVD_IPYTHON_CONTEXT it must be a string with the basename

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_resolver.py

@@ -8,12 +8,9 @@
 from _pydevd_bundle.pydevd_constants import IS_PY36_OR_GREATER, \
     MethodWrapperType, RETURN_VALUES_DICT, DebugInfoHolder, IS_PYPY, GENERATED_LEN_ATTR_NAME
 from _pydevd_bundle.pydevd_safe_repr import SafeRepr
+from _pydevd_bundle import pydevd_constants
 
-# Note: 300 is already a lot to see in the outline (after that the user should really use the shell to get things)
-# and this also means we'll pass less information to the client side (which makes debugging faster).
-MAX_ITEMS_TO_HANDLE = 300
-
-TOO_LARGE_MSG = 'Too large to show contents. Max items to show: ' + str(MAX_ITEMS_TO_HANDLE)
+TOO_LARGE_MSG = 'Maximum number of items (%s) reached. To show more items customize the value of the PYDEVD_CONTAINER_RANDOM_ACCESS_MAX_ITEMS environment variable.'
 TOO_LARGE_ATTR = 'Unable to handle:'
 
 
@@ -311,8 +308,8 @@ def get_contents_debug_adapter_protocol(self, dct, fmt=None):
             else:
                 eval_key_str = None
             ret.append((key_as_str, val, eval_key_str))
-            if i > MAX_ITEMS_TO_HANDLE:
-                ret.append((TOO_LARGE_ATTR, TOO_LARGE_MSG, None))
+            if i >= pydevd_constants.PYDEVD_CONTAINER_RANDOM_ACCESS_MAX_ITEMS:
+                ret.append((TOO_LARGE_ATTR, TOO_LARGE_MSG % (pydevd_constants.PYDEVD_CONTAINER_RANDOM_ACCESS_MAX_ITEMS,), None))
                 break
 
         # in case the class extends built-in type and has some additional fields
@@ -336,8 +333,8 @@ def get_dictionary(self, dct):
             # we need to add the id because otherwise we cannot find the real object to get its contents later on.
             key = '%s (%s)' % (self.key_to_str(key), id(key))
             ret[key] = val
-            if i > MAX_ITEMS_TO_HANDLE:
-                ret[TOO_LARGE_ATTR] = TOO_LARGE_MSG
+            if i >= pydevd_constants.PYDEVD_CONTAINER_RANDOM_ACCESS_MAX_ITEMS:
+                ret[TOO_LARGE_ATTR] = TOO_LARGE_MSG % (pydevd_constants.PYDEVD_CONTAINER_RANDOM_ACCESS_MAX_ITEMS,)
                 break
 
         # in case if the class extends built-in type and has some additional fields
@@ -351,21 +348,127 @@ def _apply_evaluate_name(parent_name, evaluate_name):
     return evaluate_name % (parent_name,)
 
 
-#=======================================================================================================================
-# TupleResolver
-#=======================================================================================================================
+class MoreItemsRange:
+
+    def __init__(self, value, from_i, to_i):
+        self.value = value
+        self.from_i = from_i
+        self.to_i = to_i
+
+    def get_contents_debug_adapter_protocol(self, _self, fmt=None):
+        l = len(self.value)
+        ret = []
+
+        format_str = '%0' + str(int(len(str(l - 1)))) + 'd'
+        if fmt is not None and fmt.get('hex', False):
+            format_str = '0x%0' + str(int(len(hex(l).lstrip('0x')))) + 'x'
+
+        for i, item in enumerate(self.value[self.from_i:self.to_i]):
+            i += self.from_i
+            ret.append((format_str % i, item, '[%s]' % i))
+        return ret
+
+    def get_dictionary(self, _self, fmt=None):
+        dct = {}
+        for key, obj, _ in self.get_contents_debug_adapter_protocol(self, fmt):
+            dct[key] = obj
+        return dct
+
+    def resolve(self, attribute):
+        '''
+        :param var: that's the original object we're dealing with.
+        :param attribute: that's the key to resolve
+            -- either the dict key in get_dictionary or the name in the dap protocol.
+        '''
+        return self.value[int(attribute)]
+
+    def __eq__(self, o):
+        return isinstance(o, MoreItemsRange) and self.value is o.value and \
+                self.from_i == o.from_i and self.to_i == o.to_i
+
+    def __str__(self):
+        return '[%s:%s]' % (self.from_i, self.to_i)
+
+    __repr__ = __str__
+
+
+class MoreItems:
+
+    def __init__(self, value, handled_items):
+        self.value = value
+        self.handled_items = handled_items
+
+    def get_contents_debug_adapter_protocol(self, _self, fmt=None):
+        total_items = len(self.value)
+        remaining = total_items - self.handled_items
+        bucket_size = pydevd_constants.PYDEVD_CONTAINER_BUCKET_SIZE
+
+        from_i = self.handled_items
+        to_i = from_i + min(bucket_size, remaining)
+
+        ret = []
+        while remaining > 0:
+            remaining -= bucket_size
+            more_items_range = MoreItemsRange(self.value, from_i, to_i)
+            ret.append((str(more_items_range), more_items_range, None))
+
+            from_i = to_i
+            to_i = from_i + min(bucket_size, remaining)
+
+        return ret
+
+    def get_dictionary(self, _self, fmt=None):
+        dct = {}
+        for key, obj, _ in self.get_contents_debug_adapter_protocol(self, fmt):
+            dct[key] = obj
+        return dct
+
+    def resolve(self, attribute):
+        from_i, to_i = attribute[1:-1].split(':')
+        from_i = int(from_i)
+        to_i = int(to_i)
+        return MoreItemsRange(self.value, from_i, to_i)
+
+    def __eq__(self, o):
+        return isinstance(o, MoreItems) and self.value is o.value
+
+    def __str__(self):
+        return '...'
+
+    __repr__ = __str__
+
+
+class ForwardInternalResolverToObject:
+    '''
+    To be used when we provide some internal object that'll actually do the resolution.
+    '''
+
+    def get_contents_debug_adapter_protocol(self, obj, fmt=None):
+        return obj.get_contents_debug_adapter_protocol(fmt)
+
+    def get_dictionary(self, var, fmt={}):
+        return var.get_dictionary(var, fmt)
+
+    def resolve(self, var, attribute):
+        return var.resolve(attribute)
+
+
 class TupleResolver:  # to enumerate tuples and lists
 
     def resolve(self, var, attribute):
         '''
-            @param var: that's the original attribute
-            @param attribute: that's the key passed in the dict (as a string)
+        :param var: that's the original object we're dealing with.
+        :param attribute: that's the key to resolve
+            -- either the dict key in get_dictionary or the name in the dap protocol.
         '''
         if attribute in (GENERATED_LEN_ATTR_NAME, TOO_LARGE_ATTR):
             return None
         try:
             return var[int(attribute)]
         except:
+            if attribute == 'more':
+                return MoreItems(var, pydevd_constants.PYDEVD_CONTAINER_INITIAL_EXPANDED_ITEMS)
+
             return getattr(var, attribute)
 
     def get_contents_debug_adapter_protocol(self, lst, fmt=None):
@@ -378,18 +481,26 @@ def get_contents_debug_adapter_protocol(self, lst, fmt=None):
 
         :return list(tuple(name:str, value:object, evaluateName:str))
         '''
-        l = len(lst)
+        lst_len = len(lst)
         ret = []
 
-        format_str = '%0' + str(int(len(str(l - 1)))) + 'd'
+        format_str = '%0' + str(int(len(str(lst_len - 1)))) + 'd'
         if fmt is not None and fmt.get('hex', False):
-            format_str = '0x%0' + str(int(len(hex(l).lstrip('0x')))) + 'x'
+            format_str = '0x%0' + str(int(len(hex(lst_len).lstrip('0x')))) + 'x'
 
+        initial_expanded = pydevd_constants.PYDEVD_CONTAINER_INITIAL_EXPANDED_ITEMS
         for i, item in enumerate(lst):
             ret.append((format_str % i, item, '[%s]' % i))
 
-            if i > MAX_ITEMS_TO_HANDLE:
-                ret.append((TOO_LARGE_ATTR, TOO_LARGE_MSG, None))
+            if i >= initial_expanded - 1:
+                if (lst_len - initial_expanded) < pydevd_constants.PYDEVD_CONTAINER_BUCKET_SIZE:
+                    # Special case: if we have just 1 more bucket just put it inline.
+                    item = MoreItemsRange(lst, initial_expanded, lst_len)
+
+                else:
+                    # Multiple buckets
+                    item = MoreItems(lst, initial_expanded)
+                ret.append(('more', item, None))
                 break
 
         # Needed in case the class extends the built-in type and has some additional fields.
@@ -408,11 +519,13 @@ def get_dictionary(self, var, fmt={}):
         if fmt is not None and fmt.get('hex', False):
             format_str = '0x%0' + str(int(len(hex(l).lstrip('0x')))) + 'x'
 
+        initial_expanded = pydevd_constants.PYDEVD_CONTAINER_INITIAL_EXPANDED_ITEMS
         for i, item in enumerate(var):
             d[format_str % i] = item
 
-            if i > MAX_ITEMS_TO_HANDLE:
-                d[TOO_LARGE_ATTR] = TOO_LARGE_MSG
+            if i >= initial_expanded - 1:
+                item = MoreItems(var, initial_expanded)
+                d['more'] = item
                 break
 
         # in case if the class extends built-in type and has some additional fields
@@ -436,8 +549,8 @@ def get_contents_debug_adapter_protocol(self, obj, fmt=None):
         for i, item in enumerate(obj):
             ret.append((str(id(item)), item, None))
 
-            if i > MAX_ITEMS_TO_HANDLE:
-                ret.append((TOO_LARGE_ATTR, TOO_LARGE_MSG, None))
+            if i >= pydevd_constants.PYDEVD_CONTAINER_RANDOM_ACCESS_MAX_ITEMS:
+                ret.append((TOO_LARGE_ATTR, TOO_LARGE_MSG % (pydevd_constants.PYDEVD_CONTAINER_RANDOM_ACCESS_MAX_ITEMS,), None))
                 break
 
         # Needed in case the class extends the built-in type and has some additional fields.
@@ -467,8 +580,8 @@ def get_dictionary(self, var):
         for i, item in enumerate(var):
             d[str(id(item))] = item
 
-            if i > MAX_ITEMS_TO_HANDLE:
-                d[TOO_LARGE_ATTR] = TOO_LARGE_MSG
+            if i >= pydevd_constants.PYDEVD_CONTAINER_RANDOM_ACCESS_MAX_ITEMS:
+                d[TOO_LARGE_ATTR] = TOO_LARGE_MSG % (pydevd_constants.PYDEVD_CONTAINER_RANDOM_ACCESS_MAX_ITEMS,)
                 break
 
         # in case if the class extends built-in type and has some additional fields
@@ -670,6 +783,7 @@ def get_frame_name(self, frame):
 orderedDictResolver = OrderedDictResolver()
 frameResolver = FrameResolver()
 dapGrouperResolver = DAPGrouperResolver()
+forwardInternalResolverToObject = ForwardInternalResolverToObject()
 
 
 class InspectStub:

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_vars.py

@@ -47,15 +47,15 @@ def dump_frames(thread_id):
 
 
 @silence_warnings_decorator
-def getVariable(dbg, thread_id, frame_id, scope, attrs):
+def getVariable(dbg, thread_id, frame_id, scope, locator):
     """
     returns the value of a variable
 
     :scope: can be BY_ID, EXPRESSION, GLOBAL, LOCAL, FRAME
 
     BY_ID means we'll traverse the list of all objects alive to get the object.
 
-    :attrs: after reaching the proper scope, we have to get the attributes until we find
+    :locator: after reaching the proper scope, we have to get the attributes until we find
             the proper location (i.e.: obj\tattr1\tattr2)
 
     :note: when BY_ID is used, the frame_id is considered the id of the object to find and
@@ -74,49 +74,49 @@ def getVariable(dbg, thread_id, frame_id, scope, attrs):
             frame_id = int(frame_id)
             for var in objects:
                 if id(var) == frame_id:
-                    if attrs is not None:
-                        attrList = attrs.split('\t')
-                        for k in attrList:
+                    if locator is not None:
+                        locator_parts = locator.split('\t')
+                        for k in locator_parts:
                             _type, _type_name, resolver = get_type(var)
                             var = resolver.resolve(var, k)
 
                     return var
 
-        # If it didn't return previously, we coudn't find it by id (i.e.: alrceady garbage collected).
+        # If it didn't return previously, we coudn't find it by id (i.e.: already garbage collected).
         sys.stderr.write('Unable to find object with id: %s\n' % (frame_id,))
         return None
 
     frame = dbg.find_frame(thread_id, frame_id)
     if frame is None:
         return {}
 
-    if attrs is not None:
-        attrList = attrs.split('\t')
+    if locator is not None:
+        locator_parts = locator.split('\t')
     else:
-        attrList = []
+        locator_parts = []
 
-    for attr in attrList:
+    for attr in locator_parts:
         attr.replace("@_@TAB_CHAR@_@", '\t')
 
     if scope == 'EXPRESSION':
-        for count in range(len(attrList)):
+        for count in range(len(locator_parts)):
             if count == 0:
                 # An Expression can be in any scope (globals/locals), therefore it needs to evaluated as an expression
-                var = evaluate_expression(dbg, frame, attrList[count], False)
+                var = evaluate_expression(dbg, frame, locator_parts[count], False)
             else:
                 _type, _type_name, resolver = get_type(var)
-                var = resolver.resolve(var, attrList[count])
+                var = resolver.resolve(var, locator_parts[count])
     else:
         if scope == "GLOBAL":
             var = frame.f_globals
-            del attrList[0]  # globals are special, and they get a single dummy unused attribute
+            del locator_parts[0]  # globals are special, and they get a single dummy unused attribute
         else:
             # in a frame access both locals and globals as Python does
             var = {}
             var.update(frame.f_globals)
             var.update(frame.f_locals)
 
-        for k in attrList:
+        for k in locator_parts:
             _type, _type_name, resolver = get_type(var)
             var = resolver.resolve(var, k)
 

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_xml.py

@@ -7,7 +7,7 @@
 from _pydev_bundle.pydev_imports import quote
 from _pydevd_bundle.pydevd_extension_api import TypeResolveProvider, StrPresentationProvider
 from _pydevd_bundle.pydevd_utils import isinstance_checked, hasattr_checked, DAPGrouper
-from _pydevd_bundle.pydevd_resolver import get_var_scope
+from _pydevd_bundle.pydevd_resolver import get_var_scope, MoreItems, MoreItemsRange
 
 try:
     import types
@@ -60,6 +60,8 @@ def _create_default_type_map():
         pass  # not available on all python versions
 
     default_type_map.append((DAPGrouper, pydevd_resolver.dapGrouperResolver))
+    default_type_map.append((MoreItems, pydevd_resolver.forwardInternalResolverToObject))
+    default_type_map.append((MoreItemsRange, pydevd_resolver.forwardInternalResolverToObject))
 
     try:
         default_type_map.append((set, pydevd_resolver.setResolver))