Fix microsoft#874: Watch window freezes the system when Name contains…

… non-ASCII symbols

Fix microsoft#1064: test_module_events is failing on Windows

Use Unicode literals throughout wrapper.py, and fix pathname handling.

Add evaluation test for Unicode character in an expression.

Fix path pattern such that its __eq__ is used deterministically.

pytests/func/test_breakpoints.py

@@ -13,7 +13,7 @@
 from pytests.helpers.pathutils import get_test_root
 from pytests.helpers.session import DebugSession
 from pytests.helpers.timeline import Event
-from pytests.helpers.pattern import ANY
+from pytests.helpers.pattern import ANY, Path
 
 
 BP_TEST_ROOT = get_test_root('bp')
@@ -33,7 +33,7 @@ def test_path_with_ampersand(run_as, start_method):
         session.start_debugging()
         hit = session.wait_for_thread_stopped('breakpoint')
         frames = hit.stacktrace.body['stackFrames']
-        assert frames[0]['source']['path'] == ANY.path(testfile)
+        assert frames[0]['source']['path'] == Path(testfile)
 
         session.send_request('continue').wait_for_response(freeze=False)
         session.wait_for_exit()
@@ -54,7 +54,7 @@ def test_path_with_unicode(run_as, start_method):
         session.start_debugging()
         hit = session.wait_for_thread_stopped('breakpoint')
         frames = hit.stacktrace.body['stackFrames']
-        assert frames[0]['source']['path'] == ANY.path(testfile)
+        assert frames[0]['source']['path'] == Path(testfile)
         assert u'ಏನಾದರೂ_ಮಾಡು' == frames[0]['name']
 
         session.send_request('continue').wait_for_response(freeze=False)
@@ -159,13 +159,13 @@ def script2():
         hit = session.wait_for_thread_stopped()
         frames = hit.stacktrace.body['stackFrames']
         assert bp_script2_line == frames[0]['line']
-        assert frames[0]['source']['path'] == ANY.path(script2)
+        assert frames[0]['source']['path'] == Path(script2)
 
         session.send_request('continue').wait_for_response(freeze=False)
         hit = session.wait_for_thread_stopped()
         frames = hit.stacktrace.body['stackFrames']
         assert bp_script1_line == frames[0]['line']
-        assert frames[0]['source']['path'] == ANY.path(script1)
+        assert frames[0]['source']['path'] == Path(script1)
 
         session.send_request('continue').wait_for_response(freeze=False)
         session.wait_for_exit()

pytests/func/test_django.py

@@ -8,7 +8,7 @@
 import pytest
 import sys
 
-from pytests.helpers.pattern import ANY
+from pytests.helpers.pattern import ANY, Path
 from pytests.helpers.session import DebugSession
 from pytests.helpers.timeline import Event
 from pytests.helpers.pathutils import get_test_root
@@ -65,7 +65,7 @@ def test_django_breakpoint_no_multiproc(bp_target, start_method):
             'name': bp_name,
             'source': {
                 'sourceReference': ANY,
-                'path': ANY.path(bp_file),
+                'path': Path(bp_file),
             },
             'line': bp_line,
             'column': 1,
@@ -155,7 +155,7 @@ def test_django_exception_no_multiproc(ex_type, start_method):
             'details': {
                 'message': 'Hello',
                 'typeName': ANY.such_that(lambda s: s.endswith('ArithmeticError')),
-                'source': ANY.path(DJANGO1_MANAGE),
+                'source': Path(DJANGO1_MANAGE),
                 'stackTrace': ANY.such_that(lambda s: True),
             }
         }
@@ -170,7 +170,7 @@ def test_django_exception_no_multiproc(ex_type, start_method):
             'name': 'bad_route_' + ex_type,
             'source': {
                 'sourceReference': ANY,
-                'path': ANY.path(DJANGO1_MANAGE),
+                'path': Path(DJANGO1_MANAGE),
             },
             'line': ex_line,
             'column': 1,
@@ -240,7 +240,7 @@ def test_django_breakpoint_multiproc(start_method):
                 'name': 'home',
                 'source': {
                     'sourceReference': ANY.int,
-                    'path': ANY.path(DJANGO1_MANAGE),
+                    'path': Path(DJANGO1_MANAGE),
                 },
                 'line': bp_line,
                 'column': 1,

pytests/func/test_evaluate.py

@@ -4,6 +4,8 @@
 
 from __future__ import print_function, with_statement, absolute_import
 
+import sys
+
 from pytests.helpers.pattern import ANY
 from pytests.helpers.session import DebugSession
 from pytests.helpers.timeline import Event
@@ -314,3 +316,45 @@ def my_func():
 
         session.send_request('continue').wait_for_response()
         session.wait_for_exit()
+
+
+def test_unicode(pyfile, run_as, start_method):
+    # On Python 3, variable names can contain Unicode characters.
+    # On Python 2, they must be ASCII, but using a Unicode character in an expression should not crash debugger.
+
+    @pyfile
+    def code_to_debug():
+        from dbgimporter import import_and_enable_debugger
+        import_and_enable_debugger()
+        import ptvsd
+        # Since Unicode variable name is a SyntaxError at parse time in Python 2,
+        # this needs to do a roundabout way of setting it to avoid parse issues.
+        globals()[u'\u16A0'] = 123
+        ptvsd.break_into_debugger()
+        print('break')
+
+    with DebugSession() as session:
+        session.initialize(
+            target=(run_as, code_to_debug),
+            start_method=start_method,
+            ignore_unobserved=[Event('continued')],
+        )
+        session.start_debugging()
+        hit = session.wait_for_thread_stopped()
+
+        resp_eval = session.send_request('evaluate', arguments={
+            'expression': '\u16A0', 'frameId': hit.frame_id,
+        }).wait_for_response()
+
+        if sys.version_info >= (3,):
+            assert resp_eval.body == ANY.dict_with({
+                'type': 'int',
+                'result': '123'
+            })
+        else:
+            assert resp_eval.body == ANY.dict_with({
+                'type': 'SyntaxError'
+            })
+
+        session.send_request('continue').wait_for_response(freeze=False)
+        session.wait_for_exit()

pytests/func/test_exception.py

@@ -8,7 +8,7 @@
 
 from pytests.helpers.session import DebugSession
 from pytests.helpers.timeline import Event
-from pytests.helpers.pattern import ANY
+from pytests.helpers.pattern import ANY, Path
 
 
 @pytest.mark.parametrize('raised', ['raisedOn', 'raisedOff'])
@@ -47,7 +47,7 @@ def raise_with_except():
             'details': ANY.dict_with({
                 'typeName': ANY.such_that(lambda s: s.endswith('ArithmeticError')),
                 'message': 'bad code',
-                'source': ANY.path(code_to_debug),
+                'source': Path(code_to_debug),
             }),
         })
 
@@ -102,7 +102,7 @@ def raise_without_except():
             'details': ANY.dict_with({
                 'typeName': ANY.such_that(lambda s: s.endswith('ArithmeticError')),
                 'message': 'bad code',
-                'source': ANY.path(code_to_debug),
+                'source': Path(code_to_debug),
             }),
         })
 

pytests/func/test_flask.py

@@ -9,7 +9,7 @@
 import pytest
 import sys
 
-from pytests.helpers.pattern import ANY
+from pytests.helpers.pattern import ANY, Path
 from pytests.helpers.session import DebugSession
 from pytests.helpers.timeline import Event
 from pytests.helpers.webhelper import get_web_content, wait_for_connection
@@ -84,7 +84,7 @@ def test_flask_breakpoint_no_multiproc(bp_target, start_method):
             'name': bp_name,
             'source': {
                 'sourceReference': ANY.int,
-                'path': ANY.path(bp_file),
+                'path': Path(bp_file),
             },
             'line': bp_line,
             'column': 1,
@@ -165,7 +165,7 @@ def test_flask_exception_no_multiproc(ex_type, start_method):
             'details': {
                 'message': 'Hello',
                 'typeName': ANY.such_that(lambda s: s.endswith('ArithmeticError')),
-                'source': ANY.path(FLASK1_APP),
+                'source': Path(FLASK1_APP),
                 'stackTrace': ANY.such_that(lambda s: True)
             }
         }
@@ -180,7 +180,7 @@ def test_flask_exception_no_multiproc(ex_type, start_method):
             'name': 'bad_route_' + ex_type,
             'source': {
                 'sourceReference': ANY.int,
-                'path': ANY.path(FLASK1_APP),
+                'path': Path(FLASK1_APP),
             },
             'line': ex_line,
             'column': 1,
@@ -258,7 +258,7 @@ def test_flask_breakpoint_multiproc(start_method):
                 'name': 'home',
                 'source': {
                     'sourceReference': ANY.int,
-                    'path': ANY.path(FLASK1_APP),
+                    'path': Path(FLASK1_APP),
                 },
                 'line': bp_line,
                 'column': 1,

pytests/func/test_path_mapping.py

@@ -6,7 +6,7 @@
 
 import os
 from shutil import copyfile
-from pytests.helpers.pattern import ANY
+from pytests.helpers.pattern import Path
 from pytests.helpers.session import DebugSession
 from pytests.helpers.timeline import Event
 
@@ -46,10 +46,10 @@ def code_to_debug():
         session.start_debugging()
         hit = session.wait_for_thread_stopped('breakpoint')
         frames = hit.stacktrace.body['stackFrames']
-        assert frames[0]['source']['path'] == ANY.path(path_local)
+        assert frames[0]['source']['path'] == Path(path_local)
 
         remote_code_path = session.read_json()
-        assert path_remote == ANY.path(remote_code_path)
+        assert path_remote == Path(remote_code_path)
 
         session.send_request('continue').wait_for_response(freeze=False)
         session.wait_for_exit()

pytests/func/test_vs_specific.py

@@ -7,7 +7,7 @@
 import pytest
 from pytests.helpers.session import DebugSession
 from pytests.helpers.timeline import Event
-from pytests.helpers.pattern import ANY
+from pytests.helpers.pattern import Path
 
 
 @pytest.mark.parametrize('module', [True, False])
@@ -87,9 +87,9 @@ def test_code():
         modules = session.all_occurrences_of(Event('module'))
         modules = [(m.body['module']['name'], m.body['module']['path']) for m in modules]
         assert modules[:3] == [
-            ('module2', ANY.path(module2)),
-            ('module1', ANY.path(module1)),
-            ('__main__', ANY.path(test_code)),
+            ('module2', Path(module2)),
+            ('module1', Path(module1)),
+            ('__main__', Path(test_code)),
         ]
 
         session.send_request('continue').wait_for_response(freeze=False)

pytests/helpers/pathutils.py

@@ -3,6 +3,10 @@
 # for license information.
 
 import os.path
+import sys
+
+import ptvsd.compat
+
 
 def get_test_root(name):
     pytests_dir = os.path.dirname(os.path.dirname(__file__))
@@ -11,10 +15,18 @@ def get_test_root(name):
         return p
     return None
 
+
 def compare_path(left, right, show=True):
+    # If there's a unicode/bytes mismatch, make both unicode.
+    if isinstance(left, ptvsd.compat.unicode):
+        if not isinstance(right, ptvsd.compat.unicode):
+            right = right.decode(sys.getfilesystemencoding())
+    elif isinstance(right, ptvsd.compat.unicode):
+        left = right.decode(sys.getfilesystemencoding())
+
     n_left = os.path.normcase(left)
     n_right = os.path.normcase(right)
     if show:
         print('LEFT : ' + n_left)
         print('RIGHT: ' + n_right)
-    return str(n_left) == str(n_right)
+    return n_left == n_right

pytests/helpers/pattern.py

@@ -63,15 +63,6 @@ def __ne__(self, other):
         items = dict(items)
         return AnyDictWith(items)
 
-    @staticmethod
-    def path(p):
-        class AnyStrPath(str):
-            def __eq__(self, other):
-                return compare_path(self, other, show=False)
-            def __ne__(self, other):
-                return not (self == other)
-        return AnyStrPath(p)
-
 
 class Maybe(BasePattern):
     """A pattern that matches if condition is True.
@@ -118,6 +109,26 @@ def __eq__(self, value):
         return self.obj is value
 
 
+class Path(object):
+    """A pattern that matches strings as path, using os.path.normcase before comparison,
+    and sys.getfilesystemencoding() to compare Unicode and non-Unicode strings.
+    """
+
+    def __init__(self, s):
+        self.s = s
+
+    def __repr__(self):
+        return 'Path(%r)' % (self.s,)
+
+    def __eq__(self, other):
+        if not (isinstance(other, bytes) or isinstance(other, unicode)):
+            return NotImplemented
+        return compare_path(self.s, other, show=False)
+
+    def __ne__(self, other):
+        return not (self == other)
+
+
 SUCCESS = Success(True)
 FAILURE = Success(False)