Fix more problems

Lib/tabnanny.py

@@ -107,6 +107,10 @@ def check(file):
         errprint("%r: Token Error: %s" % (file, msg))
         return
 
+    except SyntaxError as msg:
+        errprint("%r: Token Error: %s" % (file, msg))
+        return
+
     except IndentationError as msg:
         errprint("%r: Indentation Error: %s" % (file, msg))
         return
@@ -272,6 +276,12 @@ def format_witnesses(w):
     return prefix + " " + ', '.join(firsts)
 
 def process_tokens(tokens):
+    try:
+        _process_tokens(tokens)
+    except TabError as e:
+        raise NannyNag(e.lineno, e.msg, e.text)
+
+def _process_tokens(tokens):
     INDENT = tokenize.INDENT
     DEDENT = tokenize.DEDENT
     NEWLINE = tokenize.NEWLINE

Lib/test/test_tabnanny.py

@@ -223,7 +223,7 @@ def test_when_nannynag_error_verbose(self):
         with TemporaryPyFile(SOURCE_CODES["nannynag_errored"]) as file_path:
             out = f"{file_path!r}: *** Line 3: trouble in tab city! ***\n"
             out += "offending line: '\\tprint(\"world\")\\n'\n"
-            out += "indent not equal e.g. at tab size 1\n"
+            out += "inconsistent use of tabs and spaces in indentation\n"
 
             tabnanny.verbose = 1
             self.verify_tabnanny_check(file_path, out=out)
@@ -315,7 +315,7 @@ def validate_cmd(self, *args, stdout="", stderr="", partial=False, expect_failur
     def test_with_errored_file(self):
         """Should displays error when errored python file is given."""
         with TemporaryPyFile(SOURCE_CODES["wrong_indented"]) as file_path:
-            stderr  = f"{file_path!r}: Indentation Error: "
+            stderr  = f"{file_path!r}: Token Error: "
             stderr += ('unindent does not match any outer indentation level'
                     ' (<tokenize>, line 3)')
             self.validate_cmd(file_path, stderr=stderr, expect_failure=True)

Lib/tokenize.py

@@ -517,7 +517,10 @@ def error(message, filename=None, location=None):
                 tokens = list(tokenize(f.readline))
         else:
             filename = "<stdin>"
-            tokens = _tokenize(sys.stdin.readline, None)
+            tokens = _tokenize(
+                (x.encode('utf-8') for x in iter(sys.stdin.readline, "")
+            ), "utf-8")
+
 
         # Output the tokenization
         for token in tokens:

Lib/trace.py

@@ -360,7 +360,6 @@ def _find_strings(filename, encoding=None):
     # Add this special case so that the test in the loop passes.
     prev_ttype = token.INDENT
     with open(filename, encoding=encoding) as f:
-        print(filename)
         tok = tokenize.generate_tokens(f.readline)
         for ttype, tstr, start, end, line in tok:
             if ttype == token.STRING:

Python/Python-tokenize.c

@@ -89,8 +89,10 @@ _tokenizer_error(struct tok_state *tok)
             }
             return -1;
         case E_DEDENT:
-            PyErr_SetString(PyExc_IndentationError,
-                            "unindent does not match any outer indentation level");
+            PyErr_Format(PyExc_IndentationError,
+                        "unindent does not match any outer indentation level "
+                        "(<tokenize>, line %d)",
+                        tok->lineno);
             return -1;
         case E_INTR:
             if (!PyErr_Occurred()) {
@@ -115,7 +117,38 @@ _tokenizer_error(struct tok_state *tok)
         default:
             msg = "unknown tokenization error";
     }
-    PyErr_SetString(errtype, msg);
+
+    // TODO: Clean up this code and factor out common error paths
+
+    PyObject* errstr = NULL;
+    PyObject* error_line = NULL;
+
+    Py_ssize_t size = tok->inp - tok->buf;
+    error_line = PyUnicode_DecodeUTF8(tok->buf, size, "replace");
+    if (!error_line) {
+        goto error;
+    }
+    PyObject *tmp = Py_BuildValue("(OnnOii)", tok->filename, tok->lineno, 0, error_line, 0, 0);
+    if (!tmp) {
+        goto error;
+    }
+    Py_CLEAR(error_line);
+    errstr = PyUnicode_FromString(msg);
+    if (!errstr) {
+        goto error;
+    }
+    PyObject* value = PyTuple_Pack(2, errstr, tmp);
+    Py_DECREF(errstr);
+    Py_DECREF(tmp);
+    if (!value) {
+        goto error;
+    }
+    PyErr_SetObject(errtype, value);
+    Py_DECREF(value);
+    return 0;
+error:
+    Py_XDECREF(errstr);
+    Py_XDECREF(error_line);
     return -1;
 }