Merge pull request #158 from mull-project/develop

Treat inputs as UTF-8 (both stdin and check file)

filecheck/FileCheck.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import argparse
+import io
 import os
 import re
 import sys
@@ -275,7 +276,7 @@ def exit_handler(code):
         exit_handler(2)
 
     checks = []
-    with open(check_file) as f:
+    with open(check_file, encoding="utf-8") as f:
         for line_idx, line in enumerate(f):
             line = line.rstrip()
 
@@ -411,7 +412,11 @@ def exit_handler(code):
     # TODO: Performance implications?
     # "Getting exit code 141 when reading from stdin with a Python script with “set -o pipefail” set"
     # https://stackoverflow.com/questions/59436858/getting-exit-code-141-when-reading-from-stdin-with-a-python-script-with-set-o/59436997?noredirect=1#comment105058533_59436997
-    input_lines = sys.stdin.readlines()
+    # Also: Forcing the stdin to be UTF-8
+    # Python 3: How to specify stdin encoding
+    # https://stackoverflow.com/a/16549381/598057
+    input_stream = io.TextIOWrapper(sys.stdin.buffer, encoding='utf-8')
+    input_lines = input_stream.readlines()
     stdin_input_iter = enumerate(input_lines)
 
     try:

pyproject.toml

@@ -5,7 +5,7 @@ description = "Python port of LLVM's FileCheck, flexible pattern matching file v
 authors = ["Stanislav Pankevich <s.pankevich@gmail.com>"]
 
 [tool.poetry.dependencies]
-python = "^3.5"
+python = "^3.6"
 
 [tool.poetry.dev-dependencies]
 lit = "^0.9"

tests/integration/tests/general/01-utf8-characters/filecheck.check

@@ -0,0 +1,3 @@
+CHECK: # © A line with UTF-8 produces
+CHECK: # UnicodeDecodeError: 'ascii' codec can't decode byte
+CHECK: # if not properly handled

tests/integration/tests/general/01-utf8-characters/filecheck.input

@@ -0,0 +1,6 @@
+# © A line with UTF-8 produces
+# UnicodeDecodeError: 'ascii' codec can't decode byte
+# if not properly handled
+
+def hello_world():
+    print("hello world")

tests/integration/tests/general/01-utf8-characters/sample.itest

@@ -0,0 +1 @@
+RUN: %cat "%S/filecheck.input" | %expect_exit 0 --expect-no-content %FILECHECK_EXEC "%S/filecheck.check"