Support Python 3.10 & 3.11

.github/workflows/build-darwin.yml

@@ -12,6 +12,11 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
+      - uses: actions/setup-python@v2
+        name: Install Python 3.10
+        with:
+          python-version: "3.10"
+
       - uses: actions/setup-python@v2
         name: Install Python 3.9
         with:
@@ -46,6 +51,10 @@ jobs:
         run: |
           pip install -r requirements-dev.txt
 
+      - name: Test for Python 3.10
+        run: |
+          tox -e py310
+
       - name: Test for Python 3.9
         run: |
           tox -e py39
@@ -67,7 +76,7 @@ jobs:
         run: |
           python -m cibuildwheel --output-dir wheelhouse
         env:
-          CIBW_BUILD: cp36-* cp37-* cp38-* cp39-*
+          CIBW_BUILD: cp36-* cp37-* cp38-* cp39-* cp310-*
 
       - uses: actions/upload-artifact@v1
         if: contains(github.ref, 'tags/')

.github/workflows/build-linux.yml

@@ -12,6 +12,11 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
+      - uses: actions/setup-python@v2
+        name: Install Python 3.10
+        with:
+          python-version: "3.10"
+
       - uses: actions/setup-python@v2
         name: Install Python 3.9
         with:
@@ -46,6 +51,10 @@ jobs:
         run: |
           pip install -r requirements-dev.txt
 
+      - name: Test for Python 3.10
+        run: |
+          tox -e py310
+
       - name: Test for Python 3.9
         run: |
           tox -e py39
@@ -67,7 +76,7 @@ jobs:
         run: |
           python -m cibuildwheel --output-dir wheelhouse
         env:
-          CIBW_BUILD: cp36-* cp37-* cp38-* cp39-*
+          CIBW_BUILD: cp36-* cp37-* cp38-* cp39-* cp310-*
 
       - uses: actions/upload-artifact@v1
         if: contains(github.ref, 'tags/')

.github/workflows/build-windows.yml

@@ -18,6 +18,12 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
+      - uses: actions/setup-python@v2
+        name: Install Python 3.10
+        with:
+          python-version: "3.10"
+          architecture: ${{ matrix.config.py-architecture }} 
+
       - uses: actions/setup-python@v2
         name: Install Python 3.9
         with:
@@ -56,6 +62,12 @@ jobs:
         run: |
           pip install -r requirements-dev.txt
 
+      - name: Test for Python 3.10
+        shell: cmd
+        run: |
+          call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\Common7\Tools\VsDevCmd.bat" -no_logo -arch=${{ matrix.config.vs-arch }}
+          tox -e py310
+
       - name: Test for Python 3.9
         shell: cmd
         run: |
@@ -85,7 +97,7 @@ jobs:
         run: |
           python -m cibuildwheel --output-dir wheelhouse
         env:
-          CIBW_BUILD: cp36-* cp37-* cp38-* cp39-*
+          CIBW_BUILD: cp36-* cp37-* cp38-* cp39-* cp310-*
 
       - uses: actions/upload-artifact@v1
         if: contains(github.ref, 'tags/')

Makefile

@@ -1,5 +1,5 @@
 test:
-	python setup.py nosetests
+	python -m pytest
 
 clean:
 	$(RM) -r build dist src/cchardet/__pycache__ src/cchardet/*.cpp src/cchardet/*.pyc src/cchardet/*.so src/cchardet.egg-info src/tests/__pycache__ src/tests/*.pyc

dockerfiles/buildwheel.sh

@@ -4,7 +4,7 @@ set -e -x
 ARCH=`uname -p`
 echo "arch=$ARCH"
 
-for V in cp39-cp39 cp38-cp38 cp37-cp37m cp36-cp36m; do
+for V in cp310-cp310 cp39-cp39 cp38-cp38 cp37-cp37m cp36-cp36m; do
     PYBIN=/opt/python/$V/bin
     rm -rf build src/cchardet/__pycache__ src/cchardet/*.cpp src/cchardet/*.pyc src/cchardet/*.so src/cchardet.egg-info src/tests/__pycache__ src/tests/*.pyc
     $PYBIN/pip install -r requirements-dev.txt

dockerfiles/develop/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.9-buster
+FROM python:3.10-buster
 
 RUN pip install -U cython chardet nose
 

dockerfiles/slim/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.9-slim-buster
+FROM python:3.10-slim-buster
 
 ENV BUILD_DEPS gcc g++
 

requirements-dev.txt

@@ -1,5 +1,5 @@
 chardet==3.0.4
-nose==1.3.7
+pytest==6.2.5
 tox==3.20.1
 Cython==0.29.21
 # pylint==2.6.0

setup.py

@@ -133,6 +133,7 @@ def read(f):
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3.9',
+        'Programming Language :: Python :: 3.10', 
     ],
     keywords=[
         'cython',

src/tests/cchardet_test.py

@@ -0,0 +1,120 @@
+import glob
+import os
+
+import cchardet
+
+SKIP_LIST = [
+    "src/tests/testdata/ja/utf-16le.txt",
+    "src/tests/testdata/ja/utf-16be.txt",
+    "src/tests/testdata/es/iso-8859-15.txt",
+    "src/tests/testdata/da/iso-8859-1.txt",
+    "src/tests/testdata/he/iso-8859-8.txt",
+]
+
+# Python can't decode encoding
+SKIP_LIST_02 = [
+    "src/tests/testdata/vi/viscii.txt",
+    "src/tests/testdata/zh/euc-tw.txt",
+]
+SKIP_LIST_02.extend(SKIP_LIST)
+
+
+def test_ascii():
+    detected_encoding = cchardet.detect(b"abcdefghijklmnopqrstuvwxyz")
+    assert "ascii" == detected_encoding["encoding"].lower()
+
+
+def test_detect():
+    testfiles = glob.glob("src/tests/testdata/*/*.txt")
+    for testfile in testfiles:
+        if testfile.replace("\\", "/") in SKIP_LIST:
+            continue
+
+        base = os.path.basename(testfile)
+        expected_charset = os.path.splitext(base)[0]
+        with open(testfile, "rb") as f:
+            msg = f.read()
+            detected_encoding = cchardet.detect(msg)
+            assert (
+                expected_charset.lower()
+                == detected_encoding["encoding"].lower()
+            )
+
+
+def test_detector():
+    detector = cchardet.UniversalDetector()
+    with open(
+        "src/tests/samples/wikipediaJa_One_Thousand_and_One_Nights_SJIS.txt",
+        "rb",
+    ) as f:
+        line = f.readline()
+        while line:
+            detector.feed(line)
+            if detector.done:
+                break
+            line = f.readline()
+    detector.close()
+    detected_encoding = detector.result
+    assert "shift_jis" == detected_encoding["encoding"].lower()
+
+
+def test_github_issue_20():
+    """
+    https://github.com/PyYoshi/cChardet/issues/20
+    """
+    msg = b"\x8f"
+
+    cchardet.detect(msg)
+
+    detector = cchardet.UniversalDetector()
+    detector.feed(msg)
+    detector.close()
+
+
+def test_decode():
+    testfiles = glob.glob("src/tests/testdata/*/*.txt")
+    for testfile in testfiles:
+        if testfile.replace("\\", "/") in SKIP_LIST_02:
+            continue
+
+        base = os.path.basename(testfile)
+        expected_charset = os.path.splitext(base)[0]
+        with open(testfile, "rb") as f:
+            msg = f.read()
+            detected_encoding = cchardet.detect(msg)
+            try:
+                msg.decode(detected_encoding["encoding"])
+            except LookupError as e:
+                print(
+                    "LookupError: { file=%s, encoding=%s }"
+                    % (testfile, detected_encoding["encoding"]),
+                )
+                raise e
+
+
+def test_utf8_with_bom():
+    sample = b"\xEF\xBB\xBF"
+    detected_encoding = cchardet.detect(sample)
+    assert "utf-8-sig" == detected_encoding["encoding"].lower()
+
+
+def test_null_bytes():
+    sample = b"ABC\x00\x80\x81"
+    detected_encoding = cchardet.detect(sample)
+
+    assert detected_encoding["encoding"] is None
+
+
+# def test_iso8859_2_csv(self):
+#     testfile = 'tests/samples/iso8859-2.csv'
+#     with open(testfile, 'rb') as f:
+#         msg = f.read()
+#         detected_encoding = cchardet.detect(msg)
+#         eq_(
+#             "iso8859-2",
+#             detected_encoding['encoding'].lower(),
+#             'Expected %s, but got %s' % (
+#                 "iso8859-2",
+#                 detected_encoding['encoding'].lower()
+#             )
+#         )