Merge pull request #185 from ut-issl/feature/fix_after_change_code_en…

…coding

ソースコードの文字コードをUTF-8にした後の様々な微修正

.github/workflows/check_encoding.yml

@@ -18,5 +18,5 @@ jobs:
           architecture: 'x64'
       - run: pip install chardet
       - name: check_encoding
-        run: python check_encoding.py
+        run: python ./check_encoding.py ./check_encoding.json
         working-directory: ./Script/CI

Docs/General/CodingRule.md

@@ -129,9 +129,9 @@ finalthing();
 
 
 ### 文字コード
-- 文字コードはShift-JISで統一する．
-	- HEWがこれのみに対応するため．
-	- S2E (Visual Studio) ではWarningが出るが，やむをえない．
+- 文字コードは基本的に UTF-8 で統一する．
+	- 一部のバッチファイルなど， Shift-JIS のほうが圧倒的に楽なものについては例外を認める．
+	- User側で Shift-JIS などを要求する場合 (HEW など) は， User 側のビルドワークフローで変換するなどで対応する．
 - 改行コードはCR+LFで統一する．
 
 
@@ -685,7 +685,7 @@ typedef enum
 ```cpp
 #ifndef SILS_FW
 // #define SILS_FW //ここでdefineする！
-#endif //SILS_FW
+#endif
 
 #ifdef SILS_FW
 #define IS_LITTLE_ENDIAN // SILS環境により変更する

Examples/minimum_user_for_s2e/Doxyfile

@@ -880,7 +880,7 @@ INPUT                  =
 # https://www.gnu.org/software/libiconv/) for the list of possible encodings.
 # The default value is: UTF-8.
 
-INPUT_ENCODING         = CP932
+INPUT_ENCODING         = UTF-8
 
 # If the value of the INPUT tag contains directories, you can use the
 # FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and

Examples/minimum_user_for_s2e/src/src_user/Script/activate_sils_define.bat

@@ -1,31 +1,29 @@
 @ECHO OFF
-REM @brief sils_defineのコメントアウトを解除する
+REM @brief sils_define.h のコメントアウトを解除する
 
-REM スクリプト実行時、sils_define.hから
+REM スクリプト実行時，sils_define.h から
 REM ```
-REM //#define SILS_FW //ここでdefineする！
+REM // #define SILS_FW
 REM ```
-REM と書かれた行を検出し、コメントアウトを解除する。
-REM 以下のサイトを参考にした。
+REM と書かれた行を検出し，コメントアウトを解除する
+REM 以下のサイトを参考にした
 REM   文字列置換: https://qiita.com/wagase/items/2180d8911dcc2a748fd3
 REM   空行処理  : https://blogs.yahoo.co.jp/kerupani/15344574.html
 
 cd /d %~dp0
-
 setlocal enabledelayedexpansion
-rem for /f "tokens=1* delims=:" %%A in ('findstr /n "^" ./sils_define.h') do (
+
 for /f "tokens=1* delims=:" %%A in ('findstr /n "^" ..\Settings\sils_define.h') do (
   set line0=%%B
 
   REM 空の行はechoで表示できないので分岐しておく
   if "%%B" == "" (
-    echo.>>..\Settings\SILS_DEFINE.tmp
+    echo.>>..\Settings\sils_define.tmp
   ) else (
-    set line1=!line0://#define SILS_FW //ここでdefineする！=#define SILS_FW //ここでdefineする！!
-    echo !line1!>>..\Settings\SILS_DEFINE.tmp
+    set line1=!line0:// #define SILS_FW=#define SILS_FW!
+    echo !line1!>>..\Settings\sils_define.tmp
   )
 )
 
-REM 一時ファイルを削除し、変更を反映
-rem move SILS_DEFINE.tmp sils_define.h
-move ..\Settings\SILS_DEFINE.tmp ..\Settings\sils_define.h
+REM 一時ファイルを削除し，変更を反映
+move ..\Settings\sils_define.tmp ..\Settings\sils_define.h

Examples/minimum_user_for_s2e/src/src_user/Script/deactivate_sils_define.bat

@@ -1,12 +1,12 @@
 @ECHO OFF
-REM @brief sils_defineのコメントアウトを有効化する
+REM @brief sils_define.h のコメントアウトを有効化する
 
-REM スクリプト実行時、sils_define.hから
+REM スクリプト実行時，sils_define.h から
 REM ```
-REM #define SILS_FW //ここでdefineする！
+REM #define SILS_FW
 REM ```
-REM と書かれた行を検出し、コメントアウトする。
-REM 以下のサイトを参考にした。
+REM と書かれた行を検出し，コメントアウトを有効化する
+REM 以下のサイトを参考にした
 REM   文字列置換: https://qiita.com/wagase/items/2180d8911dcc2a748fd3
 REM   空行処理  : https://blogs.yahoo.co.jp/kerupani/15344574.html
 
@@ -18,14 +18,13 @@ for /f "tokens=1* delims=:" %%A in ('findstr /n "^" ..\Settings\sils_define.h')
 
   REM 空の行はechoで表示できないので分岐しておく
   if "%%B" == "" (
-    echo.>>..\Settings\SILS_DEFINE.tmp
+    echo.>>..\Settings\sils_define.tmp
   ) else (
-    set line0=!line0:#define SILS_FW //ここでdefineする！=//#define SILS_FW //ここでdefineする！!
+    set line0=!line0:#define SILS_FW=// #define SILS_FW!
     set line0=!line0:////=//!
-    echo !line0!>>..\Settings\SILS_DEFINE.tmp
+    echo !line0!>>..\Settings\sils_define.tmp
   )
 )
 
 REM 一時ファイルを削除し、変更を反映
-rem move SILS_DEFINE.tmp sils_define.h
-move ..\Settings\SILS_DEFINE.tmp ..\Settings\sils_define.h
+move ..\Settings\sils_define.tmp ..\Settings\sils_define.h

Examples/minimum_user_for_s2e/src/src_user/Settings/build_settings.h

@@ -3,7 +3,9 @@
 
 #include "sils_define.h"
 
-#ifndef SILS_FW
+#ifdef SILS_FW
+#define IS_LITTLE_ENDIAN // SILS環境により変更する
+#else
 // #define IS_LITTLE_ENDIAN // マイコン環境により変更する
 #endif
 

Examples/minimum_user_for_s2e/src/src_user/Settings/sils_define.h

@@ -1,12 +1,13 @@
+/**
+ * @file
+ * @brief Switch the build settings between your real OBC and SILS
+ * @note  Switchable with activate_sils_define.bat and deactivate_sils_define.bat
+ */
 #ifndef SILS_DEFINE_H_
 #define SILS_DEFINE_H_
 
 #ifndef SILS_FW
-// #define SILS_FW //ここでdefineする！
-#endif // SILS_FW
-
-#ifdef SILS_FW
-#define IS_LITTLE_ENDIAN // SILS環境により変更する
+// #define SILS_FW
 #endif
 
 #endif

Examples/minimum_user_for_s2e/src/src_user/Test/utils/c2a_enum_utils.py

@@ -19,4 +19,4 @@ def get_c2a_enum():
             + json_dict["c2a_src_rel_path"]
         )
 
-    return c2a.load_enum(c2a_src_abs_path)
+    return c2a.load_enum(c2a_src_abs_path, "utf-8")

Script/CI/check_coding_rule.json

@@ -1,5 +1,6 @@
 {
   "c2a_root_dir" : "./",
+  "input_file_encoding" : "utf-8",
   "target_dirs" : [
     "src_core/",
     "src_user/"

Script/CI/check_coding_rule.py

@@ -62,7 +62,7 @@ def check_coding_rule(settings: dict) -> int:
     for ignore_file in settings['ignore_files']:
         ignore_files.append(settings['c2a_root_dir'] + ignore_file)
 
-    preprocess_(target_dirs, ignore_dirs, ignore_files, settings['additional_type'])
+    preprocess_(target_dirs, ignore_dirs, ignore_files, settings)
 
     for target_dir in target_dirs:
         for root, dirs, files in os.walk(target_dir):
@@ -92,14 +92,14 @@ def check_coding_rule(settings: dict) -> int:
                         print(path)
                     continue
 
-                ret = check_file_(path)
+                ret = check_file_(path, settings)
                 if ret != 0:
                     # print(path)
                     flag = 1
     return flag;
 
 
-def preprocess_(target_dirs: list, ignore_dirs: list, ignore_files: list, additional_type: list):
+def preprocess_(target_dirs: list, ignore_dirs: list, ignore_files: list, settings: dict):
     global g_type_set
     for target_dir in target_dirs:
         for root, dirs, files in os.walk(target_dir):
@@ -113,21 +113,19 @@ def preprocess_(target_dirs: list, ignore_dirs: list, ignore_files: list, additi
                 path = path.replace("\\", "/")
                 if path in ignore_files:
                     continue
-                preprocess_inner_(path)
+                preprocess_inner_(path, settings)
 
     ignore_types = ["auto", "signed", "unsigned", "using", "typedef", "struct", "enum", "class"]
     for ignore_type in ignore_types:
         if ignore_type in g_type_set:
             g_type_set.remove(ignore_type)
-    # pprint.pprint(additional_type)
-    g_type_set |= set(additional_type)
+    # pprint.pprint(settings['additional_type'])
+    g_type_set |= set(settings['additional_type'])
     # pprint.pprint(g_type_set)
 
 
-def preprocess_inner_(path: str):
-    # FIXME: ユーザー定義にする
-    # with open(path, encoding='shift_jis') as f:
-    with open(path, encoding='utf-8') as f:
+def preprocess_inner_(path: str, settings: dict):
+    with open(path, encoding=settings['input_file_encoding']) as f:
         code_lines = f.read().split("\n")
 
     ptn_find_type = '^ *(\w+)\*? +\w+'
@@ -154,12 +152,10 @@ def preprocess_inner_(path: str):
 
 
 # 0: OK, 1: NG
-def check_file_(path: str) -> int:
+def check_file_(path: str, settings: dict) -> int:
     flag = 0
 
-    # FIXME: ユーザー定義にする
-    # with open(path, encoding='shift_jis') as f:
-    with open(path, encoding='utf-8') as f:
+    with open(path, encoding=settings['input_file_encoding']) as f:
         code_lines = f.read().split("\n")
 
     # print(path)

Script/CI/check_encoding.json

@@ -1,5 +1,35 @@
 {
   "root_dir" : "../../",
+  "text_file_config" : {
+    "input_encoding": "utf-8",
+    "extensions": [
+      "txt",
+      "md",
+      "json",
+      "cmake"
+    ]
+  },
+  "code_file_config" : {
+    "input_encoding" : "utf-8",
+    "extensions" : [
+      "c",
+      "h"
+    ]
+  },
+  "script_file_config" : {
+    "input_encoding" : "utf-8",
+    "extensions" : [
+      "sh",
+      "pl",
+      "py"
+    ]
+  },
+  "exceptional_file_config" : {
+    "input_encoding" : "shift_jis",
+    "extensions" : [
+      "bat"
+    ]
+  },
   "target_dirs" : [
     "Applications/",
     "Drivers/",

Script/CI/check_encoding.py

@@ -15,11 +15,18 @@
 DEBUG = 0
 # 0 : Release
 # 1 : all
-SETTING_FILE_PATH = "check_encoding.json"
-
 
 def main():
-    with open(SETTING_FILE_PATH, mode='r') as fh:
+    if len(sys.argv) != 2:
+        print("Please give a setting file as an argumente.")
+        sys.exit(1)
+
+    setting_file_path = sys.argv[1]
+    if not os.path.isfile(setting_file_path):
+        print("Setting file not found.")
+        sys.exit(1)
+
+    with open(setting_file_path, mode='r') as fh:
         settings = json.load(fh)
     if DEBUG:
         pprint.pprint(settings)
@@ -30,7 +37,7 @@ def main():
 
     flag = 0
     for target_dir in target_dirs:
-        ret = check(target_dir)
+        ret = check(target_dir, settings)
         if ret != 0:
             flag = 1
 
@@ -42,43 +49,54 @@ def main():
 
 
 # 0: OK, 1: NG
-def check(target_dir):
+def check(target_dir, settings):
     flag = 0
     for root, dirs, files in os.walk(target_dir):
         for file in files:
-            ext = (os.path.splitext(file))[1]
+            ext = (os.path.splitext(file))[1].replace('.', '')
             # print(ext)
-            # FIXME: md, txt, cmake, sh, bat もチェックするようにする
-            if ext != ".h" and ext != ".c":
+            if ext in settings['text_file_config']['extensions']:
+                encoding = settings['text_file_config']['input_encoding']
+            elif ext in settings['code_file_config']['extensions']:
+                encoding = settings['code_file_config']['input_encoding']
+            elif ext in settings['script_file_config']['extensions']:
+                encoding = settings['script_file_config']['input_encoding']
+            elif ext in settings['exceptional_file_config']['extensions']:
+                encoding = settings['exceptional_file_config']['input_encoding']
+            else:
                 continue
+
             path = root + r"/" + file
-            ret = check_encoding(path)
+            ret = check_encoding(path, encoding)
             if ret != 0:
                 print(path)
                 flag = 1
     return flag
 
 
 # 0: OK, 1: NG
-def check_encoding(path):
+def check_encoding(path, encoding):
     with open(path, 'rb') as f:
         # print(path)
         ret = chardet.detect(f.read())
         enc = ret['encoding']
     # print(enc)
-    # FIXME: ユーザー定義にする
-    # SJIS Ver
-    # if enc == 'SHIFT_JIS' or enc == 'CP932' or enc == 'ascii':
-    #     return 0
-    # # なぜか以下のような誤認もあるので
-    # if enc == 'Windows-1252' or enc == 'Windows-1254' or enc is None:
-    #     return 0
-    # UTF-8 Ver
-    if enc == 'utf-8' or enc == 'ascii':
-        return 0
-    # なぜか以下のような誤認もあるので
-    if enc == 'Windows-1252' or enc == 'ISO-8859-1' or enc is None:
-        return 0
+    if encoding == 'utf-8':
+        if enc == 'utf-8' or enc == 'ascii':
+            return 0
+        # なぜか以下のような誤認もあるので
+        if enc == 'Windows-1252' or enc == 'ISO-8859-1' or enc is None:
+            return 0
+    elif encoding == 'shift_jis':
+        if enc == 'SHIFT_JIS' or enc == 'CP932' or enc == 'ascii':
+            return 0
+        # なぜか以下のような誤認もあるので
+        if enc == 'Windows-1252' or enc == 'Windows-1254' or enc is None:
+            return 0
+    else:
+        print("Invalid encoding in setting file!")
+        return 1
+
     print(ret)
     return 1
 

common.cmake

@@ -21,6 +21,7 @@ endif()
 if(MSVC)
   target_compile_options(${PROJECT_NAME} PUBLIC "/W4")
   target_compile_options(${PROJECT_NAME} PUBLIC "/TP") # Compile C codes as C++
+  target_compile_options(${PROJECT_NAME} PUBLIC "/source-charset:utf-8")
 else()
   # SJIS
   # if (NOT CMAKE_C_COMPILER_ID STREQUAL "Clang")