diff --git a/.github/workflows/check_encoding.yml b/.github/workflows/check_encoding.yml index e335757b8..88cd80cf0 100644 --- a/.github/workflows/check_encoding.yml +++ b/.github/workflows/check_encoding.yml @@ -18,5 +18,5 @@ jobs: architecture: 'x64' - run: pip install chardet - name: check_encoding - run: python check_encoding.py + run: python ./check_encoding.py ./check_encoding.json working-directory: ./Script/CI diff --git a/Docs/General/CodingRule.md b/Docs/General/CodingRule.md index 87920aa63..158df1afe 100644 --- a/Docs/General/CodingRule.md +++ b/Docs/General/CodingRule.md @@ -129,9 +129,9 @@ finalthing(); ### 文字コード -- 文字コードはShift-JISで統一する． - - HEWがこれのみに対応するため． - - S2E (Visual Studio) ではWarningが出るが，やむをえない． +- 文字コードは基本的に UTF-8 で統一する． + - 一部のバッチファイルなど， Shift-JIS のほうが圧倒的に楽なものについては例外を認める． + - User側で Shift-JIS などを要求する場合 (HEW など) は， User 側のビルドワークフローで変換するなどで対応する． - 改行コードはCR+LFで統一する． @@ -685,7 +685,7 @@ typedef enum ```cpp #ifndef SILS_FW // #define SILS_FW //ここでdefineする！ -#endif //SILS_FW +#endif #ifdef SILS_FW #define IS_LITTLE_ENDIAN // SILS環境により変更する diff --git a/Examples/minimum_user_for_s2e/Doxyfile b/Examples/minimum_user_for_s2e/Doxyfile index 61cb719e5..01623aba1 100644 --- a/Examples/minimum_user_for_s2e/Doxyfile +++ b/Examples/minimum_user_for_s2e/Doxyfile @@ -880,7 +880,7 @@ INPUT = # https://www.gnu.org/software/libiconv/) for the list of possible encodings. # The default value is: UTF-8. -INPUT_ENCODING = CP932 +INPUT_ENCODING = UTF-8 # If the value of the INPUT tag contains directories, you can use the # FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and diff --git a/Examples/minimum_user_for_s2e/src/src_user/Script/activate_sils_define.bat b/Examples/minimum_user_for_s2e/src/src_user/Script/activate_sils_define.bat index 18920da68..7279de031 100644 --- a/Examples/minimum_user_for_s2e/src/src_user/Script/activate_sils_define.bat +++ b/Examples/minimum_user_for_s2e/src/src_user/Script/activate_sils_define.bat @@ -1,31 +1,29 @@ @ECHO OFF -REM @brief sils_definẽRgAEg +REM @brief sils_define.h ̃RgAEg -REM XNvgsAsils_define.h +REM XNvgsCsils_define.h REM ``` -REM //#define SILS_FW //defineI +REM // #define SILS_FW REM ``` -REM ƏꂽsoARgAEgB -REM ȉ̃TCgQlɂB +REM ƏꂽsoCRgAEg +REM ȉ̃TCgQlɂ REM u: https://qiita.com/wagase/items/2180d8911dcc2a748fd3 REM s : https://blogs.yahoo.co.jp/kerupani/15344574.html cd /d %~dp0 - setlocal enabledelayedexpansion -rem for /f "tokens=1* delims=:" %%A in ('findstr /n "^" ./sils_define.h') do ( + for /f "tokens=1* delims=:" %%A in ('findstr /n "^" ..\Settings\sils_define.h') do ( set line0=%%B REM ̍sechoŕ\łȂ̂ŕ򂵂Ă if "%%B" == "" ( - echo.>>..\Settings\SILS_DEFINE.tmp + echo.>>..\Settings\sils_define.tmp ) else ( - set line1=!line0://#define SILS_FW //defineI=#define SILS_FW //defineI! - echo !line1!>>..\Settings\SILS_DEFINE.tmp + set line1=!line0:// #define SILS_FW=#define SILS_FW! + echo !line1!>>..\Settings\sils_define.tmp ) ) -REM ꎞt@C폜AύX𔽉f -rem move SILS_DEFINE.tmp sils_define.h -move ..\Settings\SILS_DEFINE.tmp ..\Settings\sils_define.h +REM ꎞt@C폜CύX𔽉f +move ..\Settings\sils_define.tmp ..\Settings\sils_define.h diff --git a/Examples/minimum_user_for_s2e/src/src_user/Script/deactivate_sils_define.bat b/Examples/minimum_user_for_s2e/src/src_user/Script/deactivate_sils_define.bat index 95b531262..08ee78390 100644 --- a/Examples/minimum_user_for_s2e/src/src_user/Script/deactivate_sils_define.bat +++ b/Examples/minimum_user_for_s2e/src/src_user/Script/deactivate_sils_define.bat @@ -1,12 +1,12 @@ @ECHO OFF -REM @brief sils_definẽRgAEgL +REM @brief sils_define.h ̃RgAEgL -REM XNvgsAsils_define.h +REM XNvgsCsils_define.h REM ``` -REM #define SILS_FW //defineI +REM #define SILS_FW REM ``` -REM ƏꂽsoARgAEgB -REM ȉ̃TCgQlɂB +REM ƏꂽsoCRgAEgL +REM ȉ̃TCgQlɂ REM u: https://qiita.com/wagase/items/2180d8911dcc2a748fd3 REM s : https://blogs.yahoo.co.jp/kerupani/15344574.html @@ -18,14 +18,13 @@ for /f "tokens=1* delims=:" %%A in ('findstr /n "^" ..\Settings\sils_define.h') REM ̍sechoŕ\łȂ̂ŕ򂵂Ă if "%%B" == "" ( - echo.>>..\Settings\SILS_DEFINE.tmp + echo.>>..\Settings\sils_define.tmp ) else ( - set line0=!line0:#define SILS_FW //defineI=//#define SILS_FW //defineI! + set line0=!line0:#define SILS_FW=// #define SILS_FW! set line0=!line0:////=//! - echo !line0!>>..\Settings\SILS_DEFINE.tmp + echo !line0!>>..\Settings\sils_define.tmp ) ) REM ꎞt@C폜AύX𔽉f -rem move SILS_DEFINE.tmp sils_define.h -move ..\Settings\SILS_DEFINE.tmp ..\Settings\sils_define.h +move ..\Settings\sils_define.tmp ..\Settings\sils_define.h diff --git a/Examples/minimum_user_for_s2e/src/src_user/Settings/build_settings.h b/Examples/minimum_user_for_s2e/src/src_user/Settings/build_settings.h index 9e0c904fa..e68dc3aad 100644 --- a/Examples/minimum_user_for_s2e/src/src_user/Settings/build_settings.h +++ b/Examples/minimum_user_for_s2e/src/src_user/Settings/build_settings.h @@ -3,7 +3,9 @@ #include "sils_define.h" -#ifndef SILS_FW +#ifdef SILS_FW +#define IS_LITTLE_ENDIAN // SILS環境により変更する +#else // #define IS_LITTLE_ENDIAN // マイコン環境により変更する #endif diff --git a/Examples/minimum_user_for_s2e/src/src_user/Settings/sils_define.h b/Examples/minimum_user_for_s2e/src/src_user/Settings/sils_define.h index f33c44137..ccbf5aa7e 100644 --- a/Examples/minimum_user_for_s2e/src/src_user/Settings/sils_define.h +++ b/Examples/minimum_user_for_s2e/src/src_user/Settings/sils_define.h @@ -1,12 +1,13 @@ +/** + * @file + * @brief Switch the build settings between your real OBC and SILS + * @note Switchable with activate_sils_define.bat and deactivate_sils_define.bat + */ #ifndef SILS_DEFINE_H_ #define SILS_DEFINE_H_ #ifndef SILS_FW -// #define SILS_FW //ここでdefineする！ -#endif // SILS_FW - -#ifdef SILS_FW -#define IS_LITTLE_ENDIAN // SILS環境により変更する +// #define SILS_FW #endif #endif diff --git a/Examples/minimum_user_for_s2e/src/src_user/Test/utils/c2a_enum_utils.py b/Examples/minimum_user_for_s2e/src/src_user/Test/utils/c2a_enum_utils.py index 51b40603e..ebb399666 100644 --- a/Examples/minimum_user_for_s2e/src/src_user/Test/utils/c2a_enum_utils.py +++ b/Examples/minimum_user_for_s2e/src/src_user/Test/utils/c2a_enum_utils.py @@ -19,4 +19,4 @@ def get_c2a_enum(): + json_dict["c2a_src_rel_path"] ) - return c2a.load_enum(c2a_src_abs_path) + return c2a.load_enum(c2a_src_abs_path, "utf-8") diff --git a/Script/CI/check_coding_rule.json b/Script/CI/check_coding_rule.json index b66602079..c13b1bae0 100644 --- a/Script/CI/check_coding_rule.json +++ b/Script/CI/check_coding_rule.json @@ -1,5 +1,6 @@ { "c2a_root_dir" : "./", + "input_file_encoding" : "utf-8", "target_dirs" : [ "src_core/", "src_user/" diff --git a/Script/CI/check_coding_rule.py b/Script/CI/check_coding_rule.py index 0ad4ca7ec..a65737d6d 100644 --- a/Script/CI/check_coding_rule.py +++ b/Script/CI/check_coding_rule.py @@ -62,7 +62,7 @@ def check_coding_rule(settings: dict) -> int: for ignore_file in settings['ignore_files']: ignore_files.append(settings['c2a_root_dir'] + ignore_file) - preprocess_(target_dirs, ignore_dirs, ignore_files, settings['additional_type']) + preprocess_(target_dirs, ignore_dirs, ignore_files, settings) for target_dir in target_dirs: for root, dirs, files in os.walk(target_dir): @@ -92,14 +92,14 @@ def check_coding_rule(settings: dict) -> int: print(path) continue - ret = check_file_(path) + ret = check_file_(path, settings) if ret != 0: # print(path) flag = 1 return flag; -def preprocess_(target_dirs: list, ignore_dirs: list, ignore_files: list, additional_type: list): +def preprocess_(target_dirs: list, ignore_dirs: list, ignore_files: list, settings: dict): global g_type_set for target_dir in target_dirs: for root, dirs, files in os.walk(target_dir): @@ -113,21 +113,19 @@ def preprocess_(target_dirs: list, ignore_dirs: list, ignore_files: list, additi path = path.replace("\\", "/") if path in ignore_files: continue - preprocess_inner_(path) + preprocess_inner_(path, settings) ignore_types = ["auto", "signed", "unsigned", "using", "typedef", "struct", "enum", "class"] for ignore_type in ignore_types: if ignore_type in g_type_set: g_type_set.remove(ignore_type) - # pprint.pprint(additional_type) - g_type_set |= set(additional_type) + # pprint.pprint(settings['additional_type']) + g_type_set |= set(settings['additional_type']) # pprint.pprint(g_type_set) -def preprocess_inner_(path: str): - # FIXME: ユーザー定義にする - # with open(path, encoding='shift_jis') as f: - with open(path, encoding='utf-8') as f: +def preprocess_inner_(path: str, settings: dict): + with open(path, encoding=settings['input_file_encoding']) as f: code_lines = f.read().split("\n") ptn_find_type = '^ *(\w+)\*? +\w+' @@ -154,12 +152,10 @@ def preprocess_inner_(path: str): # 0: OK, 1: NG -def check_file_(path: str) -> int: +def check_file_(path: str, settings: dict) -> int: flag = 0 - # FIXME: ユーザー定義にする - # with open(path, encoding='shift_jis') as f: - with open(path, encoding='utf-8') as f: + with open(path, encoding=settings['input_file_encoding']) as f: code_lines = f.read().split("\n") # print(path) diff --git a/Script/CI/check_encoding.json b/Script/CI/check_encoding.json index 650d63d79..eb46f4bed 100644 --- a/Script/CI/check_encoding.json +++ b/Script/CI/check_encoding.json @@ -1,5 +1,35 @@ { "root_dir" : "../../", + "text_file_config" : { + "input_encoding": "utf-8", + "extensions": [ + "txt", + "md", + "json", + "cmake" + ] + }, + "code_file_config" : { + "input_encoding" : "utf-8", + "extensions" : [ + "c", + "h" + ] + }, + "script_file_config" : { + "input_encoding" : "utf-8", + "extensions" : [ + "sh", + "pl", + "py" + ] + }, + "exceptional_file_config" : { + "input_encoding" : "shift_jis", + "extensions" : [ + "bat" + ] + }, "target_dirs" : [ "Applications/", "Drivers/", diff --git a/Script/CI/check_encoding.py b/Script/CI/check_encoding.py index 10a08f2c6..b18623810 100644 --- a/Script/CI/check_encoding.py +++ b/Script/CI/check_encoding.py @@ -15,11 +15,18 @@ DEBUG = 0 # 0 : Release # 1 : all -SETTING_FILE_PATH = "check_encoding.json" - def main(): - with open(SETTING_FILE_PATH, mode='r') as fh: + if len(sys.argv) != 2: + print("Please give a setting file as an argumente.") + sys.exit(1) + + setting_file_path = sys.argv[1] + if not os.path.isfile(setting_file_path): + print("Setting file not found.") + sys.exit(1) + + with open(setting_file_path, mode='r') as fh: settings = json.load(fh) if DEBUG: pprint.pprint(settings) @@ -30,7 +37,7 @@ def main(): flag = 0 for target_dir in target_dirs: - ret = check(target_dir) + ret = check(target_dir, settings) if ret != 0: flag = 1 @@ -42,17 +49,25 @@ def main(): # 0: OK, 1: NG -def check(target_dir): +def check(target_dir, settings): flag = 0 for root, dirs, files in os.walk(target_dir): for file in files: - ext = (os.path.splitext(file))[1] + ext = (os.path.splitext(file))[1].replace('.', '') # print(ext) - # FIXME: md, txt, cmake, sh, bat もチェックするようにする - if ext != ".h" and ext != ".c": + if ext in settings['text_file_config']['extensions']: + encoding = settings['text_file_config']['input_encoding'] + elif ext in settings['code_file_config']['extensions']: + encoding = settings['code_file_config']['input_encoding'] + elif ext in settings['script_file_config']['extensions']: + encoding = settings['script_file_config']['input_encoding'] + elif ext in settings['exceptional_file_config']['extensions']: + encoding = settings['exceptional_file_config']['input_encoding'] + else: continue + path = root + r"/" + file - ret = check_encoding(path) + ret = check_encoding(path, encoding) if ret != 0: print(path) flag = 1 @@ -60,25 +75,28 @@ def check(target_dir): # 0: OK, 1: NG -def check_encoding(path): +def check_encoding(path, encoding): with open(path, 'rb') as f: # print(path) ret = chardet.detect(f.read()) enc = ret['encoding'] # print(enc) - # FIXME: ユーザー定義にする - # SJIS Ver - # if enc == 'SHIFT_JIS' or enc == 'CP932' or enc == 'ascii': - # return 0 - # # なぜか以下のような誤認もあるので - # if enc == 'Windows-1252' or enc == 'Windows-1254' or enc is None: - # return 0 - # UTF-8 Ver - if enc == 'utf-8' or enc == 'ascii': - return 0 - # なぜか以下のような誤認もあるので - if enc == 'Windows-1252' or enc == 'ISO-8859-1' or enc is None: - return 0 + if encoding == 'utf-8': + if enc == 'utf-8' or enc == 'ascii': + return 0 + # なぜか以下のような誤認もあるので + if enc == 'Windows-1252' or enc == 'ISO-8859-1' or enc is None: + return 0 + elif encoding == 'shift_jis': + if enc == 'SHIFT_JIS' or enc == 'CP932' or enc == 'ascii': + return 0 + # なぜか以下のような誤認もあるので + if enc == 'Windows-1252' or enc == 'Windows-1254' or enc is None: + return 0 + else: + print("Invalid encoding in setting file!") + return 1 + print(ret) return 1 diff --git a/common.cmake b/common.cmake index 2e7602b78..78200ad36 100644 --- a/common.cmake +++ b/common.cmake @@ -21,6 +21,7 @@ endif() if(MSVC) target_compile_options(${PROJECT_NAME} PUBLIC "/W4") target_compile_options(${PROJECT_NAME} PUBLIC "/TP") # Compile C codes as C++ + target_compile_options(${PROJECT_NAME} PUBLIC "/source-charset:utf-8") else() # SJIS # if (NOT CMAKE_C_COMPILER_ID STREQUAL "Clang")