Skip to content

Commit

Permalink
bpo-36763: Fix _PyPreConfig_InitCompatConfig() utf8_mode (GH-13518)
Browse files Browse the repository at this point in the history 
* _PyPreConfig_InitCompatConfig() sets utf8_mode to 0.
* Change Py_UTF8Mode default value to 0.
* Fix _PyPreConfig_Copy(): copy also _config_init attrbibute.
* _PyPreConfig_AsDict() exports _config_init
* Fix _PyPreConfig_GetGlobalConfig(): use Py_UTF8Mode if it's greater
  than 0, even if utf8_mode >= 0.
* Add unit tests on environment variables using Python API.
  • Loading branch information
@vstinner
vstinner authored May 23, 2019
1 parent bc2aa81 commit 20e1e25
Show file tree
Hide file tree
Showing 4 changed files with 77 additions and 11 deletions.
38 changes: 34 additions & 4 deletions Lib/test/test_embed.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -287,6 +287,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
IGNORE_CONFIG = object()

PRE_CONFIG_COMPAT = {
'_config_init': API_COMPAT,
'allocator': PYMEM_ALLOCATOR_NOT_SET,
'parse_argv': 0,
'configure_locale': 1,
Expand All @@ -299,11 +300,13 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
'legacy_windows_fs_encoding': 0,
})
PRE_CONFIG_PYTHON = dict(PRE_CONFIG_COMPAT,
_config_init=API_PYTHON,
parse_argv=1,
coerce_c_locale=GET_DEFAULT_CONFIG,
utf8_mode=GET_DEFAULT_CONFIG,
)
PRE_CONFIG_ISOLATED = dict(PRE_CONFIG_COMPAT,
_config_init=API_ISOLATED,
configure_locale=0,
isolated=1,
use_environment=0,
Expand Down Expand Up @@ -388,10 +391,12 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
})

CORE_CONFIG_PYTHON = dict(CORE_CONFIG_COMPAT,
_config_init=API_PYTHON,
configure_c_stdio=1,
parse_argv=1,
)
CORE_CONFIG_ISOLATED = dict(CORE_CONFIG_COMPAT,
_config_init=API_ISOLATED,
isolated=1,
use_environment=0,
user_site_directory=0,
Expand Down Expand Up @@ -611,7 +616,6 @@ def check_config(self, testname, expected_config=None,
else:
default_config = self.CORE_CONFIG_COMPAT
expected_config = dict(default_config, **expected_config)
expected_config['_config_init'] = api

self.get_expected_config(expected_preconfig,
expected_config, env,
Expand Down Expand Up @@ -708,10 +712,9 @@ def test_init_from_config(self):
self.check_config("test_init_from_config", config, preconfig,
api=API_COMPAT)

def test_init_env(self):
def test_init_compat_env(self):
preconfig = {
'allocator': PYMEM_ALLOCATOR_MALLOC,
'utf8_mode': 1,
}
config = {
'use_hash_seed': 1,
Expand All @@ -732,9 +735,36 @@ def test_init_env(self):
'faulthandler': 1,
'warnoptions': ['EnvVar'],
}
self.check_config("test_init_env", config, preconfig,
self.check_config("test_init_compat_env", config, preconfig,
api=API_COMPAT)

def test_init_python_env(self):
preconfig = {
'allocator': PYMEM_ALLOCATOR_MALLOC,
'utf8_mode': 1,
}
config = {
'use_hash_seed': 1,
'hash_seed': 42,
'tracemalloc': 2,
'import_time': 1,
'malloc_stats': 1,
'inspect': 1,
'optimization_level': 2,
'module_search_path_env': '/my/path',
'pycache_prefix': 'env_pycache_prefix',
'write_bytecode': 0,
'verbose': 1,
'buffered_stdio': 0,
'stdio_encoding': 'iso8859-1',
'stdio_errors': 'replace',
'user_site_directory': 0,
'faulthandler': 1,
'warnoptions': ['EnvVar'],
}
self.check_config("test_init_python_env", config, preconfig,
api=API_PYTHON)

def test_init_env_dev_mode(self):
preconfig = dict(allocator=PYMEM_ALLOCATOR_DEBUG)
config = dict(dev_mode=1,
Expand Down
28 changes: 26 additions & 2 deletions Programs/_testembed.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -638,7 +638,7 @@ static void set_all_env_vars(void)
}


static int test_init_env(void)
static int test_init_compat_env(void)
{
/* Test initialization from environment variables */
Py_IgnoreEnvironmentFlag = 0;
Expand All @@ -650,6 +650,29 @@ static int test_init_env(void)
}


static int test_init_python_env(void)
{
_PyInitError err;

set_all_env_vars();

_PyCoreConfig config;
err = _PyCoreConfig_InitPythonConfig(&config);
if (_PyInitError_Failed(err)) {
_Py_ExitInitError(err);
}
config.program_name = L"./_testembed";

err = _Py_InitializeFromConfig(&config);
if (_PyInitError_Failed(err)) {
_Py_ExitInitError(err);
}
dump_config();
Py_Finalize();
return 0;
}


static void set_all_env_vars_dev_mode(void)
{
putenv("PYTHONMALLOC=");
Expand Down Expand Up @@ -1257,7 +1280,8 @@ static struct TestCase TestCases[] = {
{"test_init_from_config", test_init_from_config},
{"test_init_parse_argv", test_init_parse_argv},
{"test_init_dont_parse_argv", test_init_dont_parse_argv},
{"test_init_env", test_init_env},
{"test_init_compat_env", test_init_compat_env},
{"test_init_python_env", test_init_python_env},
{"test_init_env_dev_mode", test_init_env_dev_mode},
{"test_init_env_dev_mode_alloc", test_init_env_dev_mode_alloc},
{"test_init_dont_configure_locale", test_init_dont_configure_locale},
Expand Down
5 changes: 2 additions & 3 deletions Python/coreconfig.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,9 +107,8 @@ static const char usage_6[] =
/* --- Global configuration variables ----------------------------- */

/* UTF-8 mode (PEP 540): if equals to 1, use the UTF-8 encoding, and change
stdin and stdout error handler to "surrogateescape". It is equal to
-1 by default: unknown, will be set by Py_Main() */
int Py_UTF8Mode = -1;
stdin and stdout error handler to "surrogateescape". */
int Py_UTF8Mode = 0;
int Py_DebugFlag = 0; /* Needed by parser.c */
int Py_VerboseFlag = 0; /* Needed by import.c */
int Py_QuietFlag = 0; /* Needed by sysmodule.c */
Expand Down
17 changes: 15 additions & 2 deletions Python/preconfig.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -272,7 +272,16 @@ _PyPreConfig_InitCompatConfig(_PyPreConfig *config)
config->isolated = -1;
config->use_environment = -1;
config->configure_locale = 1;
config->utf8_mode = -1;

/* bpo-36443: C locale coercion (PEP 538) and UTF-8 Mode (PEP 540)
are disabled by default using the Compat configuration.
Py_UTF8Mode=1 enables the UTF-8 mode. PYTHONUTF8 environment variable
is ignored (even if use_environment=1). */
config->utf8_mode = 0;
config->coerce_c_locale = 0;
config->coerce_c_locale_warn = 0;

config->dev_mode = -1;
config->allocator = PYMEM_ALLOCATOR_NOT_SET;
#ifdef MS_WINDOWS
Expand Down Expand Up @@ -353,6 +362,7 @@ _PyPreConfig_Copy(_PyPreConfig *config, const _PyPreConfig *config2)
{
#define COPY_ATTR(ATTR) config->ATTR = config2->ATTR

COPY_ATTR(_config_init);
COPY_ATTR(parse_argv);
COPY_ATTR(isolated);
COPY_ATTR(use_environment);
Expand Down Expand Up @@ -393,6 +403,7 @@ _PyPreConfig_AsDict(const _PyPreConfig *config)
} \
} while (0)

SET_ITEM_INT(_config_init);
SET_ITEM_INT(parse_argv);
SET_ITEM_INT(isolated);
SET_ITEM_INT(use_environment);
Expand Down Expand Up @@ -452,7 +463,9 @@ _PyPreConfig_GetGlobalConfig(_PyPreConfig *config)

COPY_FLAG(isolated, Py_IsolatedFlag);
COPY_NOT_FLAG(use_environment, Py_IgnoreEnvironmentFlag);
COPY_FLAG(utf8_mode, Py_UTF8Mode);
if (Py_UTF8Mode > 0) {
config->utf8_mode = Py_UTF8Mode;
}
#ifdef MS_WINDOWS
COPY_FLAG(legacy_windows_fs_encoding, Py_LegacyWindowsFSEncodingFlag);
#endif
Expand Down

0 comments on commit 20e1e25

Please sign in to comment.