-
-
Notifications
You must be signed in to change notification settings - Fork 30k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
bpo-41100: ctypes fixes for arm64 Mac OS #21249
Changes from 9 commits
ff883bc
36b35bc
bacf128
971ef00
da30587
80f852b
aeac14b
62fe20a
e160a59
bca75fd
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
setup.py: probe libffi for ffi_closure_alloc and ffi_prep_cif_var |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,6 +1,8 @@ | ||
#include "Python.h" | ||
#include "frameobject.h" | ||
|
||
#include <stdbool.h> | ||
|
||
#include <ffi.h> | ||
#ifdef MS_WIN32 | ||
#include <windows.h> | ||
|
@@ -18,7 +20,7 @@ CThunkObject_dealloc(PyObject *myself) | |
Py_XDECREF(self->callable); | ||
Py_XDECREF(self->restype); | ||
if (self->pcl_write) | ||
ffi_closure_free(self->pcl_write); | ||
Py_ffi_closure_free(self->pcl_write); | ||
PyObject_GC_Del(self); | ||
} | ||
|
||
|
@@ -361,8 +363,7 @@ CThunkObject *_ctypes_alloc_callback(PyObject *callable, | |
|
||
assert(CThunk_CheckExact((PyObject *)p)); | ||
|
||
p->pcl_write = ffi_closure_alloc(sizeof(ffi_closure), | ||
&p->pcl_exec); | ||
p->pcl_write = Py_ffi_closure_alloc(sizeof(ffi_closure), &p->pcl_exec); | ||
if (p->pcl_write == NULL) { | ||
PyErr_NoMemory(); | ||
goto error; | ||
|
@@ -408,13 +409,29 @@ CThunkObject *_ctypes_alloc_callback(PyObject *callable, | |
"ffi_prep_cif failed with %d", result); | ||
goto error; | ||
} | ||
#if defined(X86_DARWIN) || defined(POWERPC_DARWIN) | ||
result = ffi_prep_closure(p->pcl_write, &p->cif, closure_fcn, p); | ||
#if HAVE_FFI_PREP_CLOSURE_LOC | ||
# if USING_APPLE_OS_LIBFFI | ||
# define HAVE_FFI_PREP_CLOSURE_LOC_RUNTIME __builtin_available(macos 11, ios 13, watchos 6, tvos 13, *) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This use of __builtin_available should be safe, it is within a guard of HAVE_FFI_PREP_CLOSURE_LOC and USING_APPLE_OS_LIBFFI: The system version of libffi does not have ffi_prep_closure_loc in the 10.9 SDK (it is introduced in 10.15). There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Why test for macOS 11 and not 10.15? According to the header files this API is introduced in 10.15. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. you're right it should be 10.15 |
||
# else | ||
# define HAVE_FFI_PREP_CLOSURE_LOC_RUNTIME true | ||
# endif | ||
if (HAVE_FFI_PREP_CLOSURE_LOC_RUNTIME) { | ||
result = ffi_prep_closure_loc(p->pcl_write, &p->cif, closure_fcn, | ||
p, | ||
p->pcl_exec); | ||
} else | ||
#endif | ||
{ | ||
#if USING_APPLE_OS_LIBFFI && defined(__arm64__) | ||
PyErr_Format(PyExc_NotImplementedError, "ffi_prep_closure_loc() is missing"); | ||
goto error; | ||
#else | ||
result = ffi_prep_closure_loc(p->pcl_write, &p->cif, closure_fcn, | ||
p, | ||
p->pcl_exec); | ||
#pragma clang diagnostic push | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We should avoid There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. fixed |
||
#pragma clang diagnostic ignored "-Wdeprecated-declarations" | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. ditto |
||
result = ffi_prep_closure(p->pcl_write, &p->cif, closure_fcn, p); | ||
#pragma clang diagnostic pop | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. ditto |
||
#endif | ||
} | ||
if (result != FFI_OK) { | ||
PyErr_Format(PyExc_RuntimeError, | ||
"ffi_prep_closure failed with %d", result); | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -57,6 +57,8 @@ | |
#include "Python.h" | ||
#include "structmember.h" // PyMemberDef | ||
|
||
#include <stdbool.h> | ||
|
||
#ifdef MS_WIN32 | ||
#include <windows.h> | ||
#include <tchar.h> | ||
|
@@ -812,7 +814,8 @@ static int _call_function_pointer(int flags, | |
ffi_type **atypes, | ||
ffi_type *restype, | ||
void *resmem, | ||
int argcount) | ||
int argcount, | ||
int argtypecount) | ||
{ | ||
PyThreadState *_save = NULL; /* For Py_BLOCK_THREADS and Py_UNBLOCK_THREADS */ | ||
PyObject *error_object = NULL; | ||
|
@@ -835,14 +838,63 @@ static int _call_function_pointer(int flags, | |
if ((flags & FUNCFLAG_CDECL) == 0) | ||
cc = FFI_STDCALL; | ||
#endif | ||
if (FFI_OK != ffi_prep_cif(&cif, | ||
cc, | ||
argcount, | ||
restype, | ||
atypes)) { | ||
PyErr_SetString(PyExc_RuntimeError, | ||
"ffi_prep_cif failed"); | ||
return -1; | ||
|
||
# if USING_APPLE_OS_LIBFFI | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Proposed change: #if USING_APPLE_OS_LIBFFI && HAVE_FFI_PREP_CIF_VAR This way the __builtin_available is not used when building on older macOS versions. |
||
# define HAVE_FFI_PREP_CIF_VAR_RUNTIME __builtin_available(macos 11, ios 13, watchos 6, tvos 13, *) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Likewise: the headers claim this _var variant is present in macOS 10.15. |
||
# elif HAVE_FFI_PREP_CIF_VAR | ||
# define HAVE_FFI_PREP_CIF_VAR_RUNTIME true | ||
# else | ||
# define HAVE_FFI_PREP_CIF_VAR_RUNTIME false | ||
# endif | ||
|
||
/* Even on Apple-arm64 the calling convention for variadic functions conincides | ||
* with the standard calling convention in the case that the function called | ||
* only with its fixed arguments. Thus, we do not need a special flag to be | ||
* set on variadic functions. We treat a function as variadic if it is called | ||
* with a nonzero number of variadic arguments */ | ||
bool is_variadic = (argtypecount != 0 && argcount > argtypecount); | ||
(void) is_variadic; | ||
|
||
#if defined(__APPLE__) && defined(__arm64__) | ||
if (is_variadic) { | ||
if (HAVE_FFI_PREP_CIF_VAR_RUNTIME) { | ||
} else { | ||
PyErr_SetString(PyExc_NotImplementedError, "ffi_prep_cif_var() is missing"); | ||
return -1; | ||
} | ||
} | ||
#endif | ||
|
||
bool called_ffi_prep_cif_var = false; | ||
|
||
#if HAVE_FFI_PREP_CIF_VAR | ||
if (is_variadic) { | ||
if (HAVE_FFI_PREP_CIF_VAR_RUNTIME) { | ||
if (FFI_OK != ffi_prep_cif_var(&cif, | ||
cc, | ||
argtypecount, | ||
argcount, | ||
restype, | ||
atypes)) { | ||
PyErr_SetString(PyExc_RuntimeError, | ||
"ffi_prep_cif_var failed"); | ||
return -1; | ||
} | ||
called_ffi_prep_cif_var = true; | ||
} | ||
} | ||
#endif | ||
|
||
if (!called_ffi_prep_cif_var) { | ||
if (FFI_OK != ffi_prep_cif(&cif, | ||
cc, | ||
argcount, | ||
restype, | ||
atypes)) { | ||
PyErr_SetString(PyExc_RuntimeError, | ||
"ffi_prep_cif failed"); | ||
return -1; | ||
} | ||
} | ||
|
||
if (flags & (FUNCFLAG_USE_ERRNO | FUNCFLAG_USE_LASTERROR)) { | ||
|
@@ -1212,9 +1264,8 @@ PyObject *_ctypes_callproc(PPROC pProc, | |
|
||
if (-1 == _call_function_pointer(flags, pProc, avalues, atypes, | ||
rtype, resbuf, | ||
Py_SAFE_DOWNCAST(argcount, | ||
Py_ssize_t, | ||
int))) | ||
Py_SAFE_DOWNCAST(argcount, Py_ssize_t, int), | ||
Py_SAFE_DOWNCAST(argtype_count, Py_ssize_t, int))) | ||
goto cleanup; | ||
|
||
#ifdef WORDS_BIGENDIAN | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -89,16 +89,27 @@ static void more_core(void) | |
/******************************************************************/ | ||
|
||
/* put the item back into the free list */ | ||
void ffi_closure_free(void *p) | ||
void Py_ffi_closure_free(void *p) | ||
{ | ||
#if USING_APPLE_OS_LIBFFI | ||
if (__builtin_available(macos 11, ios 13, watchos 6, tvos 13, *)) { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I'm afraid this needs a configure test, ffi_closure_free is not available at all in older SDKs. |
||
ffi_closure_free(p); | ||
return; | ||
} | ||
#endif | ||
ITEM *item = (ITEM *)p; | ||
item->next = free_list; | ||
free_list = item; | ||
} | ||
|
||
/* return one item from the free list, allocating more if needed */ | ||
void *ffi_closure_alloc(size_t ignored, void** codeloc) | ||
void *Py_ffi_closure_alloc(size_t size, void** codeloc) | ||
{ | ||
#if USING_APPLE_OS_LIBFFI | ||
if (__builtin_available(macos 11, ios 13, watchos 6, tvos 13, *)) { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. same here |
||
return ffi_closure_alloc(size, codeloc); | ||
} | ||
#endif | ||
ITEM *item; | ||
if (!free_list) | ||
more_core(); | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -229,6 +229,13 @@ def macosx_sdk_specified(): | |
macosx_sdk_root() | ||
return MACOS_SDK_SPECIFIED | ||
|
||
def is_macosx_at_least(vers): | ||
if MACOS: | ||
dep_target = sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET') | ||
if dep_target: | ||
return tuple(map(int, dep_target.split('.'))) >= vers | ||
return False | ||
|
||
|
||
def is_macosx_sdk_path(path): | ||
""" | ||
|
@@ -239,6 +246,13 @@ def is_macosx_sdk_path(path): | |
or path.startswith('/Library/') ) | ||
|
||
|
||
def grep_headers_for(function, headers): | ||
for header in headers: | ||
with open(header, 'r') as f: | ||
if function in f.read(): | ||
return True | ||
return False | ||
|
||
def find_file(filename, std_dirs, paths): | ||
"""Searches for the directory where a given file is located, | ||
and returns a possibly-empty list of additional directories, or None | ||
|
@@ -2136,7 +2150,13 @@ def configure_ctypes(self, ext): | |
|
||
def detect_ctypes(self): | ||
# Thomas Heller's _ctypes module | ||
self.use_system_libffi = False | ||
|
||
if (not sysconfig.get_config_var("LIBFFI_INCLUDEDIR") and MACOS and | ||
(is_macosx_at_least((10,15)) or '-arch arm64' in sysconfig.get_config_var("CFLAGS"))): | ||
self.use_system_libffi = True | ||
else: | ||
self.use_system_libffi = '--with-system-ffi' in sysconfig.get_config_var("CONFIG_ARGS") | ||
|
||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I'd prefer to just drop "use_system_libffi" and unconditionally use the system headers. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. ok |
||
include_dirs = [] | ||
extra_compile_args = ['-DPy_BUILD_CORE_MODULE'] | ||
extra_link_args = [] | ||
|
@@ -2149,6 +2169,7 @@ def detect_ctypes(self): | |
|
||
if MACOS: | ||
sources.append('_ctypes/malloc_closure.c') | ||
extra_compile_args.append('-DUSING_MALLOC_CLOSURE_DOT_C=1') | ||
sources.append('_ctypes/darwin/dlfcn_simple.c') | ||
extra_compile_args.append('-DMACOSX') | ||
include_dirs.append('_ctypes/darwin') | ||
|
@@ -2183,31 +2204,45 @@ def detect_ctypes(self): | |
sources=['_ctypes/_ctypes_test.c'], | ||
libraries=['m'])) | ||
|
||
ffi_inc = sysconfig.get_config_var("LIBFFI_INCLUDEDIR") | ||
ffi_lib = None | ||
|
||
ffi_inc_dirs = self.inc_dirs.copy() | ||
if MACOS: | ||
if '--with-system-ffi' not in sysconfig.get_config_var("CONFIG_ARGS"): | ||
if not self.use_system_libffi: | ||
return | ||
# OS X 10.5 comes with libffi.dylib; the include files are | ||
# in /usr/include/ffi | ||
ffi_inc_dirs.append('/usr/include/ffi') | ||
|
||
ffi_inc = [sysconfig.get_config_var("LIBFFI_INCLUDEDIR")] | ||
if not ffi_inc or ffi_inc[0] == '': | ||
ffi_inc = find_file('ffi.h', [], ffi_inc_dirs) | ||
if ffi_inc is not None: | ||
ffi_h = ffi_inc[0] + '/ffi.h' | ||
ext.extra_compile_args.append("-DUSING_APPLE_OS_LIBFFI=1") | ||
ffi_in_sdk = os.path.join(macosx_sdk_root(), "usr/include/ffi") | ||
Comment on lines
+2153
to
+2154
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Please bear in mind that Python may be built against an external libffi on macOS that is not the OS-provided version. In MacPorts we build against our port of libffi 3.3 on all OS versions. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. See previous comment by @ronaldoussoren who prefers to unconditionally use the system's libffi. I updated this PR accordingly. Is there any reason not to use the system libffi anymore? I tested going back to 10.10 i think for the other PR adding availability checks. What's the need to support alternative libffi these days? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Same reasons as any other library: the OS often ships an older version which naturally lacks the features and bug fixes of later versions. |
||
if os.path.exists(ffi_in_sdk): | ||
ffi_inc = ffi_in_sdk | ||
ffi_lib = 'ffi' | ||
else: | ||
# OS X 10.5 comes with libffi.dylib; the include files are | ||
# in /usr/include/ffi | ||
ffi_inc_dirs.append('/usr/include/ffi') | ||
|
||
if not ffi_inc: | ||
found = find_file('ffi.h', [], ffi_inc_dirs) | ||
if found: | ||
ffi_inc = found[0] | ||
if ffi_inc: | ||
ffi_h = ffi_inc + '/ffi.h' | ||
if not os.path.exists(ffi_h): | ||
ffi_inc = None | ||
print('Header file {} does not exist'.format(ffi_h)) | ||
ffi_lib = None | ||
if ffi_inc is not None: | ||
if ffi_lib is None and ffi_inc: | ||
for lib_name in ('ffi', 'ffi_pic'): | ||
if (self.compiler.find_library_file(self.lib_dirs, lib_name)): | ||
ffi_lib = lib_name | ||
break | ||
|
||
if ffi_inc and ffi_lib: | ||
ext.include_dirs.extend(ffi_inc) | ||
ffi_headers = glob(os.path.join(ffi_inc, '*.h')) | ||
if grep_headers_for('ffi_prep_cif_var', ffi_headers): | ||
ext.extra_compile_args.append("-DHAVE_FFI_PREP_CIF_VAR=1") | ||
if grep_headers_for('ffi_prep_closure_loc', ffi_headers): | ||
ext.extra_compile_args.append("-DHAVE_FFI_PREP_CLOSURE_LOC=1") | ||
ext.include_dirs.append(ffi_inc) | ||
ext.libraries.append(ffi_lib) | ||
self.use_system_libffi = True | ||
|
||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Unnecessary removal of a blank line (not important)