Skip to content

Commit

Permalink
Trac #34221: Backport PEP420 namespace package support from Cython 3
Browse files Browse the repository at this point in the history 
After #33011, downstream packages that use Cython and `cimport` Sage
modules need to activate PEP 420 namespace package support in Cython
using `with cython_namespace_package_support` (as is done in
https://github.com/sagemath/sage/blob/develop/src/setup.py#L106 for
sagelib).

This feature is standard in the upcoming Cython 3 (#29863). Until
downstream packages are updated, we can patch our Cython.

Upstream backport PR:
- cython/cython#4918

URL: https://trac.sagemath.org/34221
Reported by: mkoeppe
Ticket author(s): Matthias Koeppe
Reviewer(s): Marc Culler, John Palmieri
  • Loading branch information
Release Manager committed Aug 8, 2022
2 parents 12be2d9 + 48a9ba2 commit 5247961
Show file tree
Hide file tree
Showing 2 changed files with 288 additions and 1 deletion.
2 changes: 1 addition & 1 deletion build/pkgs/cython/package-version.txt
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0.29.32
0.29.32.p1
287 changes: 287 additions & 0 deletions build/pkgs/cython/patches/4918.patch
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,287 @@
From 0dade9353e06b052d0da5e512cdc2ce04061904a Mon Sep 17 00:00:00 2001
From: Matthias Koeppe <mkoeppe@math.ucdavis.edu>
Date: Sat, 23 Jul 2022 10:53:49 -0700
Subject: [PATCH 1/3] Add PEP420 namespace support

Backport of https://github.com/cython/cython/pull/2946 to 0.29.x
---
Cython/Compiler/Main.py | 44 +++++++++++++++----
Cython/Utils.py | 26 ++++++++---
runtests.py | 1 +
.../build/cythonize_pep420_namespace.srctree | 44 +++++++++++++++++++
4 files changed, 99 insertions(+), 16 deletions(-)
create mode 100644 tests/build/cythonize_pep420_namespace.srctree

diff --git a/Cython/Compiler/Main.py b/Cython/Compiler/Main.py
index dc4add541e..2be7a06a1c 100644
--- a/Cython/Compiler/Main.py
+++ b/Cython/Compiler/Main.py
@@ -801,32 +801,58 @@ def search_include_directories(dirs, qualified_name, suffix, pos, include=False)
else:
dirs = (Utils.find_root_package_dir(file_desc.filename),) + dirs

+ # search for dotted filename e.g. <dir>/foo.bar.pxd
dotted_filename = qualified_name
if suffix:
dotted_filename += suffix

+ for dirname in dirs:
+ path = os.path.join(dirname, dotted_filename)
+ if os.path.exists(path):
+ return path
+
+ # search for filename in package structure e.g. <dir>/foo/bar.pxd or <dir>/foo/bar/__init__.pxd
if not include:
+
names = qualified_name.split('.')
package_names = tuple(names[:-1])
module_name = names[-1]
module_filename = module_name + suffix
package_filename = "__init__" + suffix

- for dirname in dirs:
- path = os.path.join(dirname, dotted_filename)
- if os.path.exists(path):
- return path
-
- if not include:
- package_dir = Utils.check_package_dir(dirname, package_names)
+ # search for standard packages first - PEP420
+ namespace_dirs = []
+ for dirname in dirs:
+ package_dir, is_namespace = Utils.check_package_dir(dirname, package_names)
if package_dir is not None:
+
+ if is_namespace:
+ namespace_dirs.append(package_dir)
+ continue
+
+ # matches modules of the form: <dir>/foo/bar.pxd
path = os.path.join(package_dir, module_filename)
if os.path.exists(path):
return path
- path = os.path.join(package_dir, module_name,
- package_filename)
+
+ # matches modules of the form: <dir>/foo/bar/__init__.pxd
+ path = os.path.join(package_dir, module_name, package_filename)
if os.path.exists(path):
return path
+
+ # search for namespaces second - PEP420
+ for package_dir in namespace_dirs:
+
+ # matches modules of the form: <dir>/foo/bar.pxd
+ path = os.path.join(package_dir, module_filename)
+ if os.path.exists(path):
+ return path
+
+ # matches modules of the form: <dir>/foo/bar/__init__.pxd
+ path = os.path.join(package_dir, module_name, package_filename)
+ if os.path.exists(path):
+ return path
+
return None


diff --git a/Cython/Utils.py b/Cython/Utils.py
index d59d67d78b..305ebf8412 100644
--- a/Cython/Utils.py
+++ b/Cython/Utils.py
@@ -134,16 +134,21 @@ def find_root_package_dir(file_path):
else:
return dir

+
@cached_function
-def check_package_dir(dir, package_names):
+def check_package_dir(dir_path, package_names):
+ namespace = True
for dirname in package_names:
- dir = os.path.join(dir, dirname)
- if not is_package_dir(dir):
- return None
- return dir
+ dir_path = os.path.join(dir_path, dirname)
+ has_init = contains_init(dir_path)
+ if not namespace and not has_init:
+ return None, False
+ elif has_init:
+ namespace = False
+ return dir_path, namespace

-@cached_function
-def is_package_dir(dir_path):
+
+def contains_init(dir_path):
for filename in ("__init__.py",
"__init__.pyc",
"__init__.pyx",
@@ -152,6 +157,13 @@ def is_package_dir(dir_path):
if path_exists(path):
return 1

+
+@cached_function
+def is_package_dir(dir_path):
+ if contains_init(dir_path):
+ return 1
+
+
@cached_function
def path_exists(path):
# try on the filesystem first
diff --git a/runtests.py b/runtests.py
index 91a0dd2570..7d04463846 100755
--- a/runtests.py
+++ b/runtests.py
@@ -415,6 +415,7 @@ def get_openmp_compiler_flags(language):
'run.special_methods_T561_py2'
]),
(3,3) : (operator.lt, lambda x: x in ['build.package_compilation',
+ 'build.cythonize_pep420_namespace',
'run.yield_from_py33',
'pyximport.pyximport_namespace',
]),
diff --git a/tests/build/cythonize_pep420_namespace.srctree b/tests/build/cythonize_pep420_namespace.srctree
new file mode 100644
index 0000000000..6a031e4170
--- /dev/null
+++ b/tests/build/cythonize_pep420_namespace.srctree
@@ -0,0 +1,44 @@
+PYTHON setup.py build_ext --inplace
+PYTHON -c "import runner"
+
+######## setup.py ########
+
+from Cython.Build.Dependencies import cythonize
+
+from distutils.core import setup, Extension
+
+setup(
+ ext_modules=cythonize([
+ Extension("nsp.m1.a", ["nsp/m1/a.pyx"]),
+ Extension("nsp.m2.b", ["nsp/m2/b.pyx"])
+ ]),
+)
+
+######## nsp/m1/__init__.py ########
+
+######## nsp/m1/a.pyx ########
+
+cdef class A:
+ pass
+
+######## nsp/m1/a.pxd ########
+
+cdef class A:
+ pass
+
+######## nsp/m2/__init__.py ########
+
+######## nsp/m2/b.pyx ########
+
+from nsp.m1.a cimport A
+
+cdef class B(A):
+ pass
+
+######## runner.py ########
+
+from nsp.m1.a import A
+from nsp.m2.b import B
+
+a = A()
+b = B()

From fcd3e7bd1351a0964704f2921ae33b4b57250668 Mon Sep 17 00:00:00 2001
From: Fedor Alekseev <feodor.alexeev@gmail.com>
Date: Mon, 9 Nov 2020 14:34:01 +0300
Subject: [PATCH 2/3] Support namespace packages inside regular packages

---
Cython/Utils.py | 4 +---
tests/build/cythonize_pep420_namespace.srctree | 17 ++++++++++++++++-
2 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/Cython/Utils.py b/Cython/Utils.py
index 305ebf8412..f21d5cddba 100644
--- a/Cython/Utils.py
+++ b/Cython/Utils.py
@@ -141,9 +141,7 @@ def check_package_dir(dir_path, package_names):
for dirname in package_names:
dir_path = os.path.join(dir_path, dirname)
has_init = contains_init(dir_path)
- if not namespace and not has_init:
- return None, False
- elif has_init:
+ if has_init:
namespace = False
return dir_path, namespace

diff --git a/tests/build/cythonize_pep420_namespace.srctree b/tests/build/cythonize_pep420_namespace.srctree
index 6a031e4170..04013a3004 100644
--- a/tests/build/cythonize_pep420_namespace.srctree
+++ b/tests/build/cythonize_pep420_namespace.srctree
@@ -10,7 +10,8 @@ from distutils.core import setup, Extension
setup(
ext_modules=cythonize([
Extension("nsp.m1.a", ["nsp/m1/a.pyx"]),
- Extension("nsp.m2.b", ["nsp/m2/b.pyx"])
+ Extension("nsp.m2.b", ["nsp/m2/b.pyx"]),
+ Extension("nsp.m3.c.d", ["nsp/m3/c/d.pyx"])
]),
)

@@ -31,14 +32,28 @@ cdef class A:
######## nsp/m2/b.pyx ########

from nsp.m1.a cimport A
+from nsp.m3.c.d cimport D

cdef class B(A):
pass

+######## nsp/m3/__init__.py ########
+
+######## nsp/m3/c/d.pyx ########
+
+cdef class D:
+ pass
+
+######## nsp/m3/c/d.pxd ########
+
+cdef class D:
+ pass
+
######## runner.py ########

from nsp.m1.a import A
from nsp.m2.b import B
+from nsp.m3.c.d import D

a = A()
b = B()

From 9d61b95a6a71a4c88a51ab8f30d5b3a8b93998b8 Mon Sep 17 00:00:00 2001
From: scoder <stefan_ml@behnel.de>
Date: Sat, 14 Nov 2020 09:42:09 +0100
Subject: [PATCH 3/3] Anticipate future changes.

---
tests/build/cythonize_pep420_namespace.srctree | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/build/cythonize_pep420_namespace.srctree b/tests/build/cythonize_pep420_namespace.srctree
index 04013a3004..99649376a3 100644
--- a/tests/build/cythonize_pep420_namespace.srctree
+++ b/tests/build/cythonize_pep420_namespace.srctree
@@ -11,7 +11,7 @@ setup(
ext_modules=cythonize([
Extension("nsp.m1.a", ["nsp/m1/a.pyx"]),
Extension("nsp.m2.b", ["nsp/m2/b.pyx"]),
- Extension("nsp.m3.c.d", ["nsp/m3/c/d.pyx"])
+ Extension("nsp.m3.c.d", ["nsp/m3/c/d.pyx"]),
]),
)

0 comments on commit 5247961

Please sign in to comment.