From 5cf3be2f11eb4c4a530f3dcac7eafb56a9049777 Mon Sep 17 00:00:00 2001 From: Ray Speth Date: Sun, 7 Aug 2022 23:20:37 -0400 Subject: [PATCH 01/26] Minimal implementation of ExtensibleRate --- include/cantera/base/Delegator.h | 22 ++++++++ .../cantera/kinetics/ReactionRateDelegator.h | 53 +++++++++++++++++++ interfaces/cython/cantera/delegator.pxd | 2 + interfaces/cython/cantera/delegator.pyx | 18 +++++++ interfaces/cython/cantera/kinetics.pyx | 4 +- interfaces/cython/cantera/reaction.pxd | 8 +++ interfaces/cython/cantera/reaction.pyx | 15 ++++++ src/kinetics/ReactionRateFactory.cpp | 5 ++ test/python/test_reaction.py | 35 ++++++++++++ 9 files changed, 160 insertions(+), 2 deletions(-) create mode 100644 include/cantera/kinetics/ReactionRateDelegator.h diff --git a/include/cantera/base/Delegator.h b/include/cantera/base/Delegator.h index 8834853f2d..e1c8cd7f0d 100644 --- a/include/cantera/base/Delegator.h +++ b/include/cantera/base/Delegator.h @@ -189,6 +189,17 @@ class Delegator *m_funcs_v_dp_dp_dp[name] = makeDelegate(func, when, *m_funcs_v_dp_dp_dp[name]); } + //! set delegates for member functions with the signature `double()` + void setDelegate(const std::string& name, const std::function& func, + const std::string& when) + { + if (!m_funcs_d.count(name)) { + throw NotImplementedError("Delegator::setDelegate", + "for function named '{}' with signature 'double()'.", name); + } + *m_funcs_d[name] = makeDelegate(func, when, m_base_d[name]); + } + //! Set delegates for member functions with the signature `string(size_t)` void setDelegate(const std::string& name, const std::function& func, @@ -278,6 +289,14 @@ class Delegator m_funcs_v_dp_dp_dp[name] = ⌖ } + //! Install a function with the signature `void()` as being delegatable + void install(const std::string& name, std::function& target, + const std::function& func) + { + target = func; + m_funcs_d[name] = ⌖ + } + //! Install a function with the signature `string(size_t)` as being delegatable void install(const std::string& name, std::function& target, @@ -413,6 +432,9 @@ class Delegator std::function, double*, double*, double*)>*> m_funcs_v_dp_dp_dp; // Delegates with a return value + std::map> m_base_d; + std::map*> m_funcs_d; + std::map> m_base_s_sz; std::map newMultiRate() const override { + return unique_ptr( + new MultiRate); + } + + virtual const std::string type() const override { + return "ReactionRateDelegator"; + } + + // Delegatable methods + + double evalFromStruct(const ArrheniusData& shared_data) { + return m_evalFromStruct(); + } + +private: + std::function m_evalFromStruct; +}; + +} + +#endif diff --git a/interfaces/cython/cantera/delegator.pxd b/interfaces/cython/cantera/delegator.pxd index 75c003914c..1e539f7467 100644 --- a/interfaces/cython/cantera/delegator.pxd +++ b/interfaces/cython/cantera/delegator.pxd @@ -32,6 +32,7 @@ cdef extern from "cantera/base/Delegator.h" namespace "Cantera": void setDelegate(string&, function[void(size_array1, double, double*)], string&) except +translate_exception void setDelegate(string&, function[void(size_array2, double, double*, double*)], string&) except +translate_exception void setDelegate(string&, function[void(size_array3, double*, double*, double*)], string&) except +translate_exception + void setDelegate(string&, function[int(double&)], string&) except +translate_exception void setDelegate(string&, function[int(string&, size_t)], string&) except +translate_exception void setDelegate(string&, function[int(size_t&, string&)], string&) except +translate_exception @@ -50,6 +51,7 @@ cdef extern from "cantera/cython/funcWrapper.h": PyObject*, void(PyFuncInfo&, size_array2, double, double*, double*)) cdef function[void(size_array3, double*, double*, double*)] pyOverride( PyObject*, void(PyFuncInfo&, size_array3, double*, double*, double*)) + cdef function[int(double&)] pyOverride(PyObject*, int(PyFuncInfo&, double&)) cdef function[int(string&, size_t)] pyOverride(PyObject*, int(PyFuncInfo&, string&, size_t)) cdef function[int(size_t&, const string&)] pyOverride( PyObject*, int(PyFuncInfo&, size_t&, const string&)) diff --git a/interfaces/cython/cantera/delegator.pyx b/interfaces/cython/cantera/delegator.pyx index 8126375c19..12736aafe5 100644 --- a/interfaces/cython/cantera/delegator.pyx +++ b/interfaces/cython/cantera/delegator.pyx @@ -138,6 +138,21 @@ cdef void callback_v_dp_dp_dp(PyFuncInfo& funcInfo, funcInfo.setExceptionType(exc_type) funcInfo.setExceptionValue(exc_value) +# Wrapper for functions of type double() +cdef int callback_d(PyFuncInfo& funcInfo, double& out): + try: + ret = (funcInfo.func())() + if ret is None: + return 0 + else: + (&out)[0] = ret + return 1 + except BaseException as e: + exc_type, exc_value = sys.exc_info()[:2] + funcInfo.setExceptionType(exc_type) + funcInfo.setExceptionValue(exc_value) + return -1 + # Wrapper for functions of type string(size_t) cdef int callback_s_sz(PyFuncInfo& funcInfo, string& out, size_t arg): try: @@ -271,6 +286,9 @@ cdef int assign_delegates(obj, CxxDelegator* delegator) except -1: elif callback == 'void(double*,double*,double*)': delegator.setDelegate(cxx_name, pyOverride(method, callback_v_dp_dp_dp), cxx_when) + elif callback == 'double()': + delegator.setDelegate(cxx_name, + pyOverride(method, callback_d), cxx_when) elif callback == 'string(size_t)': delegator.setDelegate(cxx_name, pyOverride(method, callback_s_sz), cxx_when) diff --git a/interfaces/cython/cantera/kinetics.pyx b/interfaces/cython/cantera/kinetics.pyx index 58f3dc4c5f..f881931484 100644 --- a/interfaces/cython/cantera/kinetics.pyx +++ b/interfaces/cython/cantera/kinetics.pyx @@ -188,8 +188,8 @@ cdef class Kinetics(_SolutionBase): def add_reaction(self, Reaction rxn): """ Add a new reaction to this phase. """ self.kinetics.addReaction(rxn._reaction) - if isinstance(rxn.rate, CustomRate): - # prevent garbage collection + if isinstance(rxn.rate, (CustomRate, ExtensibleRate)): + # prevent premature garbage collection self._custom_rates.append(rxn.rate) def multiplier(self, int i_reaction): diff --git a/interfaces/cython/cantera/reaction.pxd b/interfaces/cython/cantera/reaction.pxd index 0c0a836b50..9770198b8e 100644 --- a/interfaces/cython/cantera/reaction.pxd +++ b/interfaces/cython/cantera/reaction.pxd @@ -188,6 +188,11 @@ cdef extern from "cantera/kinetics/Custom.h" namespace "Cantera": void setRateFunction(shared_ptr[CxxFunc1]) except +translate_exception +cdef extern from "cantera/kinetics/ReactionRateDelegator.h" namespace "Cantera": + cdef cppclass CxxReactionRateDelegator "Cantera::ReactionRateDelegator" (CxxReactionRate): + CxxReactionRateDelegator() + + cdef extern from "cantera/kinetics/InterfaceRate.h" namespace "Cantera": cdef cppclass CxxInterfaceRateBase "Cantera::InterfaceRateBase": void getCoverageDependencies(CxxAnyMap) @@ -247,6 +252,9 @@ cdef class CustomRate(ReactionRate): cdef CxxCustomFunc1Rate* cxx_object(self) cdef Func1 _rate_func # prevent premature garbage collection +cdef class ExtensibleRate(ReactionRate): + pass + cdef class InterfaceRateBase(ArrheniusRateBase): cdef CxxInterfaceRateBase* interface diff --git a/interfaces/cython/cantera/reaction.pyx b/interfaces/cython/cantera/reaction.pyx index f7da168d24..0d9496d6a5 100644 --- a/interfaces/cython/cantera/reaction.pyx +++ b/interfaces/cython/cantera/reaction.pyx @@ -9,6 +9,7 @@ from cython.operator cimport dereference as deref from .kinetics cimport Kinetics from ._utils cimport * from .units cimport * +from .delegator cimport * # dictionary to store reaction rate classes cdef dict _reaction_rate_class_registry = {} @@ -709,6 +710,20 @@ cdef class CustomRate(ReactionRate): self.cxx_object().setRateFunction(self._rate_func._func) +cdef class ExtensibleRate(ReactionRate): + _reaction_rate_type = "extensible" + + delegatable_methods = { + "eval": ("evalFromStruct", "double()") + } + def __cinit__(self, *args, init=True, **kwargs): + if init: + self._rate.reset(new CxxReactionRateDelegator()) + self.set_cxx_object() + + def __init__(self, *args, **kwargs): + assign_delegates(self, dynamic_cast[CxxDelegatorPtr](self.rate)) + super().__init__(*args, **kwargs) cdef class InterfaceRateBase(ArrheniusRateBase): """ diff --git a/src/kinetics/ReactionRateFactory.cpp b/src/kinetics/ReactionRateFactory.cpp index 3cd354b368..2c5dc87872 100644 --- a/src/kinetics/ReactionRateFactory.cpp +++ b/src/kinetics/ReactionRateFactory.cpp @@ -16,6 +16,7 @@ #include "cantera/kinetics/InterfaceRate.h" #include "cantera/kinetics/PlogRate.h" #include "cantera/kinetics/TwoTempPlasmaRate.h" +#include "cantera/kinetics/ReactionRateDelegator.h" namespace Cantera { @@ -98,6 +99,10 @@ ReactionRateFactory::ReactionRateFactory() reg("sticking-Blowers-Masel", [](const AnyMap& node, const UnitStack& rate_units) { return new StickingBlowersMaselRate(node, rate_units); }); + + reg("extensible", [](const AnyMap& node, const UnitStack& rate_units) { + return new ReactionRateDelegator(node, rate_units); + }); } shared_ptr newReactionRate(const std::string& type) diff --git a/test/python/test_reaction.py b/test/python/test_reaction.py index c05896c6fc..9afed0020f 100644 --- a/test/python/test_reaction.py +++ b/test/python/test_reaction.py @@ -1498,6 +1498,41 @@ def func(T): assert (gas.forward_rate_constants == gas.T).all() +class TestExtensible(ReactionTests, utilities.CanteraTest): + # test Extensible reaction rate + class UserRate(ct.ExtensibleRate): + def __init__(self, soln): + super().__init__() + self.soln = soln + + def replace_eval(self): + T = self.soln.T + return 38.7 * T**2.7 * exp(-3150.15428/T) + + # probe O + H2 <=> H + OH + _rate_cls = UserRate + _equation = "H2 + O <=> H + OH" + _index = 0 + _rate_type = "extensible" + _yaml = None + + def setUp(self): + super().setUp() + self._rate_obj = self.UserRate(self.soln) + + def test_no_rate(self): + pytest.skip("ExtensibleRate does not support 'empty' rates") + + def from_yaml(self): + pytest.skip("ExtensibleRate does not support YAML") + + def from_rate(self, rate): + pytest.skip("ExtensibleRate does not support dict-based instantiation") + + def test_roundtrip(self): + pytest.skip("ExtensibleRate does not support roundtrip conversion") + + class InterfaceReactionTests(ReactionTests): # test suite for surface reaction expressions From ebb20efaec0cea68ec4250d81929e4a94f57e265 Mon Sep 17 00:00:00 2001 From: Ray Speth Date: Thu, 11 Aug 2022 17:48:52 -0400 Subject: [PATCH 02/26] Pass an arbitrary argument to user rate function --- include/cantera/base/Delegator.h | 23 ++++++++++--------- .../cantera/kinetics/ReactionRateDelegator.h | 9 +++++--- interfaces/cython/cantera/delegator.pxd | 4 ++-- interfaces/cython/cantera/delegator.pyx | 11 +++++---- interfaces/cython/cantera/reaction.pyx | 2 +- test/python/test_reaction.py | 9 ++------ 6 files changed, 29 insertions(+), 29 deletions(-) diff --git a/include/cantera/base/Delegator.h b/include/cantera/base/Delegator.h index e1c8cd7f0d..964117c1f1 100644 --- a/include/cantera/base/Delegator.h +++ b/include/cantera/base/Delegator.h @@ -189,15 +189,16 @@ class Delegator *m_funcs_v_dp_dp_dp[name] = makeDelegate(func, when, *m_funcs_v_dp_dp_dp[name]); } - //! set delegates for member functions with the signature `double()` - void setDelegate(const std::string& name, const std::function& func, + //! set delegates for member functions with the signature `double(void*)` + void setDelegate(const std::string& name, + const std::function& func, const std::string& when) { - if (!m_funcs_d.count(name)) { + if (!m_funcs_d_vp.count(name)) { throw NotImplementedError("Delegator::setDelegate", - "for function named '{}' with signature 'double()'.", name); + "for function named '{}' with signature 'double(void*)'.", name); } - *m_funcs_d[name] = makeDelegate(func, when, m_base_d[name]); + *m_funcs_d_vp[name] = makeDelegate(func, when, m_base_d_vp[name]); } //! Set delegates for member functions with the signature `string(size_t)` @@ -289,12 +290,12 @@ class Delegator m_funcs_v_dp_dp_dp[name] = ⌖ } - //! Install a function with the signature `void()` as being delegatable - void install(const std::string& name, std::function& target, - const std::function& func) + //! Install a function with the signature `double(void*)` as being delegatable + void install(const std::string& name, std::function& target, + const std::function& func) { target = func; - m_funcs_d[name] = ⌖ + m_funcs_d_vp[name] = ⌖ } //! Install a function with the signature `string(size_t)` as being delegatable @@ -432,8 +433,8 @@ class Delegator std::function, double*, double*, double*)>*> m_funcs_v_dp_dp_dp; // Delegates with a return value - std::map> m_base_d; - std::map*> m_funcs_d; + std::map> m_base_d_vp; + std::map*> m_funcs_d_vp; std::map> m_base_s_sz; diff --git a/include/cantera/kinetics/ReactionRateDelegator.h b/include/cantera/kinetics/ReactionRateDelegator.h index 22ae67e2e8..1fb9ce27f9 100644 --- a/include/cantera/kinetics/ReactionRateDelegator.h +++ b/include/cantera/kinetics/ReactionRateDelegator.h @@ -18,7 +18,7 @@ class ReactionRateDelegator : public Delegator, public ReactionRate public: ReactionRateDelegator() { install("evalFromStruct", m_evalFromStruct, - [this]() { + [](void*) { throw NotImplementedError("ReactionRateDelegator::evalFromStruct"); return 0.0; // necessary to set lambda's function signature } @@ -41,11 +41,14 @@ class ReactionRateDelegator : public Delegator, public ReactionRate // Delegatable methods double evalFromStruct(const ArrheniusData& shared_data) { - return m_evalFromStruct(); + // @TODO: replace passing pointer to temperature with a language-specific + // wrapper of the ReactionData object + double T = shared_data.temperature; + return m_evalFromStruct(&T); } private: - std::function m_evalFromStruct; + std::function m_evalFromStruct; }; } diff --git a/interfaces/cython/cantera/delegator.pxd b/interfaces/cython/cantera/delegator.pxd index 1e539f7467..afbb62b254 100644 --- a/interfaces/cython/cantera/delegator.pxd +++ b/interfaces/cython/cantera/delegator.pxd @@ -32,7 +32,7 @@ cdef extern from "cantera/base/Delegator.h" namespace "Cantera": void setDelegate(string&, function[void(size_array1, double, double*)], string&) except +translate_exception void setDelegate(string&, function[void(size_array2, double, double*, double*)], string&) except +translate_exception void setDelegate(string&, function[void(size_array3, double*, double*, double*)], string&) except +translate_exception - void setDelegate(string&, function[int(double&)], string&) except +translate_exception + void setDelegate(string&, function[int(double&, void*)], string&) except +translate_exception void setDelegate(string&, function[int(string&, size_t)], string&) except +translate_exception void setDelegate(string&, function[int(size_t&, string&)], string&) except +translate_exception @@ -51,7 +51,7 @@ cdef extern from "cantera/cython/funcWrapper.h": PyObject*, void(PyFuncInfo&, size_array2, double, double*, double*)) cdef function[void(size_array3, double*, double*, double*)] pyOverride( PyObject*, void(PyFuncInfo&, size_array3, double*, double*, double*)) - cdef function[int(double&)] pyOverride(PyObject*, int(PyFuncInfo&, double&)) + cdef function[int(double&, void*)] pyOverride(PyObject*, int(PyFuncInfo&, double&, void*)) cdef function[int(string&, size_t)] pyOverride(PyObject*, int(PyFuncInfo&, string&, size_t)) cdef function[int(size_t&, const string&)] pyOverride( PyObject*, int(PyFuncInfo&, size_t&, const string&)) diff --git a/interfaces/cython/cantera/delegator.pyx b/interfaces/cython/cantera/delegator.pyx index 12736aafe5..a97b22ba36 100644 --- a/interfaces/cython/cantera/delegator.pyx +++ b/interfaces/cython/cantera/delegator.pyx @@ -6,6 +6,7 @@ import sys from ._utils import CanteraError from ._utils cimport stringify, pystr +from cython.operator import dereference as deref # ## Implementation for each delegated function type # @@ -138,10 +139,10 @@ cdef void callback_v_dp_dp_dp(PyFuncInfo& funcInfo, funcInfo.setExceptionType(exc_type) funcInfo.setExceptionValue(exc_value) -# Wrapper for functions of type double() -cdef int callback_d(PyFuncInfo& funcInfo, double& out): +# Wrapper for functions of type double(void*) +cdef int callback_d_vp(PyFuncInfo& funcInfo, double& out, void* obj): try: - ret = (funcInfo.func())() + ret = (funcInfo.func())(deref(obj)) if ret is None: return 0 else: @@ -286,9 +287,9 @@ cdef int assign_delegates(obj, CxxDelegator* delegator) except -1: elif callback == 'void(double*,double*,double*)': delegator.setDelegate(cxx_name, pyOverride(method, callback_v_dp_dp_dp), cxx_when) - elif callback == 'double()': + elif callback == 'double(void*)': delegator.setDelegate(cxx_name, - pyOverride(method, callback_d), cxx_when) + pyOverride(method, callback_d_vp), cxx_when) elif callback == 'string(size_t)': delegator.setDelegate(cxx_name, pyOverride(method, callback_s_sz), cxx_when) diff --git a/interfaces/cython/cantera/reaction.pyx b/interfaces/cython/cantera/reaction.pyx index 0d9496d6a5..3151187977 100644 --- a/interfaces/cython/cantera/reaction.pyx +++ b/interfaces/cython/cantera/reaction.pyx @@ -714,7 +714,7 @@ cdef class ExtensibleRate(ReactionRate): _reaction_rate_type = "extensible" delegatable_methods = { - "eval": ("evalFromStruct", "double()") + "eval": ("evalFromStruct", "double(void*)") } def __cinit__(self, *args, init=True, **kwargs): if init: diff --git a/test/python/test_reaction.py b/test/python/test_reaction.py index 9afed0020f..8f5f0141bd 100644 --- a/test/python/test_reaction.py +++ b/test/python/test_reaction.py @@ -1501,12 +1501,7 @@ def func(T): class TestExtensible(ReactionTests, utilities.CanteraTest): # test Extensible reaction rate class UserRate(ct.ExtensibleRate): - def __init__(self, soln): - super().__init__() - self.soln = soln - - def replace_eval(self): - T = self.soln.T + def replace_eval(self, T): return 38.7 * T**2.7 * exp(-3150.15428/T) # probe O + H2 <=> H + OH @@ -1518,7 +1513,7 @@ def replace_eval(self): def setUp(self): super().setUp() - self._rate_obj = self.UserRate(self.soln) + self._rate_obj = self.UserRate() def test_no_rate(self): pytest.skip("ExtensibleRate does not support 'empty' rates") From 150a8a89309f1ddab1035a11fd4c497de206c84a Mon Sep 17 00:00:00 2001 From: Ray Speth Date: Fri, 12 Aug 2022 13:05:09 -0400 Subject: [PATCH 03/26] Make Delegator ownership optional for ExtensibleRate --- interfaces/cython/cantera/ctcxx.pxd | 1 + interfaces/cython/cantera/reaction.pxd | 2 +- interfaces/cython/cantera/reaction.pyx | 13 +++++++++++-- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/interfaces/cython/cantera/ctcxx.pxd b/interfaces/cython/cantera/ctcxx.pxd index 4ab4103ecf..28b77d7ff8 100644 --- a/interfaces/cython/cantera/ctcxx.pxd +++ b/interfaces/cython/cantera/ctcxx.pxd @@ -33,3 +33,4 @@ cdef extern from "": cppclass shared_ptr "std::shared_ptr" [T]: T* get() void reset(T*) + void reset() diff --git a/interfaces/cython/cantera/reaction.pxd b/interfaces/cython/cantera/reaction.pxd index 9770198b8e..fde60b5e39 100644 --- a/interfaces/cython/cantera/reaction.pxd +++ b/interfaces/cython/cantera/reaction.pxd @@ -253,7 +253,7 @@ cdef class CustomRate(ReactionRate): cdef Func1 _rate_func # prevent premature garbage collection cdef class ExtensibleRate(ReactionRate): - pass + cdef set_cxx_object(self, CxxReactionRate* rate=*) cdef class InterfaceRateBase(ArrheniusRateBase): cdef CxxInterfaceRateBase* interface diff --git a/interfaces/cython/cantera/reaction.pyx b/interfaces/cython/cantera/reaction.pyx index 3151187977..04492ec073 100644 --- a/interfaces/cython/cantera/reaction.pyx +++ b/interfaces/cython/cantera/reaction.pyx @@ -722,8 +722,17 @@ cdef class ExtensibleRate(ReactionRate): self.set_cxx_object() def __init__(self, *args, **kwargs): - assign_delegates(self, dynamic_cast[CxxDelegatorPtr](self.rate)) - super().__init__(*args, **kwargs) + if self._rate.get() is not NULL: + assign_delegates(self, dynamic_cast[CxxDelegatorPtr](self.rate)) + # ReactionRate does not define __init__, so it does not need to be called + + cdef set_cxx_object(self, CxxReactionRate* rate=NULL): + if rate is NULL: + self.rate = self._rate.get() + else: + self._rate.reset() + self.rate = rate + assign_delegates(self, dynamic_cast[CxxDelegatorPtr](self.rate)) cdef class InterfaceRateBase(ArrheniusRateBase): """ From 68f459758957666cb8e1d5149d1f15e20f115f35 Mon Sep 17 00:00:00 2001 From: Ray Speth Date: Fri, 12 Aug 2022 15:26:51 -0400 Subject: [PATCH 04/26] [SCons] Extract Python module-building environment setup --- interfaces/cython/SConscript | 78 +++----------------------------- interfaces/cython/setup.cfg.in | 2 +- site_scons/buildutils.py | 81 +++++++++++++++++++++++++++++++++- 3 files changed, 88 insertions(+), 73 deletions(-) diff --git a/interfaces/cython/SConscript b/interfaces/cython/SConscript index 6e41e0486d..f3ea901076 100644 --- a/interfaces/cython/SConscript +++ b/interfaces/cython/SConscript @@ -1,8 +1,6 @@ """Cython-based Python Module""" import re from pathlib import Path -from pkg_resources import parse_version -import json from buildutils import * Import('env', 'build', 'install') @@ -15,70 +13,8 @@ build(dataFiles) # Install Python samples install(localenv.RecursiveInstall, "$inst_sampledir/python", "#samples/python") -# Get information needed to build the Python module -script = """\ -from sysconfig import * -import numpy -import json -import site -vars = get_config_vars() -vars["plat"] = get_platform() -vars["numpy_include"] = numpy.get_include() -vars["site_packages"] = [d for d in site.getsitepackages() if d.endswith("-packages")] -vars["user_site_packages"] = site.getusersitepackages() -print(json.dumps(vars)) -""" -info = json.loads(get_command_output(localenv["python_cmd"], "-c", script)) -module_ext = info["EXT_SUFFIX"] -inc = info["INCLUDEPY"] -pylib = info.get("LDLIBRARY") -prefix = info["prefix"] -py_version_short = parse_version(info["py_version_short"]) -py_version_full = parse_version(info["py_version"]) -py_version_nodot = info["py_version_nodot"] -numpy_include = info["numpy_include"] -site_packages = info["site_packages"] -user_site_packages = info["user_site_packages"] -localenv.Prepend(CPPPATH=[Dir('#include'), inc, numpy_include]) -localenv.Prepend(LIBS=localenv['cantera_libs']) - -# Fix the module extension for Windows from the sysconfig library. -# See https://github.com/python/cpython/pull/22088 and -# https://bugs.python.org/issue39825 -if ( - py_version_full < parse_version("3.8.7") - and localenv["OS"] == "Windows" - and module_ext == ".pyd" -): - module_ext = f".cp{py_version_nodot}-{info['plat'].replace('-', '_')}.pyd" - -# Don't print deprecation warnings for internal Python changes. -# Only applies to Python 3.8. The field that is deprecated in Python 3.8 -# and causes the warnings to appear will be removed in Python 3.9 so no -# further warnings should be issued. -if localenv["HAS_CLANG"] and py_version_short == parse_version("3.8"): - localenv.Append(CXXFLAGS='-Wno-deprecated-declarations') - -if "icc" in localenv["CC"]: - localenv.Append(CPPDEFINES={"CYTHON_FALLTHROUGH": " __attribute__((fallthrough))"}) - -if localenv['OS'] == 'Darwin': - localenv.Append(LINKFLAGS='-undefined dynamic_lookup') -elif localenv['OS'] == 'Windows': - localenv.Append(LIBPATH=prefix + '/libs') - if localenv['toolchain'] == 'mingw': - localenv.Append(LIBS=f"python{py_version_nodot}") - if localenv['OS_BITS'] == 64: - localenv.Append(CPPDEFINES='MS_WIN64') - # Fix for https://bugs.python.org/issue11566. Fixed in 3.7.3 and higher. - # See https://github.com/python/cpython/pull/11283 - if py_version_full < parse_version("3.7.3"): - localenv.Append(CPPDEFINES={"_hypot": "hypot"}) - -if "numpy_1_7_API" in localenv: - localenv.Append(CPPDEFINES="NPY_NO_DEPRECATED_API=NPY_1_7_API_VERSION") - -localenv["module_ext"] = module_ext +setup_python_env(localenv) + setup_cfg = localenv.SubstFile("setup.cfg", "setup.cfg.in") readme = localenv.Command("README.rst", "#README.rst", Copy("$TARGET", "$SOURCE")) license = localenv.Command("LICENSE.txt", "#build/ext/LICENSE.txt", @@ -104,15 +40,15 @@ for pyxfile in multi_glob(localenv, "cantera", "pyx"): f"#build/temp-py/{pyxfile.name.split('.')[0]}", cythonized) cython_obj.append(obj) +module_ext = localenv["py_module_ext"] ext = localenv.LoadableModule(f"cantera/_cantera{module_ext}", cython_obj, LIBPREFIX="", SHLIBSUFFIX=module_ext, SHLIBPREFIX="", LIBSUFFIXES=[module_ext]) build_cmd = ("$python_cmd_esc -m pip wheel -v --no-build-isolation --no-deps " "--wheel-dir=build/python/dist build/python") -plat = info['plat'].replace('-', '_').replace('.', '_') -wheel_name = (f"Cantera-{env['cantera_version']}-cp{py_version_nodot}" - f"-cp{py_version_nodot}-{plat}.whl") +wheel_name = ("Cantera-${cantera_version}-cp${py_version_nodot}" + "-cp${py_version_nodot}-${plat}.whl") mod = build(localenv.Command(f"#build/python/dist/{wheel_name}", "setup.cfg", build_cmd)) env['python_module'] = mod @@ -153,9 +89,9 @@ else: ignore_errors=True) if user_install: - test_prefix = Path(user_site_packages).parents[2] + test_prefix = Path(localenv["user_site_packages"]).parents[2] elif python_prefix is None: - test_prefix = Path(site_packages[0]).parents[2] + test_prefix = Path(localenv["site_packages"][0]).parents[2] else: test_prefix = Path(python_prefix) diff --git a/interfaces/cython/setup.cfg.in b/interfaces/cython/setup.cfg.in index d1b01ffc79..e857de7f14 100644 --- a/interfaces/cython/setup.cfg.in +++ b/interfaces/cython/setup.cfg.in @@ -51,7 +51,7 @@ packages = # The module extension needs to be here since we don't want setuptools to compile # the extension, so there are no ``source`` files in the setup.py ``extension`` and # we have to treat the module as package data. -cantera = *.pxd, *@module_ext@, test/*.txt, examples/*.txt, data/*.* +cantera = *.pxd, *@py_module_ext@, test/*.txt, examples/*.txt, data/*.* [options.extras_require] hdf5 = h5py diff --git a/site_scons/buildutils.py b/site_scons/buildutils.py index fe83dfca71..0b6d655370 100644 --- a/site_scons/buildutils.py +++ b/site_scons/buildutils.py @@ -11,10 +11,12 @@ import shutil import enum from pathlib import Path +from pkg_resources import parse_version import logging from typing import TYPE_CHECKING from collections.abc import Mapping as MappingABC from SCons.Variables import PathVariable, EnumVariable, BoolVariable +from SCons.Script import Dir try: import numpy as np @@ -25,7 +27,7 @@ "logger", "remove_directory", "remove_file", "test_results", "add_RegressionTest", "get_command_output", "listify", "which", "ConfigBuilder", "multi_glob", "get_spawn", "quoted", - "get_pip_install_location", "compiler_flag_list") + "get_pip_install_location", "compiler_flag_list", "setup_python_env") if TYPE_CHECKING: from typing import Iterable, TypeVar, Union, List, Dict, Tuple, Optional, \ @@ -1246,6 +1248,83 @@ def get_command_output(cmd: str, *args: str, ignore_errors=False): ) return data.stdout.strip() +_python_info = None +def setup_python_env(env): + """Set up an environment for compiling Python extension modules""" + + global _python_info + if _python_info is None: + # Get information needed to build the Python module + script = textwrap.dedent("""\ + from sysconfig import * + import numpy + import json + import site + vars = get_config_vars() + vars["plat"] = get_platform() + vars["numpy_include"] = numpy.get_include() + vars["site_packages"] = [d for d in site.getsitepackages() if d.endswith("-packages")] + vars["user_site_packages"] = site.getusersitepackages() + print(json.dumps(vars)) + """) + _python_info = json.loads(get_command_output(env["python_cmd"], "-c", script)) + + info = _python_info + module_ext = info["EXT_SUFFIX"] + inc = info["INCLUDEPY"] + pylib = info.get("LDLIBRARY") + prefix = info["prefix"] + py_version_short = parse_version(info["py_version_short"]) + py_version_full = parse_version(info["py_version"]) + py_version_nodot = info["py_version_nodot"] + plat = info['plat'].replace('-', '_').replace('.', '_') + numpy_include = info["numpy_include"] + env.Prepend(CPPPATH=[Dir('#include'), inc, numpy_include]) + env.Prepend(LIBS=env['cantera_libs']) + + # Fix the module extension for Windows from the sysconfig library. + # See https://github.com/python/cpython/pull/22088 and + # https://bugs.python.org/issue39825 + if (py_version_full < parse_version("3.8.7") + and env["OS"] == "Windows" + and module_ext == ".pyd" + ): + module_ext = f".cp{py_version_nodot}-{info['plat'].replace('-', '_')}.pyd" + + env["py_module_ext"] = module_ext + env["py_version_nodot"] = py_version_nodot + env["py_plat"] = plat + env["site_packages"] = info["site_packages"] + env["user_site_packages"] = info["user_site_packages"] + + # Don't print deprecation warnings for internal Python changes. + # Only applies to Python 3.8. The field that is deprecated in Python 3.8 + # and causes the warnings to appear will be removed in Python 3.9 so no + # further warnings should be issued. + if env["HAS_CLANG"] and py_version_short == parse_version("3.8"): + env.Append(CXXFLAGS='-Wno-deprecated-declarations') + + if "icc" in env["CC"]: + env.Append(CPPDEFINES={"CYTHON_FALLTHROUGH": " __attribute__((fallthrough))"}) + + if env['OS'] == 'Darwin': + env.Append(LINKFLAGS='-undefined dynamic_lookup') + elif env['OS'] == 'Windows': + env.Append(LIBPATH=prefix + '/libs') + if env['toolchain'] == 'mingw': + env.Append(LIBS=f"python{py_version_nodot}") + if env['OS_BITS'] == 64: + env.Append(CPPDEFINES='MS_WIN64') + # Fix for https://bugs.python.org/issue11566. Fixed in 3.7.3 and higher. + # See https://github.com/python/cpython/pull/11283 + if py_version_full < parse_version("3.7.3"): + env.Append(CPPDEFINES={"_hypot": "hypot"}) + + if "numpy_1_7_API" in env: + env.Append(CPPDEFINES="NPY_NO_DEPRECATED_API=NPY_1_7_API_VERSION") + + + return env def get_pip_install_location( python_cmd: str, From 1b443a25a1d17d8b6bedbdbaf208d3c1a18147a0 Mon Sep 17 00:00:00 2001 From: Ray Speth Date: Sat, 13 Aug 2022 12:43:58 -0400 Subject: [PATCH 05/26] Add machinery for managing user extensions --- include/cantera/base/ExtensionManager.h | 29 ++++++++++++ .../cantera/base/ExtensionManagerFactory.h | 44 +++++++++++++++++++ include/cantera/base/ctexceptions.h | 1 + include/cantera/base/global.h | 8 ++++ src/base/ExtensionManagerFactory.cpp | 36 +++++++++++++++ src/base/application.cpp | 11 +++++ src/base/application.h | 11 +++++ src/base/global.cpp | 15 +++++++ src/kinetics/KineticsFactory.cpp | 3 ++ test/kinetics/kineticsFromYaml.cpp | 12 +++++ 10 files changed, 170 insertions(+) create mode 100644 include/cantera/base/ExtensionManager.h create mode 100644 include/cantera/base/ExtensionManagerFactory.h create mode 100644 src/base/ExtensionManagerFactory.cpp diff --git a/include/cantera/base/ExtensionManager.h b/include/cantera/base/ExtensionManager.h new file mode 100644 index 0000000000..4bee0b25f3 --- /dev/null +++ b/include/cantera/base/ExtensionManager.h @@ -0,0 +1,29 @@ +//! @file ExtensionManager.h + +#ifndef CT_EXTENSIONMANAGER_H +#define CT_EXTENSIONMANAGER_H + +// This file is part of Cantera. See License.txt in the top-level directory or +// at https://cantera.org/license.txt for license and copyright information. + +#include "cantera/base/ctexceptions.h" + +namespace Cantera +{ + +//! Base class for managing user-defined Cantera extensions written in other languages +class ExtensionManager +{ +public: + virtual ~ExtensionManager() = default; + + //! Register ReactionRate defined in a user extension with ReactionRateFactory + //! @param extensionName + virtual void registerRateBuilders(const std::string& extensionName) { + throw NotImplementedError("ExtensionManager::registerRateBuilders"); + }; +}; + +} + +#endif diff --git a/include/cantera/base/ExtensionManagerFactory.h b/include/cantera/base/ExtensionManagerFactory.h new file mode 100644 index 0000000000..cc25361e54 --- /dev/null +++ b/include/cantera/base/ExtensionManagerFactory.h @@ -0,0 +1,44 @@ +//! @file ExtensionManagerFactory.h + +#ifndef CT_EXTENSIONMANAGERFACTORY_H +#define CT_EXTENSIONMANAGERFACTORY_H + +// This file is part of Cantera. See License.txt in the top-level directory or +// at https://cantera.org/license.txt for license and copyright information. + +#include "FactoryBase.h" +#include "ExtensionManager.h" + +namespace Cantera +{ + +//! A factory class for creating ExtensionManager objects +class ExtensionManagerFactory : public Factory +{ +public: + //! Create a new ExtensionManager + static shared_ptr build(const std::string& extensionType) { + return shared_ptr(factory().create(extensionType)); + } + + //! Delete the static instance of this factory + virtual void deleteFactory(); + +private: + //! Static function that returns the static instance of the factory, creating it + //! if necessary. + static ExtensionManagerFactory& factory(); + + //! static member of the single factory instance + static ExtensionManagerFactory* s_factory; + + //! Private constructor prevents direct usage + ExtensionManagerFactory(); + + //! Decl for locking mutex for thermo factory singleton + static std::mutex s_mutex; +}; + +} + +#endif diff --git a/include/cantera/base/ctexceptions.h b/include/cantera/base/ctexceptions.h index ef915cf61d..0a945aee63 100644 --- a/include/cantera/base/ctexceptions.h +++ b/include/cantera/base/ctexceptions.h @@ -11,6 +11,7 @@ #ifndef CT_CTEXCEPTIONS_H #define CT_CTEXCEPTIONS_H +#include "ct_defs.h" #include "cantera/base/fmt.h" #include diff --git a/include/cantera/base/global.h b/include/cantera/base/global.h index 545bba7d9c..376991d04a 100644 --- a/include/cantera/base/global.h +++ b/include/cantera/base/global.h @@ -26,6 +26,7 @@ namespace Cantera { class Logger; +class AnyMap; /*! * @defgroup inputfiles Input File Handling @@ -77,6 +78,13 @@ void addDirectory(const std::string& dir); std::string getDataDirectories(const std::string& sep); //! @} +//! @copydoc Application::loadExtension +void loadExtension(const std::string& extType, const std::string& name); + +//! Load extensions providing user-defined models from the `extensions` section of the +//! given node. @see Application::loadExtension +void loadExtensions(const AnyMap& node); + //! Delete and free all memory associated with the application /*! * Delete all global data. It should be called at the end of the diff --git a/src/base/ExtensionManagerFactory.cpp b/src/base/ExtensionManagerFactory.cpp new file mode 100644 index 0000000000..becae86a4f --- /dev/null +++ b/src/base/ExtensionManagerFactory.cpp @@ -0,0 +1,36 @@ +//! @file ExtensionManagerFactory.cpp + +// This file is part of Cantera. See License.txt in the top-level directory or +// at https://cantera.org/license.txt for license and copyright information. + +#include "cantera/base/ExtensionManagerFactory.h" + +using namespace std; + +namespace Cantera +{ + +ExtensionManagerFactory* ExtensionManagerFactory::s_factory = 0; +mutex ExtensionManagerFactory::s_mutex; + +ExtensionManagerFactory::ExtensionManagerFactory() +{ +} + +ExtensionManagerFactory& ExtensionManagerFactory::factory() +{ + unique_lock lock(s_mutex); + if (!s_factory) { + s_factory = new ExtensionManagerFactory(); + } + return *s_factory; +} + +void ExtensionManagerFactory::deleteFactory() +{ + unique_lock lock(s_mutex); + delete s_factory; + s_factory = 0; +} + +} diff --git a/src/base/application.cpp b/src/base/application.cpp index 9e712171fc..221f2d46e7 100644 --- a/src/base/application.cpp +++ b/src/base/application.cpp @@ -6,6 +6,7 @@ #include "application.h" #include "cantera/base/ctexceptions.h" #include "cantera/base/stringUtils.h" +#include "cantera/base/ExtensionManagerFactory.h" #include #include @@ -396,6 +397,16 @@ std::string Application::findInputFile(const std::string& name) throw CanteraError("Application::findInputFile", msg); } +void Application::loadExtension(const string& extType, const string& name) +{ + if (m_loaded_extensions.count({extType, name})) { + return; + } + auto manager = ExtensionManagerFactory::build(extType); + manager->registerRateBuilders(name); + m_loaded_extensions.insert({extType, name}); +} + Application* Application::s_app = 0; } // namespace Cantera diff --git a/src/base/application.h b/src/base/application.h index a66f525324..bf5039a86d 100644 --- a/src/base/application.h +++ b/src/base/application.h @@ -283,6 +283,15 @@ class Application return boost::algorithm::join(inputDirs, sep); } + //! Load an extension implementing user-defined models + //! @param extType Specifies the interface / language of the extension, for example + //! "python" + //! @param extName Specifies the name of the extension. The meaning of this + //! parameter depends on the specific extension interface. For example, for + //! Python extensions, this is the name of the Python module containing the + //! models. + void loadExtension(const std::string& extType, const std::string& name); + #ifdef _WIN32 long int readStringRegistryKey(const std::string& keyName, const std::string& valueName, std::string& value, const std::string& defaultValue); @@ -433,6 +442,8 @@ class Application bool m_fatal_warnings; bool m_use_legacy_rate_constants; + std::set> m_loaded_extensions; + ThreadMessages pMessenger; private: diff --git a/src/base/global.cpp b/src/base/global.cpp index c155b5e4ab..1ac3a2b2ec 100644 --- a/src/base/global.cpp +++ b/src/base/global.cpp @@ -146,6 +146,21 @@ std::string findInputFile(const std::string& name) return app()->findInputFile(name); } +void loadExtension(const std::string& extType, const std::string& name) +{ + app()->loadExtension(extType, name); +} + +void loadExtensions(const AnyMap& node) +{ + if (!node.hasKey("extensions")) { + return; + } + for (auto& extension : node["extensions"].asVector()) { + loadExtension(extension["type"].asString(), extension["name"].asString()); + } +} + bool debugModeEnabled() { #ifdef NDEBUG diff --git a/src/kinetics/KineticsFactory.cpp b/src/kinetics/KineticsFactory.cpp index 81f3662246..0ccc728c2b 100644 --- a/src/kinetics/KineticsFactory.cpp +++ b/src/kinetics/KineticsFactory.cpp @@ -83,6 +83,8 @@ void addReactions(Kinetics& kin, const AnyMap& phaseNode, const AnyMap& rootNode kin.skipUndeclaredThirdBodies( phaseNode.getBool("skip-undeclared-third-bodies", false)); + loadExtensions(rootNode); + // Find sections containing reactions to add vector sections, rules; @@ -152,6 +154,7 @@ void addReactions(Kinetics& kin, const AnyMap& phaseNode, const AnyMap& rootNode string node(slash.end(), sections[i].end()); AnyMap reactions = AnyMap::fromYamlFile(fileName, rootNode.getString("__file__", "")); + loadExtensions(reactions); for (const auto& R : reactions[node].asVector()) { #ifdef NDEBUG try { diff --git a/test/kinetics/kineticsFromYaml.cpp b/test/kinetics/kineticsFromYaml.cpp index 2d011d3e6f..1b410532ea 100644 --- a/test/kinetics/kineticsFromYaml.cpp +++ b/test/kinetics/kineticsFromYaml.cpp @@ -697,6 +697,18 @@ TEST(KineticsFromYaml, ReactionsFieldWithoutKineticsModel2) InputFileError); } +TEST(KineticsFromYaml, InvalidExtension) +{ + AnyMap input = AnyMap::fromYamlFile("h2o2.yaml"); + newSolution(input["phases"].asVector()[0], input); + std::vector extensions(1); + extensions[0]["type"] = "nonexistent"; + extensions[0]["name"] = "fake"; + input["extensions"] = extensions; + EXPECT_THROW(newSolution(input["phases"].asVector()[0], input), + CanteraError); +} + class ReactionToYaml : public testing::Test { public: From 263a12ad576fa55c3956983f564cc06fcf7a9257 Mon Sep 17 00:00:00 2001 From: Ray Speth Date: Fri, 12 Aug 2022 18:39:53 -0400 Subject: [PATCH 06/26] Scan user-provided Python module for ExtensibleRate classes --- .github/workflows/main.yml | 2 +- .gitignore | 1 + SConstruct | 1 + include/cantera/base/config.h.in | 1 + .../extensions/PythonExtensionManager.h | 26 ++++++++ interfaces/cython/SConscript | 4 ++ site_scons/buildutils.py | 1 + src/SConscript | 24 ++++++++ src/base/ExtensionManagerFactory.cpp | 7 +++ src/extensions/PythonExtensionManager.cpp | 60 +++++++++++++++++++ src/extensions/pythonExtensions.pyx | 32 ++++++++++ 11 files changed, 158 insertions(+), 1 deletion(-) create mode 100644 include/cantera/extensions/PythonExtensionManager.h create mode 100644 src/extensions/PythonExtensionManager.cpp create mode 100644 src/extensions/pythonExtensions.pyx diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 45f8f42730..82bc33caf6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -41,7 +41,7 @@ jobs: - name: Install Apt dependencies run: | sudo apt update - sudo apt install libboost-dev gfortran libopenmpi-dev + sudo apt install libboost-dev gfortran libopenmpi-dev libpython3-dev - name: Upgrade pip run: python3 -m pip install -U pip setuptools wheel - name: Install Python dependencies diff --git a/.gitignore b/.gitignore index a681c88e6a..8501cdf623 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ include/cantera/base/system.h.gch include/cantera/ext/ interfaces/matlab/ctpath.m interfaces/matlab/Contents.m +src/extensions/pythonExtensions.h stage/ .sconsign.dblite .sconf_temp diff --git a/SConstruct b/SConstruct index 6c65fe74b6..d6e4fa5bd6 100644 --- a/SConstruct +++ b/SConstruct @@ -1910,6 +1910,7 @@ cdefine("CT_USE_SYSTEM_EIGEN_PREFIXED", "system_eigen_prefixed") cdefine('CT_USE_SYSTEM_FMT', 'system_fmt') cdefine('CT_USE_SYSTEM_YAMLCPP', 'system_yamlcpp') cdefine('CT_USE_DEMANGLE', 'has_demangle') +cdefine('CT_HAS_PYTHON', 'python_package', 'full') config_h_build = env.Command('build/src/config.h.build', 'include/cantera/base/config.h.in', diff --git a/include/cantera/base/config.h.in b/include/cantera/base/config.h.in index 59379abc45..00471f5025 100644 --- a/include/cantera/base/config.h.in +++ b/include/cantera/base/config.h.in @@ -47,6 +47,7 @@ typedef int ftnlen; // Fortran hidden string length type {CT_USE_SYSTEM_FMT!s} {CT_USE_SYSTEM_YAMLCPP!s} {CT_USE_DEMANGLE!s} +{CT_HAS_PYTHON!s} //--------- operating system -------------------------------------- diff --git a/include/cantera/extensions/PythonExtensionManager.h b/include/cantera/extensions/PythonExtensionManager.h new file mode 100644 index 0000000000..1e5f46418e --- /dev/null +++ b/include/cantera/extensions/PythonExtensionManager.h @@ -0,0 +1,26 @@ +//! @file PythonExtensionManager.h + +#ifndef CT_PYTHONEXTENSIONMANAGER_H +#define CT_PYTHONEXTENSIONMANAGER_H + +// This file is part of Cantera. See License.txt in the top-level directory or +// at https://cantera.org/license.txt for license and copyright information. + +#include "cantera/base/ExtensionManager.h" + +namespace Cantera +{ + +//! Class for managing user-defined Cantera extensions written in Python +//! +//! Handles Python initialization if the main application is not the Python interpreter. +class PythonExtensionManager : public ExtensionManager +{ +public: + PythonExtensionManager(); + virtual void registerRateBuilders(const std::string& extensionName) override; +}; + +} + +#endif diff --git a/interfaces/cython/SConscript b/interfaces/cython/SConscript index f3ea901076..d4102dcbb6 100644 --- a/interfaces/cython/SConscript +++ b/interfaces/cython/SConscript @@ -14,6 +14,10 @@ build(dataFiles) install(localenv.RecursiveInstall, "$inst_sampledir/python", "#samples/python") setup_python_env(localenv) +# Python module shouldn't explicitly link to Python library (added in other cases to +# support Python-based extensions), except when using MinGW +if localenv["toolchain"] != "mingw": + localenv["LIBS"] = [lib for lib in localenv["LIBS"] if not lib.startswith("python")] setup_cfg = localenv.SubstFile("setup.cfg", "setup.cfg.in") readme = localenv.Command("README.rst", "#README.rst", Copy("$TARGET", "$SOURCE")) diff --git a/site_scons/buildutils.py b/site_scons/buildutils.py index 0b6d655370..eb6271df74 100644 --- a/site_scons/buildutils.py +++ b/site_scons/buildutils.py @@ -1293,6 +1293,7 @@ def setup_python_env(env): env["py_module_ext"] = module_ext env["py_version_nodot"] = py_version_nodot + env["py_version_short"] = info["py_version_short"] env["py_plat"] = plat env["site_packages"] = info["site_packages"] env["user_site_packages"] = info["user_site_packages"] diff --git a/src/SConscript b/src/SConscript index a668fd4290..275fc1a8c0 100644 --- a/src/SConscript +++ b/src/SConscript @@ -1,4 +1,5 @@ from buildutils import * +from pathlib import Path Import('env', 'build', 'install', 'libraryTargets') @@ -70,6 +71,29 @@ for subdir, extensions, setup in libs: env2.Requires(objects, pch) libraryTargets.extend(objects) +# Handling of extensions written in Python +if env["python_package"] == "full": + pyenv = setup_python_env(localenv.Clone()) + pyenv.PrependENVPath('PYTHONPATH', Dir('#interfaces/cython').abspath) + build_dir = Path(Dir('#build').abspath).as_posix() + cythonized = pyenv.Command( + ["extensions/pythonExtensions.cpp", "extensions/pythonExtensions.h"], + "extensions/pythonExtensions.pyx", + ('''${python_cmd} -c "import Cython.Build; ''' + f'''Cython.Build.cythonize(r'${{SOURCE}}', build_dir='{build_dir}')"''') + ) + for pxd in multi_glob(localenv, "#interfaces/cython/cantera", "pxd"): + localenv.Depends(cythonized, pxd) + + obj = pyenv.SharedObject(cythonized[0]) + env.Command('#src/extensions/pythonExtensions.h', '#build/src/extensions/pythonExtensions.h', + Copy('$TARGET', '$SOURCE')) + libraryTargets.append(obj) + libraryTargets.append(pyenv.SharedObject("extensions/PythonExtensionManager.cpp")) + libpython = pyenv.subst("python${py_version_short}") + localenv.Append(LIBS=libpython) + env["cantera_libs"].append(libpython) + # build the Cantera static library lib = build(localenv.StaticLibrary('../lib/cantera', libraryTargets, diff --git a/src/base/ExtensionManagerFactory.cpp b/src/base/ExtensionManagerFactory.cpp index becae86a4f..35f2ac1a33 100644 --- a/src/base/ExtensionManagerFactory.cpp +++ b/src/base/ExtensionManagerFactory.cpp @@ -5,6 +5,10 @@ #include "cantera/base/ExtensionManagerFactory.h" +#ifdef CT_HAS_PYTHON +#include "cantera/extensions/PythonExtensionManager.h" +#endif + using namespace std; namespace Cantera @@ -15,6 +19,9 @@ mutex ExtensionManagerFactory::s_mutex; ExtensionManagerFactory::ExtensionManagerFactory() { + #ifdef CT_HAS_PYTHON + reg("python", []() { return new PythonExtensionManager(); }); + #endif } ExtensionManagerFactory& ExtensionManagerFactory::factory() diff --git a/src/extensions/PythonExtensionManager.cpp b/src/extensions/PythonExtensionManager.cpp new file mode 100644 index 0000000000..5547922dcf --- /dev/null +++ b/src/extensions/PythonExtensionManager.cpp @@ -0,0 +1,60 @@ +//! @file PythonExtensionManager.cpp + +// This file is part of Cantera. See License.txt in the top-level directory or +// at https://cantera.org/license.txt for license and copyright information. + +#include "cantera/extensions/PythonExtensionManager.h" + +#include "cantera/kinetics/ReactionRateDelegator.h" +#include "pythonExtensions.h" // generated by Cython + +namespace Cantera +{ + +PythonExtensionManager::PythonExtensionManager() +{ + if (!Py_IsInitialized()) { + Py_Initialize(); + } + + // PEP 489 Multi-phase initialization + PyModuleDef* modDef = (PyModuleDef*) PyInit_pythonExtensions(); + if (!modDef->m_slots || !PyModuleDef_Init(modDef)) { + throw CanteraError("PythonExtensionManager::PythonExtensionManager", + "Failed to import 'pythonExtensions' module"); + } + + // Following example creation of minimal ModuleSpec from Python's import.c + PyObject *attrs = Py_BuildValue("{ss}", "name", "pythonExtensions"); + if (attrs == NULL) { + throw CanteraError("PythonExtensionManager::PythonExtensionManager", + "Py_BuildValue failed"); + } + PyObject *spec = _PyNamespace_New(attrs); + Py_DECREF(attrs); + if (spec == NULL) { + throw CanteraError("PythonExtensionManager::PythonExtensionManager", + "_PyNamespace_New failed"); + } + PyObject* pyModule = PyModule_FromDefAndSpec(modDef, spec); + if (!pyModule) { + CanteraError("PythonExtensionManager::PythonExtensionManager", + "PyModule_FromDefAndSpec failed"); + } + if (!PyModule_ExecDef(pyModule, modDef)) { + CanteraError("PythonExtensionManager::PythonExtensionManager", + "PyModule_ExecDef failed"); + } + Py_DECREF(spec); + Py_DECREF(pyModule); +} + +void PythonExtensionManager::registerRateBuilders(const std::string& extensionName) +{ + char* c_rateTypes = ct_getPythonExtensibleRateTypes(extensionName); + std::string rateTypes(c_rateTypes); + free(c_rateTypes); + writelog("Module returned types: '{}'\n", rateTypes); +} + +}; diff --git a/src/extensions/pythonExtensions.pyx b/src/extensions/pythonExtensions.pyx new file mode 100644 index 0000000000..52ae8bc9fa --- /dev/null +++ b/src/extensions/pythonExtensions.pyx @@ -0,0 +1,32 @@ +# This file is part of Cantera. See License.txt in the top-level directory or +# at https://cantera.org/license.txt for license and copyright information. + +#cython: language_level=3 +#distutils: language=c++ + +from libcpp.string cimport string +from libc.stdlib cimport malloc +from libc.string cimport strcpy + +import importlib +import inspect + +import cantera as ct +from cantera.reaction cimport ExtensibleRate + +cdef public char* ct_getPythonExtensibleRateTypes(const string& module_name): + """ + Load the named module and find classes derived from ExtensibleRate. + + Returns a string where each line contains the class name and the corresponding + rate name, separated by a space + """ + mod = importlib.import_module(module_name.decode()) + names = "\n".join( + f"{name} {cls._reaction_rate_type}" + for name, cls in inspect.getmembers(mod) + if inspect.isclass(cls) and issubclass(cls, ct.ExtensibleRate)) + tmp = bytes(names.encode()) + cdef char* c_string = malloc((len(tmp) + 1) * sizeof(char)) + strcpy(c_string, tmp) + return c_string From 984503255acd6ddc949fecc39376e9c69067762d Mon Sep 17 00:00:00 2001 From: Ray Speth Date: Sat, 27 Aug 2022 17:13:39 -0400 Subject: [PATCH 07/26] Register ExtensibleRate types with ReactionRateFactory --- interfaces/cython/cantera/_cantera.pyx | 1 + src/extensions/PythonExtensionManager.cpp | 35 +++++++++++++++++++++-- src/extensions/pythonExtensions.pyx | 26 +++++++++++++++-- 3 files changed, 56 insertions(+), 6 deletions(-) diff --git a/interfaces/cython/cantera/_cantera.pyx b/interfaces/cython/cantera/_cantera.pyx index 943080c4ef..4503566d6c 100644 --- a/interfaces/cython/cantera/_cantera.pyx +++ b/interfaces/cython/cantera/_cantera.pyx @@ -7,6 +7,7 @@ import sys import importlib import importlib.abc +import importlib.util # Chooses the right init function # See https://stackoverflow.com/a/52714500 diff --git a/src/extensions/PythonExtensionManager.cpp b/src/extensions/PythonExtensionManager.cpp index 5547922dcf..fba5fc9fb1 100644 --- a/src/extensions/PythonExtensionManager.cpp +++ b/src/extensions/PythonExtensionManager.cpp @@ -5,9 +5,15 @@ #include "cantera/extensions/PythonExtensionManager.h" +#include "cantera/kinetics/ReactionRateFactory.h" #include "cantera/kinetics/ReactionRateDelegator.h" #include "pythonExtensions.h" // generated by Cython +#include + +namespace ba = boost::algorithm; +using namespace std; + namespace Cantera { @@ -49,12 +55,35 @@ PythonExtensionManager::PythonExtensionManager() Py_DECREF(pyModule); } -void PythonExtensionManager::registerRateBuilders(const std::string& extensionName) +void PythonExtensionManager::registerRateBuilders(const string& extensionName) { char* c_rateTypes = ct_getPythonExtensibleRateTypes(extensionName); - std::string rateTypes(c_rateTypes); + string rateTypes(c_rateTypes); free(c_rateTypes); - writelog("Module returned types: '{}'\n", rateTypes); + + // Each line in rateTypes is a (class name, rate name) pair, separated by a tab + vector lines; + ba::split(lines, rateTypes, boost::is_any_of("\n")); + for (auto& line : lines) { + vector tokens; + ba::split(tokens, line, boost::is_any_of("\t")); + if (tokens.size() != 2) { + CanteraError("PythonExtensionManager::registerRateBuilders", + "Got unparsable input from ct_getPythonExtensibleRateTypes:" + "\n'''{}\n'''", rateTypes); + } + string rateName = tokens[0]; + + // Create a function that constructs and links a C++ ReactionRateDelegator + // object and a Python ExtensibleRate object of a particular type, and register + // this as the builder for reactions of this type + auto builder = [rateName, extensionName](const AnyMap& params, const UnitStack& units) { + auto delegator = make_unique(); + ct_addPythonExtensibleRate(delegator.get(), extensionName, rateName); + return delegator.release(); + }; + ReactionRateFactory::factory()->reg(tokens[1], builder); + } } }; diff --git a/src/extensions/pythonExtensions.pyx b/src/extensions/pythonExtensions.pyx index 52ae8bc9fa..ad5c37c619 100644 --- a/src/extensions/pythonExtensions.pyx +++ b/src/extensions/pythonExtensions.pyx @@ -7,14 +7,22 @@ from libcpp.string cimport string from libc.stdlib cimport malloc from libc.string cimport strcpy +from cpython.ref cimport Py_INCREF import importlib import inspect import cantera as ct -from cantera.reaction cimport ExtensibleRate +from cantera.reaction cimport ExtensibleRate, CxxReactionRate +from cantera.delegator cimport CxxDelegator, assign_delegates -cdef public char* ct_getPythonExtensibleRateTypes(const string& module_name): + +cdef extern from "cantera/kinetics/ReactionRateDelegator.h" namespace "Cantera": + cdef cppclass CxxReactionRateDelegator "Cantera::ReactionRateDelegator" (CxxDelegator, CxxReactionRate): + CxxReactionRateDelegator() + + +cdef public char* ct_getPythonExtensibleRateTypes(const string& module_name) except NULL: """ Load the named module and find classes derived from ExtensibleRate. @@ -23,10 +31,22 @@ cdef public char* ct_getPythonExtensibleRateTypes(const string& module_name): """ mod = importlib.import_module(module_name.decode()) names = "\n".join( - f"{name} {cls._reaction_rate_type}" + f"{name}\t{cls._reaction_rate_type}" for name, cls in inspect.getmembers(mod) if inspect.isclass(cls) and issubclass(cls, ct.ExtensibleRate)) tmp = bytes(names.encode()) cdef char* c_string = malloc((len(tmp) + 1) * sizeof(char)) strcpy(c_string, tmp) return c_string + + +cdef public int ct_addPythonExtensibleRate(CxxReactionRateDelegator* delegator, + const string& module_name, + const string& class_name) except -1: + + mod = importlib.import_module(module_name.decode()) + cdef ExtensibleRate rate = getattr(mod, class_name.decode())(init=False) + Py_INCREF(rate) + rate.set_cxx_object(delegator) + assign_delegates(rate, delegator) + return 0 From c302229d301277bf4641d4d6d91006756d51aebf Mon Sep 17 00:00:00 2001 From: Ray Speth Date: Sat, 27 Aug 2022 18:02:15 -0400 Subject: [PATCH 08/26] Allow Delegator to delete Python rates --- include/cantera/base/Delegator.h | 14 ++++++++++++++ src/extensions/PythonExtensionManager.cpp | 13 ++++++++++++- src/extensions/pythonExtensions.pyx | 9 ++++----- 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/include/cantera/base/Delegator.h b/include/cantera/base/Delegator.h index 964117c1f1..d220052294 100644 --- a/include/cantera/base/Delegator.h +++ b/include/cantera/base/Delegator.h @@ -9,6 +9,7 @@ #include "cantera/base/global.h" #include "cantera/base/ctexceptions.h" #include +#include namespace Cantera { @@ -100,6 +101,12 @@ namespace Cantera class Delegator { public: + ~Delegator() { + for (auto& func : m_cleanup_funcs) { + func(); + } + } + //! Set delegates for member functions with the signature `void()`. void setDelegate(const std::string& name, const std::function& func, const std::string& when) @@ -227,6 +234,10 @@ class Delegator *m_funcs_sz_csr[name] = makeDelegate(func, when, m_base_sz_csr[name]); } + void addCleanupFunc(const std::function& func) { + m_cleanup_funcs.push_back(func); + } + protected: //! Install a function with the signature `void()` as being delegatable void install(const std::string& name, std::function& target, @@ -446,6 +457,9 @@ class Delegator std::map*> m_funcs_sz_csr; //! @} + + //! Cleanup functions to be called from the destructor + std::list> m_cleanup_funcs; }; } diff --git a/src/extensions/PythonExtensionManager.cpp b/src/extensions/PythonExtensionManager.cpp index fba5fc9fb1..9d554f6fe5 100644 --- a/src/extensions/PythonExtensionManager.cpp +++ b/src/extensions/PythonExtensionManager.cpp @@ -72,6 +72,7 @@ void PythonExtensionManager::registerRateBuilders(const string& extensionName) "Got unparsable input from ct_getPythonExtensibleRateTypes:" "\n'''{}\n'''", rateTypes); } + string rateName = tokens[0]; // Create a function that constructs and links a C++ ReactionRateDelegator @@ -79,7 +80,17 @@ void PythonExtensionManager::registerRateBuilders(const string& extensionName) // this as the builder for reactions of this type auto builder = [rateName, extensionName](const AnyMap& params, const UnitStack& units) { auto delegator = make_unique(); - ct_addPythonExtensibleRate(delegator.get(), extensionName, rateName); + PyObject* extRate = ct_newPythonExtensibleRate(delegator.get(), + extensionName, rateName); + if (extRate == nullptr) { + throw CanteraError("PythonExtensionManager::registerRateBuilders", + "ct_newPythonExtensibleRate failed"); + } + + // Make the delegator responsible for eventually deleting the Python object + Py_IncRef(extRate); + delegator->addCleanupFunc([extRate]() { Py_DecRef(extRate); }); + return delegator.release(); }; ReactionRateFactory::factory()->reg(tokens[1], builder); diff --git a/src/extensions/pythonExtensions.pyx b/src/extensions/pythonExtensions.pyx index ad5c37c619..e2a7ff1fa9 100644 --- a/src/extensions/pythonExtensions.pyx +++ b/src/extensions/pythonExtensions.pyx @@ -40,13 +40,12 @@ cdef public char* ct_getPythonExtensibleRateTypes(const string& module_name) exc return c_string -cdef public int ct_addPythonExtensibleRate(CxxReactionRateDelegator* delegator, - const string& module_name, - const string& class_name) except -1: +cdef public object ct_newPythonExtensibleRate(CxxReactionRateDelegator* delegator, + const string& module_name, + const string& class_name): mod = importlib.import_module(module_name.decode()) cdef ExtensibleRate rate = getattr(mod, class_name.decode())(init=False) - Py_INCREF(rate) rate.set_cxx_object(delegator) assign_delegates(rate, delegator) - return 0 + return rate From 0cbe50ca933331529828e3e50ab1ff3fe793498c Mon Sep 17 00:00:00 2001 From: Ray Speth Date: Sat, 27 Aug 2022 20:15:52 -0400 Subject: [PATCH 09/26] Provide better error messages for Python extensions --- src/extensions/PythonExtensionManager.cpp | 43 +++++++++++++++++++++-- src/extensions/pythonExtensions.pyx | 10 ++++++ 2 files changed, 50 insertions(+), 3 deletions(-) diff --git a/src/extensions/PythonExtensionManager.cpp b/src/extensions/PythonExtensionManager.cpp index 9d554f6fe5..6791defad3 100644 --- a/src/extensions/PythonExtensionManager.cpp +++ b/src/extensions/PythonExtensionManager.cpp @@ -14,6 +14,38 @@ namespace ba = boost::algorithm; using namespace std; +namespace { + +std::string getPythonExceptionInfo() +{ + if (!PyErr_Occurred()) { + return "no Python exception raised"; + } + + PyObject* ex_type; + PyObject* ex_value; + PyObject* traceback; + PyErr_Fetch(&ex_type, &ex_value, &traceback); + PyErr_NormalizeException(&ex_type, &ex_value, &traceback); + if (traceback == nullptr) { + traceback = Py_None; + } + char* c_exstr = ct_getExceptionString(ex_type, ex_value, traceback); + string message; + if (c_exstr != nullptr) { + message = c_exstr; + free(c_exstr); + } else { + message = "Couldn't get exception message"; + } + Py_XDECREF(ex_type); + Py_XDECREF(ex_value); + Py_XDECREF(traceback); + return message; +} + +} // end anonymous namespace + namespace Cantera { @@ -27,7 +59,7 @@ PythonExtensionManager::PythonExtensionManager() PyModuleDef* modDef = (PyModuleDef*) PyInit_pythonExtensions(); if (!modDef->m_slots || !PyModuleDef_Init(modDef)) { throw CanteraError("PythonExtensionManager::PythonExtensionManager", - "Failed to import 'pythonExtensions' module"); + "Failed to import 'pythonExtensions' module"); } // Following example creation of minimal ModuleSpec from Python's import.c @@ -40,7 +72,7 @@ PythonExtensionManager::PythonExtensionManager() Py_DECREF(attrs); if (spec == NULL) { throw CanteraError("PythonExtensionManager::PythonExtensionManager", - "_PyNamespace_New failed"); + "_PyNamespace_New failed"); } PyObject* pyModule = PyModule_FromDefAndSpec(modDef, spec); if (!pyModule) { @@ -58,6 +90,10 @@ PythonExtensionManager::PythonExtensionManager() void PythonExtensionManager::registerRateBuilders(const string& extensionName) { char* c_rateTypes = ct_getPythonExtensibleRateTypes(extensionName); + if (c_rateTypes == nullptr) { + throw CanteraError("PythonExtensionManager::registerRateBuilders", + "Problem loading module:\n{}", getPythonExceptionInfo()); + } string rateTypes(c_rateTypes); free(c_rateTypes); @@ -84,7 +120,8 @@ void PythonExtensionManager::registerRateBuilders(const string& extensionName) extensionName, rateName); if (extRate == nullptr) { throw CanteraError("PythonExtensionManager::registerRateBuilders", - "ct_newPythonExtensibleRate failed"); + "Problem in ct_newPythonExtensibleRate:\n{}", + getPythonExceptionInfo()); } // Make the delegator responsible for eventually deleting the Python object diff --git a/src/extensions/pythonExtensions.pyx b/src/extensions/pythonExtensions.pyx index e2a7ff1fa9..3afaa145c5 100644 --- a/src/extensions/pythonExtensions.pyx +++ b/src/extensions/pythonExtensions.pyx @@ -22,6 +22,16 @@ cdef extern from "cantera/kinetics/ReactionRateDelegator.h" namespace "Cantera": CxxReactionRateDelegator() +cdef public char* ct_getExceptionString(object exType, object exValue, object exTraceback): + import traceback + result = str(exValue) + "\n\n" + result += "".join(traceback.format_exception(exType, exValue, exTraceback)) + tmp = bytes(result.encode()) + cdef char* c_string = malloc((len(tmp) + 1) * sizeof(char)) + strcpy(c_string, tmp) + return c_string + + cdef public char* ct_getPythonExtensibleRateTypes(const string& module_name) except NULL: """ Load the named module and find classes derived from ExtensibleRate. From b5a647afd540cb0d425cb54b0df16e88df50466d Mon Sep 17 00:00:00 2001 From: Ray Speth Date: Sat, 27 Aug 2022 21:09:25 -0400 Subject: [PATCH 10/26] Use decorator to register ExtensibleRate objects --- .../extensions/PythonExtensionManager.h | 10 +++ .../cantera/kinetics/ReactionRateDelegator.h | 4 - interfaces/cython/cantera/delegator.pxd | 5 ++ interfaces/cython/cantera/delegator.pyx | 17 +++++ src/extensions/PythonExtensionManager.cpp | 75 +++++++++---------- src/extensions/pythonExtensions.pyx | 18 ----- src/kinetics/ReactionRateFactory.cpp | 5 -- test/data/extensible-reactions.yaml | 14 ++++ test/python/test_reaction.py | 41 +++++++--- test/python/user_ext.py | 6 ++ 10 files changed, 121 insertions(+), 74 deletions(-) create mode 100644 test/data/extensible-reactions.yaml create mode 100644 test/python/user_ext.py diff --git a/include/cantera/extensions/PythonExtensionManager.h b/include/cantera/extensions/PythonExtensionManager.h index 1e5f46418e..a29d0c2fd5 100644 --- a/include/cantera/extensions/PythonExtensionManager.h +++ b/include/cantera/extensions/PythonExtensionManager.h @@ -14,11 +14,21 @@ namespace Cantera //! Class for managing user-defined Cantera extensions written in Python //! //! Handles Python initialization if the main application is not the Python interpreter. +//! +//! Imports a user-specified module, which must be on the Python path and registers +//! user-defined classes that are marked with the `@extension` decorator. See the +//! documentation for +//! `@extension` +//! in the Python documentation for more information. class PythonExtensionManager : public ExtensionManager { public: PythonExtensionManager(); virtual void registerRateBuilders(const std::string& extensionName) override; + + //! Function called from Cython to register an ExtensibleRate implementation + static void registerPythonRateBuilder(const std::string& moduleName, + const std::string& className, const std::string& rateName); }; } diff --git a/include/cantera/kinetics/ReactionRateDelegator.h b/include/cantera/kinetics/ReactionRateDelegator.h index 1fb9ce27f9..e7001f39b8 100644 --- a/include/cantera/kinetics/ReactionRateDelegator.h +++ b/include/cantera/kinetics/ReactionRateDelegator.h @@ -25,10 +25,6 @@ class ReactionRateDelegator : public Delegator, public ReactionRate ); } - ReactionRateDelegator(const AnyMap& node, const UnitStack& rate_units) - : ReactionRateDelegator() - {} - virtual unique_ptr newMultiRate() const override { return unique_ptr( new MultiRate); diff --git a/interfaces/cython/cantera/delegator.pxd b/interfaces/cython/cantera/delegator.pxd index afbb62b254..3bd73f6151 100644 --- a/interfaces/cython/cantera/delegator.pxd +++ b/interfaces/cython/cantera/delegator.pxd @@ -57,6 +57,11 @@ cdef extern from "cantera/cython/funcWrapper.h": PyObject*, int(PyFuncInfo&, size_t&, const string&)) +cdef extern from "cantera/extensions/PythonExtensionManager.h" namespace "Cantera": + cdef cppclass CxxPythonExtensionManager "Cantera::PythonExtensionManager": + @staticmethod + void registerPythonRateBuilder(string&, string&, string&) except +translate_exception + ctypedef CxxDelegator* CxxDelegatorPtr cdef int assign_delegates(object, CxxDelegator*) except -1 diff --git a/interfaces/cython/cantera/delegator.pyx b/interfaces/cython/cantera/delegator.pyx index a97b22ba36..96ee2f92fd 100644 --- a/interfaces/cython/cantera/delegator.pyx +++ b/interfaces/cython/cantera/delegator.pyx @@ -6,6 +6,7 @@ import sys from ._utils import CanteraError from ._utils cimport stringify, pystr +from .reaction import ExtensibleRate from cython.operator import dereference as deref # ## Implementation for each delegated function type @@ -309,3 +310,19 @@ cdef int assign_delegates(obj, CxxDelegator* delegator) except -1: obj._delegates.append(method) return 0 + +def extension(*, name): + """ + A decorator for declaring Cantera extensions that should be registered with + the corresponding factory classes to create objects with the specified *name*. + """ + def decorator(cls): + if issubclass(cls, ExtensibleRate): + cls._reaction_rate_type = name + CxxPythonExtensionManager.registerPythonRateBuilder( + stringify(cls.__module__), stringify(cls.__name__), stringify(name)) + else: + raise TypeError(f"{cls} is not extensible") + return cls + + return decorator diff --git a/src/extensions/PythonExtensionManager.cpp b/src/extensions/PythonExtensionManager.cpp index 6791defad3..d68b860daa 100644 --- a/src/extensions/PythonExtensionManager.cpp +++ b/src/extensions/PythonExtensionManager.cpp @@ -56,6 +56,9 @@ PythonExtensionManager::PythonExtensionManager() } // PEP 489 Multi-phase initialization + + // The 'pythonExtensions' Cython module defines some functions that are used + // to instantiate ExtensibleSomething objects. PyModuleDef* modDef = (PyModuleDef*) PyInit_pythonExtensions(); if (!modDef->m_slots || !PyModuleDef_Init(modDef)) { throw CanteraError("PythonExtensionManager::PythonExtensionManager", @@ -89,49 +92,45 @@ PythonExtensionManager::PythonExtensionManager() void PythonExtensionManager::registerRateBuilders(const string& extensionName) { - char* c_rateTypes = ct_getPythonExtensibleRateTypes(extensionName); - if (c_rateTypes == nullptr) { + // Each rate builder class is decorated with @extension, which calls the + // registerPythonRateBuilder method to register that class. So all we have + // to do here is load the module. + PyObject* module_name = PyUnicode_FromString(extensionName.c_str()); + PyObject* py_module = PyImport_Import(module_name); + Py_DECREF(module_name); + if (py_module == nullptr) { throw CanteraError("PythonExtensionManager::registerRateBuilders", "Problem loading module:\n{}", getPythonExceptionInfo()); } - string rateTypes(c_rateTypes); - free(c_rateTypes); - - // Each line in rateTypes is a (class name, rate name) pair, separated by a tab - vector lines; - ba::split(lines, rateTypes, boost::is_any_of("\n")); - for (auto& line : lines) { - vector tokens; - ba::split(tokens, line, boost::is_any_of("\t")); - if (tokens.size() != 2) { - CanteraError("PythonExtensionManager::registerRateBuilders", - "Got unparsable input from ct_getPythonExtensibleRateTypes:" - "\n'''{}\n'''", rateTypes); +} + +void PythonExtensionManager::registerPythonRateBuilder( + const std::string& moduleName, const std::string& className, + const std::string& rateName) +{ + // Make sure the helper module has been loaded + PythonExtensionManager mgr; + + // Create a function that constructs and links a C++ ReactionRateDelegator + // object and a Python ExtensibleRate object of a particular type, and register + // this as the builder for reactions of this type + auto builder = [moduleName, className](const AnyMap& params, const UnitStack& units) { + auto delegator = make_unique(); + PyObject* extRate = ct_newPythonExtensibleRate(delegator.get(), + moduleName, className); + if (extRate == nullptr) { + throw CanteraError("PythonExtensionManager::registerRateBuilders", + "Problem in ct_newPythonExtensibleRate:\n{}", + getPythonExceptionInfo()); } - string rateName = tokens[0]; - - // Create a function that constructs and links a C++ ReactionRateDelegator - // object and a Python ExtensibleRate object of a particular type, and register - // this as the builder for reactions of this type - auto builder = [rateName, extensionName](const AnyMap& params, const UnitStack& units) { - auto delegator = make_unique(); - PyObject* extRate = ct_newPythonExtensibleRate(delegator.get(), - extensionName, rateName); - if (extRate == nullptr) { - throw CanteraError("PythonExtensionManager::registerRateBuilders", - "Problem in ct_newPythonExtensibleRate:\n{}", - getPythonExceptionInfo()); - } - - // Make the delegator responsible for eventually deleting the Python object - Py_IncRef(extRate); - delegator->addCleanupFunc([extRate]() { Py_DecRef(extRate); }); - - return delegator.release(); - }; - ReactionRateFactory::factory()->reg(tokens[1], builder); - } + // Make the delegator responsible for eventually deleting the Python object + Py_IncRef(extRate); + delegator->addCleanupFunc([extRate]() { Py_DecRef(extRate); }); + + return delegator.release(); + }; + ReactionRateFactory::factory()->reg(rateName, builder); } }; diff --git a/src/extensions/pythonExtensions.pyx b/src/extensions/pythonExtensions.pyx index 3afaa145c5..30577f5155 100644 --- a/src/extensions/pythonExtensions.pyx +++ b/src/extensions/pythonExtensions.pyx @@ -32,24 +32,6 @@ cdef public char* ct_getExceptionString(object exType, object exValue, object ex return c_string -cdef public char* ct_getPythonExtensibleRateTypes(const string& module_name) except NULL: - """ - Load the named module and find classes derived from ExtensibleRate. - - Returns a string where each line contains the class name and the corresponding - rate name, separated by a space - """ - mod = importlib.import_module(module_name.decode()) - names = "\n".join( - f"{name}\t{cls._reaction_rate_type}" - for name, cls in inspect.getmembers(mod) - if inspect.isclass(cls) and issubclass(cls, ct.ExtensibleRate)) - tmp = bytes(names.encode()) - cdef char* c_string = malloc((len(tmp) + 1) * sizeof(char)) - strcpy(c_string, tmp) - return c_string - - cdef public object ct_newPythonExtensibleRate(CxxReactionRateDelegator* delegator, const string& module_name, const string& class_name): diff --git a/src/kinetics/ReactionRateFactory.cpp b/src/kinetics/ReactionRateFactory.cpp index 2c5dc87872..3cd354b368 100644 --- a/src/kinetics/ReactionRateFactory.cpp +++ b/src/kinetics/ReactionRateFactory.cpp @@ -16,7 +16,6 @@ #include "cantera/kinetics/InterfaceRate.h" #include "cantera/kinetics/PlogRate.h" #include "cantera/kinetics/TwoTempPlasmaRate.h" -#include "cantera/kinetics/ReactionRateDelegator.h" namespace Cantera { @@ -99,10 +98,6 @@ ReactionRateFactory::ReactionRateFactory() reg("sticking-Blowers-Masel", [](const AnyMap& node, const UnitStack& rate_units) { return new StickingBlowersMaselRate(node, rate_units); }); - - reg("extensible", [](const AnyMap& node, const UnitStack& rate_units) { - return new ReactionRateDelegator(node, rate_units); - }); } shared_ptr newReactionRate(const std::string& type) diff --git a/test/data/extensible-reactions.yaml b/test/data/extensible-reactions.yaml new file mode 100644 index 0000000000..732c9330a5 --- /dev/null +++ b/test/data/extensible-reactions.yaml @@ -0,0 +1,14 @@ +extensions: +- type: python + name: user_ext + +phases: +- name: gas + thermo: ideal-gas + species: [{h2o2.yaml/species: all}] + kinetics: gas + state: {T: 300.0, P: 1 atm} + +reactions: +- equation: H + O2 = HO2 + type: square-rate diff --git a/test/python/test_reaction.py b/test/python/test_reaction.py index 8f5f0141bd..7e728b2e7d 100644 --- a/test/python/test_reaction.py +++ b/test/python/test_reaction.py @@ -1,5 +1,6 @@ from math import exp from pathlib import Path +import sys import textwrap import cantera as ct @@ -1498,28 +1499,37 @@ def func(T): assert (gas.forward_rate_constants == gas.T).all() +@ct.extension(name="user-rate-1") +class UserRate1(ct.ExtensibleRate): + def replace_eval(self, T): + return 38.7 * T**2.7 * exp(-3150.15428/T) + + class TestExtensible(ReactionTests, utilities.CanteraTest): # test Extensible reaction rate - class UserRate(ct.ExtensibleRate): - def replace_eval(self, T): - return 38.7 * T**2.7 * exp(-3150.15428/T) # probe O + H2 <=> H + OH - _rate_cls = UserRate + _rate_cls = UserRate1 _equation = "H2 + O <=> H + OH" _index = 0 - _rate_type = "extensible" - _yaml = None + _rate_type = "user-rate-1" + _rate = { + "type": "user-rate-1", + } + _yaml = """ + equation: H2 + O <=> H + OH + type: user-rate-1 + """ def setUp(self): super().setUp() - self._rate_obj = self.UserRate() + self._rate_obj = UserRate1() def test_no_rate(self): pytest.skip("ExtensibleRate does not support 'empty' rates") - def from_yaml(self): - pytest.skip("ExtensibleRate does not support YAML") + def test_from_dict(self): + pytest.skip("ExtensibleRate does not support serialization") def from_rate(self, rate): pytest.skip("ExtensibleRate does not support dict-based instantiation") @@ -1528,6 +1538,19 @@ def test_roundtrip(self): pytest.skip("ExtensibleRate does not support roundtrip conversion") +class TestExtensible2(utilities.CanteraTest): + def test_load_module(self): + here = str(Path(__file__).parent) + if here not in sys.path: + sys.path.append(here) + + gas = ct.Solution("extensible-reactions.yaml", transport_model=None) + + for T in np.linspace(300, 3000, 10): + gas.TP = T, None + assert gas.forward_rate_constants[0] == pytest.approx(T**2) + + class InterfaceReactionTests(ReactionTests): # test suite for surface reaction expressions diff --git a/test/python/user_ext.py b/test/python/user_ext.py new file mode 100644 index 0000000000..6ae29a3550 --- /dev/null +++ b/test/python/user_ext.py @@ -0,0 +1,6 @@ +import cantera as ct + +@ct.extension(name="square-rate") +class SquareRate(ct.ExtensibleRate): + def replace_eval(self, T): + return T**2 From 0132c48429fee5aa8f4103f24a46e3bbab39ef1a Mon Sep 17 00:00:00 2001 From: Ray Speth Date: Sun, 28 Aug 2022 22:56:23 -0400 Subject: [PATCH 11/26] Implement delegation of ReactionRate::setParameters --- include/cantera/base/Delegator.h | 31 +++++++++++++++++++ .../cantera/kinetics/ReactionRateDelegator.h | 8 +++++ interfaces/cython/cantera/delegator.pxd | 4 +++ interfaces/cython/cantera/delegator.pyx | 19 +++++++++++- interfaces/cython/cantera/reaction.pyx | 5 ++- interfaces/cython/cantera/units.pxd | 4 +++ src/extensions/PythonExtensionManager.cpp | 2 ++ src/extensions/pythonExtensions.pyx | 1 - test/data/extensible-reactions.yaml | 1 + test/python/test_reaction.py | 24 ++++++++------ test/python/user_ext.py | 5 ++- 11 files changed, 91 insertions(+), 13 deletions(-) diff --git a/include/cantera/base/Delegator.h b/include/cantera/base/Delegator.h index d220052294..b6fa33ae2f 100644 --- a/include/cantera/base/Delegator.h +++ b/include/cantera/base/Delegator.h @@ -140,6 +140,21 @@ class Delegator *m_funcs_v_d[name] = makeDelegate(func, when, *m_funcs_v_d[name]); } + //! set delegates for member functions with the signature + //! `void(AnyMap&, UnitStack&)` + void setDelegate(const std::string& name, + const std::function& func, + const std::string& when) + { + if (!m_funcs_v_cAMr_cUSr.count(name)) { + throw NotImplementedError("Delegator::setDelegate", + "for function named '{}' with signature " + "'void(const AnyMap&, const UnitStack&)'.", + name); + } + *m_funcs_v_cAMr_cUSr[name] = makeDelegate(func, when, *m_funcs_v_cAMr_cUSr[name]); + } + //! Set delegates for member functions with the signature `void(double*)` void setDelegate(const std::string& name, const std::function, double*)>& func, @@ -263,6 +278,18 @@ class Delegator m_funcs_v_d[name] = ⌖ } + //! Install a function with the signature `void(const AnyMap&, const UnitStack&)` + //! as being delegatable + void install(const std::string& name, + std::function& target, + const std::function& func) + { + target = func; + m_funcs_v_cAMr_cUSr[name] = ⌖ + } + + + //! Install a function with the signature `void(double*)` as being delegatable void install(const std::string& name, std::function, double*)>& target, @@ -425,6 +452,8 @@ class Delegator //! - `d` for `double` //! - `s` for `std::string` //! - `sz` for `size_t` + //! - `AM` for `AnyMap` + //! - `US` for `UnitStack` //! - prefix `c` for `const` arguments //! - suffix `r` for reference arguments //! - suffix `p` for pointer arguments @@ -434,6 +463,8 @@ class Delegator std::map*> m_funcs_v; std::map*> m_funcs_v_b; std::map*> m_funcs_v_d; + std::map*> m_funcs_v_cAMr_cUSr; std::map, double*)>*> m_funcs_v_dp; std::map newMultiRate() const override { @@ -43,8 +46,13 @@ class ReactionRateDelegator : public Delegator, public ReactionRate return m_evalFromStruct(&T); } + void setParameters(const AnyMap& node, const UnitStack& units) override { + m_setParameters(node, units); + } + private: std::function m_evalFromStruct; + std::function m_setParameters; }; } diff --git a/interfaces/cython/cantera/delegator.pxd b/interfaces/cython/cantera/delegator.pxd index 3bd73f6151..98711df968 100644 --- a/interfaces/cython/cantera/delegator.pxd +++ b/interfaces/cython/cantera/delegator.pxd @@ -6,6 +6,7 @@ from .ctcxx cimport * from .func1 cimport * +from .units cimport CxxUnitStack cdef extern from "" namespace "std" nogil: cdef cppclass size_array1 "std::array": @@ -28,6 +29,7 @@ cdef extern from "cantera/base/Delegator.h" namespace "Cantera": void setDelegate(string&, function[void()], string&) except +translate_exception void setDelegate(string&, function[void(cbool)], string&) except +translate_exception void setDelegate(string&, function[void(double)], string&) except +translate_exception + void setDelegate(string&, function[void(const CxxAnyMap&, const CxxUnitStack&)], string&) except +translate_exception void setDelegate(string&, function[void(size_array1, double*)], string&) except +translate_exception void setDelegate(string&, function[void(size_array1, double, double*)], string&) except +translate_exception void setDelegate(string&, function[void(size_array2, double, double*, double*)], string&) except +translate_exception @@ -43,6 +45,8 @@ cdef extern from "cantera/cython/funcWrapper.h": cdef function[void(double)] pyOverride(PyObject*, void(PyFuncInfo&, double)) cdef function[void(cbool)] pyOverride(PyObject*, void(PyFuncInfo&, cbool)) cdef function[void()] pyOverride(PyObject*, void(PyFuncInfo&)) + cdef function[void(const CxxAnyMap&, const CxxUnitStack&)] pyOverride( + PyObject*, void(PyFuncInfo&, const CxxAnyMap&, const CxxUnitStack&)) cdef function[void(size_array1, double*)] pyOverride( PyObject*, void(PyFuncInfo&, size_array1, double*)) cdef function[void(size_array1, double, double*)] pyOverride( diff --git a/interfaces/cython/cantera/delegator.pyx b/interfaces/cython/cantera/delegator.pyx index 96ee2f92fd..9189c12e7f 100644 --- a/interfaces/cython/cantera/delegator.pyx +++ b/interfaces/cython/cantera/delegator.pyx @@ -5,7 +5,8 @@ import inspect import sys from ._utils import CanteraError -from ._utils cimport stringify, pystr +from ._utils cimport stringify, pystr, anymap_to_dict +from .units cimport Units from .reaction import ExtensibleRate from cython.operator import dereference as deref @@ -103,6 +104,19 @@ cdef void callback_v_b(PyFuncInfo& funcInfo, cbool arg): funcInfo.setExceptionType(exc_type) funcInfo.setExceptionValue(exc_value) +# Wrapper for functions of type void(const AnyMap&, const UnitStack&) +cdef void callback_v_cAMr_cUSr(PyFuncInfo& funcInfo, const CxxAnyMap& arg1, + const CxxUnitStack& arg2): + + pyArg1 = anymap_to_dict(arg1) # cast away constness + pyArg2 = Units.copy(arg2.product()) + try: + (funcInfo.func())(pyArg1, pyArg2) + except BaseException as e: + exc_type, exc_value = sys.exc_info()[:2] + funcInfo.setExceptionType(exc_type) + funcInfo.setExceptionValue(exc_value) + # Wrapper for functions of type void(double*) cdef void callback_v_dp(PyFuncInfo& funcInfo, size_array1 sizes, double* arg): cdef double[:] view = arg if sizes[0] else None @@ -276,6 +290,9 @@ cdef int assign_delegates(obj, CxxDelegator* delegator) except -1: elif callback == 'void(double)': delegator.setDelegate(cxx_name, pyOverride(method, callback_v_d), cxx_when) + elif callback == 'void(AnyMap&,UnitStack&)': + delegator.setDelegate(cxx_name, + pyOverride(method, callback_v_cAMr_cUSr), cxx_when) elif callback == 'void(double*)': delegator.setDelegate(cxx_name, pyOverride(method, callback_v_dp), cxx_when) diff --git a/interfaces/cython/cantera/reaction.pyx b/interfaces/cython/cantera/reaction.pyx index 04492ec073..c93d59ed53 100644 --- a/interfaces/cython/cantera/reaction.pyx +++ b/interfaces/cython/cantera/reaction.pyx @@ -710,11 +710,13 @@ cdef class CustomRate(ReactionRate): self.cxx_object().setRateFunction(self._rate_func._func) + cdef class ExtensibleRate(ReactionRate): _reaction_rate_type = "extensible" delegatable_methods = { - "eval": ("evalFromStruct", "double(void*)") + "eval": ("evalFromStruct", "double(void*)"), + "set_parameters": ("setParameters", "void(AnyMap&, UnitStack&)") } def __cinit__(self, *args, init=True, **kwargs): if init: @@ -734,6 +736,7 @@ cdef class ExtensibleRate(ReactionRate): self.rate = rate assign_delegates(self, dynamic_cast[CxxDelegatorPtr](self.rate)) + cdef class InterfaceRateBase(ArrheniusRateBase): """ Base class collecting commonly used features of Arrhenius-type rate objects diff --git a/interfaces/cython/cantera/units.pxd b/interfaces/cython/cantera/units.pxd index 79267ad7d2..04e364fcaa 100644 --- a/interfaces/cython/cantera/units.pxd +++ b/interfaces/cython/cantera/units.pxd @@ -23,6 +23,10 @@ cdef extern from "cantera/base/Units.h" namespace "Cantera": stdmap[string, string] defaults() void setDefaults(stdmap[string, string]&) except +translate_exception + cdef cppclass CxxUnitStack "Cantera::UnitStack": + CxxUnitStack() + CxxUnits product() + cdef class Units: cdef CxxUnits units diff --git a/src/extensions/PythonExtensionManager.cpp b/src/extensions/PythonExtensionManager.cpp index d68b860daa..40132d48e3 100644 --- a/src/extensions/PythonExtensionManager.cpp +++ b/src/extensions/PythonExtensionManager.cpp @@ -123,6 +123,8 @@ void PythonExtensionManager::registerPythonRateBuilder( "Problem in ct_newPythonExtensibleRate:\n{}", getPythonExceptionInfo()); } + //! Call setParameters after the delegated functions have been connected + delegator->setParameters(params, units); // Make the delegator responsible for eventually deleting the Python object Py_IncRef(extRate); diff --git a/src/extensions/pythonExtensions.pyx b/src/extensions/pythonExtensions.pyx index 30577f5155..6facd1b34f 100644 --- a/src/extensions/pythonExtensions.pyx +++ b/src/extensions/pythonExtensions.pyx @@ -39,5 +39,4 @@ cdef public object ct_newPythonExtensibleRate(CxxReactionRateDelegator* delegato mod = importlib.import_module(module_name.decode()) cdef ExtensibleRate rate = getattr(mod, class_name.decode())(init=False) rate.set_cxx_object(delegator) - assign_delegates(rate, delegator) return rate diff --git a/test/data/extensible-reactions.yaml b/test/data/extensible-reactions.yaml index 732c9330a5..8b469ac6af 100644 --- a/test/data/extensible-reactions.yaml +++ b/test/data/extensible-reactions.yaml @@ -12,3 +12,4 @@ phases: reactions: - equation: H + O2 = HO2 type: square-rate + A: 3.14 diff --git a/test/python/test_reaction.py b/test/python/test_reaction.py index 7e728b2e7d..d362168fe4 100644 --- a/test/python/test_reaction.py +++ b/test/python/test_reaction.py @@ -1501,8 +1501,15 @@ def func(T): @ct.extension(name="user-rate-1") class UserRate1(ct.ExtensibleRate): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.A = np.nan + + def after_set_parameters(self, params, units): + self.A = params["A"] + def replace_eval(self, T): - return 38.7 * T**2.7 * exp(-3150.15428/T) + return self.A * T**2.7 * exp(-3150.15428/T) class TestExtensible(ReactionTests, utilities.CanteraTest): @@ -1515,27 +1522,26 @@ class TestExtensible(ReactionTests, utilities.CanteraTest): _rate_type = "user-rate-1" _rate = { "type": "user-rate-1", + "A": 38.7 } _yaml = """ equation: H2 + O <=> H + OH type: user-rate-1 + A: 38.7 """ def setUp(self): super().setUp() - self._rate_obj = UserRate1() + self._rate_obj = ct.ReactionRate.from_dict(self._rate) def test_no_rate(self): - pytest.skip("ExtensibleRate does not support 'empty' rates") + pytest.skip("ExtensibleRate does not yet support validation") def test_from_dict(self): - pytest.skip("ExtensibleRate does not support serialization") - - def from_rate(self, rate): - pytest.skip("ExtensibleRate does not support dict-based instantiation") + pytest.skip("ExtensibleRate does not yet support serialization") def test_roundtrip(self): - pytest.skip("ExtensibleRate does not support roundtrip conversion") + pytest.skip("ExtensibleRate does not yet support roundtrip conversion") class TestExtensible2(utilities.CanteraTest): @@ -1548,7 +1554,7 @@ def test_load_module(self): for T in np.linspace(300, 3000, 10): gas.TP = T, None - assert gas.forward_rate_constants[0] == pytest.approx(T**2) + assert gas.forward_rate_constants[0] == pytest.approx(3.14 * T**2) class InterfaceReactionTests(ReactionTests): diff --git a/test/python/user_ext.py b/test/python/user_ext.py index 6ae29a3550..487b646ca9 100644 --- a/test/python/user_ext.py +++ b/test/python/user_ext.py @@ -2,5 +2,8 @@ @ct.extension(name="square-rate") class SquareRate(ct.ExtensibleRate): + def after_set_parameters(self, node, units): + self.A = node["A"] + def replace_eval(self, T): - return T**2 + return self.A * T**2 From 7a0ca240476f55377e8819b05d19dd123ad801d2 Mon Sep 17 00:00:00 2001 From: Ray Speth Date: Sun, 28 Aug 2022 23:40:27 -0400 Subject: [PATCH 12/26] Handle sharing of ExtensibleRate among copies of ReactionRateDelegator --- include/cantera/base/Delegator.h | 13 ++++--------- include/cantera/base/ExtensionManager.h | 8 ++++++++ src/extensions/PythonExtensionManager.cpp | 20 ++++++++++++++++---- 3 files changed, 28 insertions(+), 13 deletions(-) diff --git a/include/cantera/base/Delegator.h b/include/cantera/base/Delegator.h index b6fa33ae2f..b8a0eed334 100644 --- a/include/cantera/base/Delegator.h +++ b/include/cantera/base/Delegator.h @@ -8,6 +8,7 @@ #include "cantera/base/global.h" #include "cantera/base/ctexceptions.h" +#include "cantera/base/ExtensionManager.h" #include #include @@ -101,12 +102,6 @@ namespace Cantera class Delegator { public: - ~Delegator() { - for (auto& func : m_cleanup_funcs) { - func(); - } - } - //! Set delegates for member functions with the signature `void()`. void setDelegate(const std::string& name, const std::function& func, const std::string& when) @@ -249,8 +244,8 @@ class Delegator *m_funcs_sz_csr[name] = makeDelegate(func, when, m_base_sz_csr[name]); } - void addCleanupFunc(const std::function& func) { - m_cleanup_funcs.push_back(func); + void holdExternalHandle(const shared_ptr& handle) { + m_handles.push_back(handle); } protected: @@ -490,7 +485,7 @@ class Delegator //! @} //! Cleanup functions to be called from the destructor - std::list> m_cleanup_funcs; + std::list> m_handles; }; } diff --git a/include/cantera/base/ExtensionManager.h b/include/cantera/base/ExtensionManager.h index 4bee0b25f3..1f41d96883 100644 --- a/include/cantera/base/ExtensionManager.h +++ b/include/cantera/base/ExtensionManager.h @@ -24,6 +24,14 @@ class ExtensionManager }; }; +//! A base class for managing the lifetime of an external object, such as a Python +//! object used by a Delegator +class ExternalHandle +{ +public: + virtual ~ExternalHandle() = default; +}; + } #endif diff --git a/src/extensions/PythonExtensionManager.cpp b/src/extensions/PythonExtensionManager.cpp index 40132d48e3..0abd3e86d9 100644 --- a/src/extensions/PythonExtensionManager.cpp +++ b/src/extensions/PythonExtensionManager.cpp @@ -16,6 +16,20 @@ using namespace std; namespace { +class PythonHandle : public Cantera::ExternalHandle +{ +public: + explicit PythonHandle(PyObject* obj) : m_obj(obj) {} + + ~PythonHandle() { + Py_XDECREF(m_obj); + } + +private: + PyObject* m_obj; +}; + + std::string getPythonExceptionInfo() { if (!PyErr_Occurred()) { @@ -126,10 +140,8 @@ void PythonExtensionManager::registerPythonRateBuilder( //! Call setParameters after the delegated functions have been connected delegator->setParameters(params, units); - // Make the delegator responsible for eventually deleting the Python object - Py_IncRef(extRate); - delegator->addCleanupFunc([extRate]() { Py_DecRef(extRate); }); - + // The delegator is responsible for eventually deleting the Python object + delegator->holdExternalHandle(make_shared(extRate)); return delegator.release(); }; ReactionRateFactory::factory()->reg(rateName, builder); From bc4685c66c46427fde7ba9d758404a53778308d9 Mon Sep 17 00:00:00 2001 From: Ray Speth Date: Mon, 29 Aug 2022 18:45:40 -0400 Subject: [PATCH 13/26] Provide user-defined name for ExtensibleRate objects --- include/cantera/kinetics/ReactionRateDelegator.h | 7 ++++++- interfaces/cython/cantera/reaction.pxd | 1 + interfaces/cython/cantera/reaction.pyx | 2 ++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/include/cantera/kinetics/ReactionRateDelegator.h b/include/cantera/kinetics/ReactionRateDelegator.h index a330872b6d..c2dbae89e2 100644 --- a/include/cantera/kinetics/ReactionRateDelegator.h +++ b/include/cantera/kinetics/ReactionRateDelegator.h @@ -33,8 +33,12 @@ class ReactionRateDelegator : public Delegator, public ReactionRate new MultiRate); } + void setType(const std::string& type) { + m_rateType = type; + } + virtual const std::string type() const override { - return "ReactionRateDelegator"; + return m_rateType; } // Delegatable methods @@ -51,6 +55,7 @@ class ReactionRateDelegator : public Delegator, public ReactionRate } private: + std::string m_rateType; std::function m_evalFromStruct; std::function m_setParameters; }; diff --git a/interfaces/cython/cantera/reaction.pxd b/interfaces/cython/cantera/reaction.pxd index fde60b5e39..c1afedcf19 100644 --- a/interfaces/cython/cantera/reaction.pxd +++ b/interfaces/cython/cantera/reaction.pxd @@ -191,6 +191,7 @@ cdef extern from "cantera/kinetics/Custom.h" namespace "Cantera": cdef extern from "cantera/kinetics/ReactionRateDelegator.h" namespace "Cantera": cdef cppclass CxxReactionRateDelegator "Cantera::ReactionRateDelegator" (CxxReactionRate): CxxReactionRateDelegator() + void setType(string&) cdef extern from "cantera/kinetics/InterfaceRate.h" namespace "Cantera": diff --git a/interfaces/cython/cantera/reaction.pyx b/interfaces/cython/cantera/reaction.pyx index c93d59ed53..2815bb546c 100644 --- a/interfaces/cython/cantera/reaction.pyx +++ b/interfaces/cython/cantera/reaction.pyx @@ -735,6 +735,8 @@ cdef class ExtensibleRate(ReactionRate): self._rate.reset() self.rate = rate assign_delegates(self, dynamic_cast[CxxDelegatorPtr](self.rate)) + (self.rate).setType( + stringify(self._reaction_rate_type)) cdef class InterfaceRateBase(ArrheniusRateBase): From 3c6d6621c9c57eccef9435e725c4e392db47397a Mon Sep 17 00:00:00 2001 From: Ray Speth Date: Mon, 29 Aug 2022 18:46:07 -0400 Subject: [PATCH 14/26] Add C++ test for Python ExtensibleRate --- test/SConscript | 3 +++ test/kinetics/kineticsFromYaml.cpp | 13 +++++++++++++ 2 files changed, 16 insertions(+) diff --git a/test/SConscript b/test/SConscript index ba9ab4c584..0299664b31 100644 --- a/test/SConscript +++ b/test/SConscript @@ -39,6 +39,9 @@ localenv['ENV']['CANTERA_DATA'] = (Dir('#build/data').abspath + os.pathsep + Dir('#samples/data').abspath + os.pathsep + Dir('#test/data').abspath) +# For Python model extensions +localenv.PrependENVPath('PYTHONPATH', Dir('#test/python').abspath) + PASSED_FILES = {} # Add build/lib in order to find Cantera shared library diff --git a/test/kinetics/kineticsFromYaml.cpp b/test/kinetics/kineticsFromYaml.cpp index 1b410532ea..f5e2970f86 100644 --- a/test/kinetics/kineticsFromYaml.cpp +++ b/test/kinetics/kineticsFromYaml.cpp @@ -513,6 +513,19 @@ TEST(Reaction, TwoTempPlasmaFromYaml) EXPECT_DOUBLE_EQ(rate->activationElectronEnergy(), 700 * GasConstant); } +TEST(Reaction, PythonExtensibleRate) +{ + #ifndef CT_HAS_PYTHON + GTEST_SKIP(); + #endif + auto sol = newSolution("extensible-reactions.yaml"); + auto R = sol->kinetics()->reaction(0); + EXPECT_EQ(R->type(), "square-rate"); + auto rate = R->rate(); + EXPECT_EQ(rate->type(), "square-rate"); + EXPECT_DOUBLE_EQ(rate->eval(300), 3.14 * 300 * 300); +} + TEST(Kinetics, GasKineticsFromYaml1) { AnyMap infile = AnyMap::fromYamlFile("ideal-gas.yaml"); From 5bbd17fb0b7dca4d8cc1b97f90847ffffd939f2e Mon Sep 17 00:00:00 2001 From: Ray Speth Date: Tue, 30 Aug 2022 15:41:31 -0400 Subject: [PATCH 15/26] Update custom_reaction example to also show ExtensibleRate --- samples/python/kinetics/custom_reactions.py | 35 ++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/samples/python/kinetics/custom_reactions.py b/samples/python/kinetics/custom_reactions.py index 51940fab0e..5eaa16781f 100644 --- a/samples/python/kinetics/custom_reactions.py +++ b/samples/python/kinetics/custom_reactions.py @@ -30,6 +30,31 @@ gas1 = ct.Solution(thermo='ideal-gas', kinetics='gas', species=species, reactions=custom_reactions) +# construct reactions based on ExtensibleRate: replace 2nd reaction with equivalent +# ExtensibleRate +@ct.extension(name="extensible-Arrhenius") +class ExtensibleArrhenius(ct.ExtensibleRate): + def after_set_parameters(self, params, units): + self.A = params["A"] + self.b = params["b"] + self.Ea_R = params["Ea_R"] + + def replace_eval(self, T): + return self.A * T**self.b * exp(-self.Ea_R/T) + +extensible_yaml = """ + equation: H2 + O <=> H + OH + type: extensible-Arrhenius + A: 38.7 + b: 2.7 + Ea_R: 3150.15428 + """ + +extensible_reactions = gas0.reactions() +extensible_reactions[2] = ct.Reaction.from_yaml(extensible_yaml, gas0) +gas2 = ct.Solution(thermo="ideal-gas", kinetics="gas", + species=species, reactions=extensible_reactions) + # construct test case - simulate ignition def ignition(gas): @@ -64,6 +89,14 @@ def ignition(gas): for i in range(repeat): sim1 += ignition(gas1) sim1 /= repeat -print('- One Python reaction: ' +print('- One Custom reaction: ' '{0:.2f} ms (T_final={1:.2f}) ... ' '{2:+.2f}%'.format(sim1, gas1.T, 100 * sim1 / sim0 - 100)) + +sim2 = 0 +for i in range(repeat): + sim2 += ignition(gas2) +sim2 /= repeat +print('- One Extensible reaction: ' + '{0:.2f} ms (T_final={1:.2f}) ... ' + '{2:+.2f}%'.format(sim2, gas2.T, 100 * sim2 / sim0 - 100)) From b0b1e93fabd647983d5aaa8bba549a19024481e0 Mon Sep 17 00:00:00 2001 From: Ray Speth Date: Tue, 30 Aug 2022 18:07:29 -0400 Subject: [PATCH 16/26] Add more documentation for ExtensibleRate --- doc/sphinx/cython/kinetics.rst | 5 +++ doc/sphinx/cython/utilities.rst | 2 ++ .../cantera/kinetics/ReactionRateDelegator.h | 5 +++ interfaces/cython/cantera/delegator.pyx | 35 +++++++++++++++++++ interfaces/cython/cantera/reaction.pyx | 34 ++++++++++++++++++ interfaces/cython/cantera/reactor.pyx | 2 +- 6 files changed, 82 insertions(+), 1 deletion(-) diff --git a/doc/sphinx/cython/kinetics.rst b/doc/sphinx/cython/kinetics.rst index c3ac82816c..96b4068d5f 100644 --- a/doc/sphinx/cython/kinetics.rst +++ b/doc/sphinx/cython/kinetics.rst @@ -116,6 +116,11 @@ CustomRate .. autoclass:: CustomRate(k) :no-undoc-members: +ExtensibleRate +^^^^^^^^^^^^^^ +.. autoclass:: ExtensibleRate() + :no-undoc-members: + InterfaceRateBase ^^^^^^^^^^^^^^^^^ .. autoclass:: InterfaceRateBase diff --git a/doc/sphinx/cython/utilities.rst b/doc/sphinx/cython/utilities.rst index b78564b2e2..9fbad19f67 100644 --- a/doc/sphinx/cython/utilities.rst +++ b/doc/sphinx/cython/utilities.rst @@ -43,6 +43,8 @@ Global Functions .. autofunction:: debug_mode_enabled .. autofunction:: add_module_directory +.. autofunction:: extension(name: str) + Exceptions ---------- diff --git a/include/cantera/kinetics/ReactionRateDelegator.h b/include/cantera/kinetics/ReactionRateDelegator.h index c2dbae89e2..bd68b4b449 100644 --- a/include/cantera/kinetics/ReactionRateDelegator.h +++ b/include/cantera/kinetics/ReactionRateDelegator.h @@ -13,6 +13,7 @@ namespace Cantera { +//! Delegate methods of the ReactionRate class to external functions class ReactionRateDelegator : public Delegator, public ReactionRate { public: @@ -33,6 +34,7 @@ class ReactionRateDelegator : public Delegator, public ReactionRate new MultiRate); } + //! Set the reaction type based on the user-provided reaction rate parameterization void setType(const std::string& type) { m_rateType = type; } @@ -43,6 +45,9 @@ class ReactionRateDelegator : public Delegator, public ReactionRate // Delegatable methods + //! Evaluate reaction rate + //! + //! @param shared_data data shared by all reactions of a given type double evalFromStruct(const ArrheniusData& shared_data) { // @TODO: replace passing pointer to temperature with a language-specific // wrapper of the ReactionData object diff --git a/interfaces/cython/cantera/delegator.pyx b/interfaces/cython/cantera/delegator.pyx index 9189c12e7f..ec80273a28 100644 --- a/interfaces/cython/cantera/delegator.pyx +++ b/interfaces/cython/cantera/delegator.pyx @@ -332,6 +332,41 @@ def extension(*, name): """ A decorator for declaring Cantera extensions that should be registered with the corresponding factory classes to create objects with the specified *name*. + + This decorator can be used in combination with an ``extensions`` section in a YAML + input file to trigger registration of extensions marked with this decorator, + For example, consider an input file containing top level ``extensions`` and + ``reactions`` sections such as: + + .. code:: yaml + + extensions: + - type: python + name: my_cool_module + + ... # phases and species sections + + reactions: + - equation: O + H2 <=> H + OH # Reaction 3 + type: cool-rate + A: 3.87e+04 + b: 2.7 + Ea: 6260.0 + + and a Python module ``my_cool_module.py``:: + + import cantera as ct + + @ct.extension(name="cool-rate") + class CoolRate(ct.ExtensibleRate): + def after_set_parameters(self, params, units): + ... + def replace_eval(self, T): + ... + + Loading this input file from any Cantera user interface would cause Cantera to load + the ``my_cool_module.py`` module and register the ``CoolRate`` class to handle + reactions whose ``type`` in the YAML file is set to ``cool-rate``. """ def decorator(cls): if issubclass(cls, ExtensibleRate): diff --git a/interfaces/cython/cantera/reaction.pyx b/interfaces/cython/cantera/reaction.pyx index 2815bb546c..40f3ecc786 100644 --- a/interfaces/cython/cantera/reaction.pyx +++ b/interfaces/cython/cantera/reaction.pyx @@ -712,6 +712,40 @@ cdef class CustomRate(ReactionRate): cdef class ExtensibleRate(ReactionRate): + """ + A base class for a reaction rate with delegated methods. Classes derived from this + class should be decorated with the `extension` decorator to specify the name + of the rate parameterization and to make these rates constructible through factory + functions and input files. + + The following methods of the C++ :ct:`ReactionRate` class can be modified by a + Python class that inherits from this class. For each method, the name below should + be prefixed with ``before_``, ``after_``, or ``replace_``, indicating whether this + method should be called before, after, or instead of the corresponding method from + the base class. + + For methods that return a value and have a ``before`` method specified, if that + method returns a value other than ``None`` that value will be returned without + calling the base class method; otherwise, the value from the base class method will + be returned. For methods that return a value and have an ``after`` method specified, + the returned value wil be the sum of the values from the supplied method and the + base class method. + + ``set_parameters(self, params: dict, units: Units) -> None`` + Responsible for setting rate parameters based on the input data. For example, + for reactions created from YAML, ``params`` is the YAML reaction entry converted + to a ``dict``. ``units`` specifies the units of the rate coefficient. + + ``eval(self, T: float) -> float`` + Responsible for calculating the forward rate constant based on the current state + of the phase. This method must *replace* the base class method, as there is no + base class implementation. Currently, the state information provided is the + temperature, ``T`` [K]. + + **Warning:** The delegatable methods defined here are an experimental part of the + Cantera API and may change without notice. + """ + _reaction_rate_type = "extensible" delegatable_methods = { diff --git a/interfaces/cython/cantera/reactor.pyx b/interfaces/cython/cantera/reactor.pyx index 149701e598..8c9e06ae63 100644 --- a/interfaces/cython/cantera/reactor.pyx +++ b/interfaces/cython/cantera/reactor.pyx @@ -468,7 +468,7 @@ cdef class ExtensibleReactor(Reactor): The following methods of the C++ :ct:`Reactor` class can be modified by a Python class which inherits from this class. For each method, the name below should be prefixed with ``before_``, ``after_``, or ``replace_``, indicating - whether the this method should be called before, after, or instead of the + whether this method should be called before, after, or instead of the corresponding method from the base class. For methods that return a value and have a ``before`` method specified, if From dc8bff0c6f124e57c008f43f7491830deda31b61 Mon Sep 17 00:00:00 2001 From: Ray Speth Date: Tue, 30 Aug 2022 22:21:29 -0400 Subject: [PATCH 17/26] Add library directory containing libpython3.x --- site_scons/buildutils.py | 5 +++++ src/SConscript | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/site_scons/buildutils.py b/site_scons/buildutils.py index eb6271df74..82c7ee24b5 100644 --- a/site_scons/buildutils.py +++ b/site_scons/buildutils.py @@ -1297,6 +1297,11 @@ def setup_python_env(env): env["py_plat"] = plat env["site_packages"] = info["site_packages"] env["user_site_packages"] = info["user_site_packages"] + if env["OS"] != "Windows": + env["py_libpl"] = info["LIBPL"] + else: + env["py_libpl"] = info["installed_base"] + "\\libs" + env["py_libs"] = info.get("LIBS", "") # Don't print deprecation warnings for internal Python changes. # Only applies to Python 3.8. The field that is deprecated in Python 3.8 diff --git a/src/SConscript b/src/SConscript index 275fc1a8c0..0d0e9165a7 100644 --- a/src/SConscript +++ b/src/SConscript @@ -91,7 +91,7 @@ if env["python_package"] == "full": libraryTargets.append(obj) libraryTargets.append(pyenv.SharedObject("extensions/PythonExtensionManager.cpp")) libpython = pyenv.subst("python${py_version_short}") - localenv.Append(LIBS=libpython) + localenv.Append(LIBS=libpython, LIBPATH=pyenv["py_libpl"], LDFLAGS=pyenv["py_libs"]) env["cantera_libs"].append(libpython) From eb2823ebc124d8bc188daa6fbe8a949a6865a9c5 Mon Sep 17 00:00:00 2001 From: Ray Speth Date: Tue, 30 Aug 2022 22:10:12 -0400 Subject: [PATCH 18/26] Get ExtensibleRate to work on Windows GitHub Actions --- site_scons/buildutils.py | 11 ++++++++--- src/SConscript | 9 +++++---- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/site_scons/buildutils.py b/site_scons/buildutils.py index 82c7ee24b5..c1b2a1748e 100644 --- a/site_scons/buildutils.py +++ b/site_scons/buildutils.py @@ -1260,11 +1260,13 @@ def setup_python_env(env): import numpy import json import site + import sys vars = get_config_vars() vars["plat"] = get_platform() vars["numpy_include"] = numpy.get_include() vars["site_packages"] = [d for d in site.getsitepackages() if d.endswith("-packages")] vars["user_site_packages"] = site.getusersitepackages() + vars["abiflags"] = getattr(sys, "abiflags", "") print(json.dumps(vars)) """) _python_info = json.loads(get_command_output(env["python_cmd"], "-c", script)) @@ -1298,10 +1300,13 @@ def setup_python_env(env): env["site_packages"] = info["site_packages"] env["user_site_packages"] = info["user_site_packages"] if env["OS"] != "Windows": - env["py_libpl"] = info["LIBPL"] + env["py_libpath"] = info["LIBPL"] + py_lib = "python" + info["py_version_short"] + info["abiflags"] else: - env["py_libpl"] = info["installed_base"] + "\\libs" - env["py_libs"] = info.get("LIBS", "") + env["py_libpath"] = info["installed_base"] + "\\libs" + py_lib = "python" + py_version_nodot + env["py_libs"] = [py_lib] + [lib[2:] for lib in info.get("LIBS", "").split() + if lib.startswith("-l")] # Don't print deprecation warnings for internal Python changes. # Only applies to Python 3.8. The field that is deprecated in Python 3.8 diff --git a/src/SConscript b/src/SConscript index 0d0e9165a7..f43f4722ad 100644 --- a/src/SConscript +++ b/src/SConscript @@ -75,6 +75,7 @@ for subdir, extensions, setup in libs: if env["python_package"] == "full": pyenv = setup_python_env(localenv.Clone()) pyenv.PrependENVPath('PYTHONPATH', Dir('#interfaces/cython').abspath) + pyenv['PCH'] = '' # ignore precompiled header here build_dir = Path(Dir('#build').abspath).as_posix() cythonized = pyenv.Command( ["extensions/pythonExtensions.cpp", "extensions/pythonExtensions.h"], @@ -90,10 +91,10 @@ if env["python_package"] == "full": Copy('$TARGET', '$SOURCE')) libraryTargets.append(obj) libraryTargets.append(pyenv.SharedObject("extensions/PythonExtensionManager.cpp")) - libpython = pyenv.subst("python${py_version_short}") - localenv.Append(LIBS=libpython, LIBPATH=pyenv["py_libpl"], LDFLAGS=pyenv["py_libs"]) - env["cantera_libs"].append(libpython) - + localenv.Append(LIBS=pyenv["py_libs"], LIBPATH=pyenv["py_libpath"]) + env["cantera_libs"].extend(pyenv["py_libs"]) + env.Append(LIBPATH=pyenv["py_libpath"]) + env["extra_lib_dirs"].append(pyenv["py_libpath"]) # build the Cantera static library lib = build(localenv.StaticLibrary('../lib/cantera', libraryTargets, From 68a8758f19ed1529fc227e09a59c667aeee71b11 Mon Sep 17 00:00:00 2001 From: Ray Speth Date: Sun, 4 Sep 2022 13:42:28 -0400 Subject: [PATCH 19/26] Improve error reporting during python extension module initialization --- src/extensions/PythonExtensionManager.cpp | 24 +++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/src/extensions/PythonExtensionManager.cpp b/src/extensions/PythonExtensionManager.cpp index 0abd3e86d9..c995123b81 100644 --- a/src/extensions/PythonExtensionManager.cpp +++ b/src/extensions/PythonExtensionManager.cpp @@ -75,28 +75,44 @@ PythonExtensionManager::PythonExtensionManager() // to instantiate ExtensibleSomething objects. PyModuleDef* modDef = (PyModuleDef*) PyInit_pythonExtensions(); if (!modDef->m_slots || !PyModuleDef_Init(modDef)) { + if (PyErr_Occurred()) { + PyErr_PrintEx(0); + } throw CanteraError("PythonExtensionManager::PythonExtensionManager", "Failed to import 'pythonExtensions' module"); } // Following example creation of minimal ModuleSpec from Python's import.c PyObject *attrs = Py_BuildValue("{ss}", "name", "pythonExtensions"); - if (attrs == NULL) { + if (attrs == nullptr) { + if (PyErr_Occurred()) { + PyErr_PrintEx(0); + } throw CanteraError("PythonExtensionManager::PythonExtensionManager", "Py_BuildValue failed"); } PyObject *spec = _PyNamespace_New(attrs); Py_DECREF(attrs); - if (spec == NULL) { + if (spec == nullptr) { + if (PyErr_Occurred()) { + PyErr_PrintEx(0); + } throw CanteraError("PythonExtensionManager::PythonExtensionManager", "_PyNamespace_New failed"); } PyObject* pyModule = PyModule_FromDefAndSpec(modDef, spec); - if (!pyModule) { + if (pyModule == nullptr) { + if (PyErr_Occurred()) { + PyErr_PrintEx(0); + } CanteraError("PythonExtensionManager::PythonExtensionManager", "PyModule_FromDefAndSpec failed"); } - if (!PyModule_ExecDef(pyModule, modDef)) { + int code = PyModule_ExecDef(pyModule, modDef); + if (code) { + if (PyErr_Occurred()) { + PyErr_PrintEx(0); + } CanteraError("PythonExtensionManager::PythonExtensionManager", "PyModule_ExecDef failed"); } From 214a2aa4763708ec264c782e14fd4b355126b4ed Mon Sep 17 00:00:00 2001 From: Ray Speth Date: Sun, 4 Sep 2022 16:08:34 -0400 Subject: [PATCH 20/26] Set PYTHONHOME when embedding Python on Windows Without this, the default Python path is not populated correctly, and is based on the path to the user's executable instead. --- site_scons/buildutils.py | 1 + src/SConscript | 3 +++ src/extensions/PythonExtensionManager.cpp | 12 ++++++++++++ 3 files changed, 16 insertions(+) diff --git a/site_scons/buildutils.py b/site_scons/buildutils.py index c1b2a1748e..7a981db158 100644 --- a/site_scons/buildutils.py +++ b/site_scons/buildutils.py @@ -1297,6 +1297,7 @@ def setup_python_env(env): env["py_version_nodot"] = py_version_nodot env["py_version_short"] = info["py_version_short"] env["py_plat"] = plat + env["py_base"] = info["installed_base"] env["site_packages"] = info["site_packages"] env["user_site_packages"] = info["user_site_packages"] if env["OS"] != "Windows": diff --git a/src/SConscript b/src/SConscript index f43f4722ad..36edf469ac 100644 --- a/src/SConscript +++ b/src/SConscript @@ -87,6 +87,9 @@ if env["python_package"] == "full": localenv.Depends(cythonized, pxd) obj = pyenv.SharedObject(cythonized[0]) + if env["OS"] == "Windows": + escaped_home = '\\"' + pyenv["py_base"].replace("\\", "\\\\") + '\\"' + pyenv.Append(CPPDEFINES={"CT_PYTHONHOME": escaped_home}) env.Command('#src/extensions/pythonExtensions.h', '#build/src/extensions/pythonExtensions.h', Copy('$TARGET', '$SOURCE')) libraryTargets.append(obj) diff --git a/src/extensions/PythonExtensionManager.cpp b/src/extensions/PythonExtensionManager.cpp index c995123b81..3ba5063f0a 100644 --- a/src/extensions/PythonExtensionManager.cpp +++ b/src/extensions/PythonExtensionManager.cpp @@ -11,6 +11,10 @@ #include +#ifdef _WIN32 +#include +#endif + namespace ba = boost::algorithm; using namespace std; @@ -66,6 +70,14 @@ namespace Cantera PythonExtensionManager::PythonExtensionManager() { if (!Py_IsInitialized()) { + #if defined(CT_PYTHONHOME) && defined(_WIN32) + const char* old_pythonhome = getenv("PYTHONHOME"); + if (old_pythonhome == nullptr || old_pythonhome[0] == '\0') { + std::string pythonhome = "PYTHONHOME="; + pythonhome += CT_PYTHONHOME; + _putenv(pythonhome.c_str()); + } + #endif Py_Initialize(); } From 0dc2eb0db997d7bfdd3e4e5aa93c841001a96ca7 Mon Sep 17 00:00:00 2001 From: Ray Speth Date: Mon, 5 Sep 2022 10:58:44 -0400 Subject: [PATCH 21/26] Improve handling of dynamic library paths on Mac/Windows --- SConstruct | 2 +- test/SConscript | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/SConstruct b/SConstruct index d6e4fa5bd6..85d0b93107 100644 --- a/SConstruct +++ b/SConstruct @@ -415,7 +415,7 @@ config_options = [ """Environment variables to propagate through to SCons. Either the string 'all' or a comma separated list of variable names, for example, 'LD_LIBRARY_PATH,HOME'.""", - "PATH,LD_LIBRARY_PATH,PYTHONPATH"), + "PATH,LD_LIBRARY_PATH,DYLD_LIBRARY_PATH,PYTHONPATH"), BoolOption( "use_pch", "Use a precompiled-header to speed up compilation", diff --git a/test/SConscript b/test/SConscript index 0299664b31..6b2150d347 100644 --- a/test/SConscript +++ b/test/SConscript @@ -45,7 +45,12 @@ localenv.PrependENVPath('PYTHONPATH', Dir('#test/python').abspath) PASSED_FILES = {} # Add build/lib in order to find Cantera shared library -localenv.PrependENVPath('LD_LIBRARY_PATH', Dir('#build/lib').abspath) +if env["OS"] == "Windows": + localenv.PrependENVPath('PATH', Dir('#build/lib').abspath) +elif env['OS'] == 'Darwin': + localenv.PrependENVPath('DYLD_LIBRARY_PATH', Dir('#build/lib').abspath) +else: + localenv.PrependENVPath('LD_LIBRARY_PATH', Dir('#build/lib').abspath) def addTestProgram(subdir, progName, env_vars={}): """ From bebeeb7f2a562a148ed90b6c8958358e328d0864 Mon Sep 17 00:00:00 2001 From: Ray Speth Date: Mon, 5 Sep 2022 11:45:09 -0400 Subject: [PATCH 22/26] Ensure registration of extension rates in host Cantera library If a user application is linked statically to the Cantera library, ExtensibleRate objects need to be registered in this copy of the Cantera library rather than the one that is embedded in the Python module. This is achieved by accessing the ReactionRateFactory from the main application rather than from the Python module. --- interfaces/cython/cantera/delegator.pyx | 11 +++++++++++ src/extensions/PythonExtensionManager.cpp | 1 + src/extensions/pythonExtensions.pyx | 15 +++++++++++++++ 3 files changed, 27 insertions(+) diff --git a/interfaces/cython/cantera/delegator.pyx b/interfaces/cython/cantera/delegator.pyx index ec80273a28..c793b75319 100644 --- a/interfaces/cython/cantera/delegator.pyx +++ b/interfaces/cython/cantera/delegator.pyx @@ -328,6 +328,11 @@ cdef int assign_delegates(obj, CxxDelegator* delegator) except -1: return 0 +# Specifications for ReactionRate delegators that have not yet been registered with +# ReactionRateFactory. This list is read by PythonExtensionManager::registerRateBuilders +# and then cleared. +_rate_delegators = [] + def extension(*, name): """ A decorator for declaring Cantera extensions that should be registered with @@ -371,8 +376,14 @@ def extension(*, name): def decorator(cls): if issubclass(cls, ExtensibleRate): cls._reaction_rate_type = name + # Registering immediately supports the case where the main + # application is Python CxxPythonExtensionManager.registerPythonRateBuilder( stringify(cls.__module__), stringify(cls.__name__), stringify(name)) + + # Deferred registration supports the case where the main application + # is not Python + _rate_delegators.append((cls.__module__, cls.__name__, name)) else: raise TypeError(f"{cls} is not extensible") return cls diff --git a/src/extensions/PythonExtensionManager.cpp b/src/extensions/PythonExtensionManager.cpp index 3ba5063f0a..9cbaf44c99 100644 --- a/src/extensions/PythonExtensionManager.cpp +++ b/src/extensions/PythonExtensionManager.cpp @@ -144,6 +144,7 @@ void PythonExtensionManager::registerRateBuilders(const string& extensionName) throw CanteraError("PythonExtensionManager::registerRateBuilders", "Problem loading module:\n{}", getPythonExceptionInfo()); } + ct_registerReactionDelegators(); } void PythonExtensionManager::registerPythonRateBuilder( diff --git a/src/extensions/pythonExtensions.pyx b/src/extensions/pythonExtensions.pyx index 6facd1b34f..9520e4b185 100644 --- a/src/extensions/pythonExtensions.pyx +++ b/src/extensions/pythonExtensions.pyx @@ -13,6 +13,7 @@ import importlib import inspect import cantera as ct +from cantera._utils cimport stringify from cantera.reaction cimport ExtensibleRate, CxxReactionRate from cantera.delegator cimport CxxDelegator, assign_delegates @@ -22,6 +23,12 @@ cdef extern from "cantera/kinetics/ReactionRateDelegator.h" namespace "Cantera": CxxReactionRateDelegator() +cdef extern from "cantera/extensions/PythonExtensionManager.h" namespace "Cantera": + cdef cppclass CxxPythonExtensionManager "Cantera::PythonExtensionManager": + @staticmethod + void registerPythonRateBuilder(string&, string&, string&) + + cdef public char* ct_getExceptionString(object exType, object exValue, object exTraceback): import traceback result = str(exValue) + "\n\n" @@ -40,3 +47,11 @@ cdef public object ct_newPythonExtensibleRate(CxxReactionRateDelegator* delegato cdef ExtensibleRate rate = getattr(mod, class_name.decode())(init=False) rate.set_cxx_object(delegator) return rate + + +cdef public ct_registerReactionDelegators(): + for module, cls, name in ct.delegator._rate_delegators: + CxxPythonExtensionManager.registerPythonRateBuilder( + stringify(module), stringify(cls), stringify(name)) + + ct.delegator._rate_delegators.clear() From 06267d8d728f9586e81dcf8f3566968e00b19180 Mon Sep 17 00:00:00 2001 From: Ray Speth Date: Mon, 5 Sep 2022 19:49:04 -0400 Subject: [PATCH 23/26] [CI] Switch to Homebrew Python on macOS This avoids weird linker issues where the GitHub Actions Python required linkage to libintl but the only available version of libintl, installed in Homebrew, was targeting a different macOS version. --- .github/workflows/main.yml | 14 +++++--------- site_scons/buildutils.py | 4 ++-- src/SConscript | 2 +- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 82bc33caf6..816f79c387 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -108,26 +108,22 @@ jobs: name: Checkout the repository with: submodules: recursive - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - architecture: x64 - name: Install Brew dependencies - run: brew install boost libomp + run: | + brew install boost libomp scons python@${{ matrix.python-version }} - name: Upgrade pip run: python3 -m pip install -U pip 'setuptools>=47.0.0,<48' wheel - name: Install Python dependencies - run: python3 -m pip install ruamel.yaml scons numpy cython h5py pandas pytest + run: python3 -m pip install ruamel.yaml numpy cython h5py pandas pytest pytest-github-actions-annotate-failures - name: Install typing_extensions for Python 3.7 if: matrix.python-version == '3.7' run: python3 -m pip install typing_extensions - name: Build Cantera - run: python3 `which scons` build env_vars=all -j3 debug=n --debug=time + run: scons build env_vars=all -j3 debug=n --debug=time - name: Test Cantera run: - python3 `which scons` test show_long_tests=yes verbose_tests=yes --debug=time + scons test show_long_tests=yes verbose_tests=yes --debug=time # Coverage is its own job because macOS builds of the samples # use Homebrew gfortran which is not compatible for coverage diff --git a/site_scons/buildutils.py b/site_scons/buildutils.py index 7a981db158..06eabdda67 100644 --- a/site_scons/buildutils.py +++ b/site_scons/buildutils.py @@ -1301,10 +1301,10 @@ def setup_python_env(env): env["site_packages"] = info["site_packages"] env["user_site_packages"] = info["user_site_packages"] if env["OS"] != "Windows": - env["py_libpath"] = info["LIBPL"] + env["py_libpath"] = [info["LIBPL"], info["LIBDIR"]] py_lib = "python" + info["py_version_short"] + info["abiflags"] else: - env["py_libpath"] = info["installed_base"] + "\\libs" + env["py_libpath"] = [info["installed_base"] + "\\libs"] py_lib = "python" + py_version_nodot env["py_libs"] = [py_lib] + [lib[2:] for lib in info.get("LIBS", "").split() if lib.startswith("-l")] diff --git a/src/SConscript b/src/SConscript index 36edf469ac..ce856f807c 100644 --- a/src/SConscript +++ b/src/SConscript @@ -97,7 +97,7 @@ if env["python_package"] == "full": localenv.Append(LIBS=pyenv["py_libs"], LIBPATH=pyenv["py_libpath"]) env["cantera_libs"].extend(pyenv["py_libs"]) env.Append(LIBPATH=pyenv["py_libpath"]) - env["extra_lib_dirs"].append(pyenv["py_libpath"]) + env["extra_lib_dirs"].extend(pyenv["py_libpath"]) # build the Cantera static library lib = build(localenv.StaticLibrary('../lib/cantera', libraryTargets, From 2ec2012a9167538a5ad77cfaf65e9bf7fa7eeb8a Mon Sep 17 00:00:00 2001 From: Ray Speth Date: Tue, 6 Sep 2022 21:54:23 -0400 Subject: [PATCH 24/26] Test error conditions for ExtensibleRate --- test/python/test_reaction.py | 36 ++++++++++++++++++++++++++++++++- test/python/user_ext_invalid.py | 11 ++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 test/python/user_ext_invalid.py diff --git a/test/python/test_reaction.py b/test/python/test_reaction.py index d362168fe4..825bbc3fcd 100644 --- a/test/python/test_reaction.py +++ b/test/python/test_reaction.py @@ -1543,19 +1543,53 @@ def test_from_dict(self): def test_roundtrip(self): pytest.skip("ExtensibleRate does not yet support roundtrip conversion") + def test_set_parameters_error(self): + with pytest.raises(KeyError): + # Instantiate with no A factor + ct.ReactionRate.from_dict({"type": "user-rate-1"}) + + def test_eval_error(self): + # Instantiate with non-numeric A factor to cause an exception during evaluation + R = ct.ReactionRate.from_dict({"type": "user-rate-1", "A": "xyz"}) + with pytest.raises(TypeError): + R(500) + class TestExtensible2(utilities.CanteraTest): - def test_load_module(self): + _input_template = """ + extensions: + - type: python + name: {module} + + phases: + - name: gas + thermo: ideal-gas + species: [{{h2o2.yaml/species: all}}] + kinetics: gas + state: {{T: 300.0, P: 1 atm}} + """ + + @classmethod + def setUpClass(cls): here = str(Path(__file__).parent) if here not in sys.path: sys.path.append(here) + def test_load_module(self): gas = ct.Solution("extensible-reactions.yaml", transport_model=None) for T in np.linspace(300, 3000, 10): gas.TP = T, None assert gas.forward_rate_constants[0] == pytest.approx(3.14 * T**2) + def test_missing_module(self): + with pytest.raises(ct.CanteraError, match="No module named 'fake_ext'"): + ct.Solution(yaml=self._input_template.format(module="fake_ext")) + + def test_invalid_module(self): + with pytest.raises(ct.CanteraError, match="SyntaxError"): + ct.Solution(yaml=self._input_template.format(module="user_ext_invalid")) + class InterfaceReactionTests(ReactionTests): # test suite for surface reaction expressions diff --git a/test/python/user_ext_invalid.py b/test/python/user_ext_invalid.py new file mode 100644 index 0000000000..3c4ba51f4f --- /dev/null +++ b/test/python/user_ext_invalid.py @@ -0,0 +1,11 @@ +import cantera as ct + +this is a syntax error + +@ct.extension(name="square-rate") +class SquareRate(ct.ExtensibleRate): + def after_set_parameters(self, node, units): + self.A = node["A"] + + def replace_eval(self, T): + return self.A * T**2 From ecd63cd7f72cd9abef4e03aef1b99f3d6a1ecbce Mon Sep 17 00:00:00 2001 From: Ray Speth Date: Fri, 9 Sep 2022 13:34:44 -0400 Subject: [PATCH 25/26] Update documentation of "version added" --- CONTRIBUTING.md | 9 ++++++++- include/cantera/base/ExtensionManager.h | 2 ++ include/cantera/base/ExtensionManagerFactory.h | 2 ++ include/cantera/base/global.h | 2 ++ include/cantera/extensions/PythonExtensionManager.h | 2 ++ include/cantera/kinetics/ReactionRateDelegator.h | 2 ++ interfaces/cython/cantera/delegator.pyx | 2 ++ interfaces/cython/cantera/reaction.pyx | 2 ++ samples/python/kinetics/custom_reactions.py | 2 +- src/base/application.h | 1 + 10 files changed, 24 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5daefbace4..5607e58152 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -69,6 +69,9 @@ (for example, comment lines starting with `//!` or comment blocks starting with `/*!`; do not use `///` or `/**` in new code) * Doxygen-style groupings should bracket code using `//! @{` and `//! @}` +* Indicate the version added for new functions and classes with an annotation like + `@since New in Cantera X.Y` where `X.Y` is the next Cantera version. This notation + should also be used indicate significant changes in behavior. * Avoid defining non-trivial functions in header files * Header files should include an 'include guard' * Protected and private member variable names are generally prefixed with @@ -99,7 +102,11 @@ * Style generally follows PEP8 (https://www.python.org/dev/peps/pep-0008/) * Code in `.py` and `.pyx` files needs to be written to work with Python 3 -* The minimum Python version that Cantera supports is Python 3.6, so code should only use features added in Python 3.6 or earlier +* The minimum Python version that Cantera supports is Python 3.7, so code should only + use features added in Python 3.7 or earlier +* Indicate the version added for new functions and classes with an annotation like + `.. versionadded:: X.Y` where `X.Y` is the next Cantera version. Significant changes + in behavior should be indicated with `.. versionchanged:: X.Y`. * Please use double quotes in all new Python code ## C# diff --git a/include/cantera/base/ExtensionManager.h b/include/cantera/base/ExtensionManager.h index 1f41d96883..fcb4dd5135 100644 --- a/include/cantera/base/ExtensionManager.h +++ b/include/cantera/base/ExtensionManager.h @@ -12,6 +12,8 @@ namespace Cantera { //! Base class for managing user-defined Cantera extensions written in other languages +//! +//! @since New in Cantera 3.0 class ExtensionManager { public: diff --git a/include/cantera/base/ExtensionManagerFactory.h b/include/cantera/base/ExtensionManagerFactory.h index cc25361e54..77b48fb873 100644 --- a/include/cantera/base/ExtensionManagerFactory.h +++ b/include/cantera/base/ExtensionManagerFactory.h @@ -13,6 +13,8 @@ namespace Cantera { //! A factory class for creating ExtensionManager objects +//! +//! @since New in Cantera 3.0 class ExtensionManagerFactory : public Factory { public: diff --git a/include/cantera/base/global.h b/include/cantera/base/global.h index 376991d04a..914f49a5a9 100644 --- a/include/cantera/base/global.h +++ b/include/cantera/base/global.h @@ -83,6 +83,8 @@ void loadExtension(const std::string& extType, const std::string& name); //! Load extensions providing user-defined models from the `extensions` section of the //! given node. @see Application::loadExtension +//! +//! @since New in Cantera 3.0 void loadExtensions(const AnyMap& node); //! Delete and free all memory associated with the application diff --git a/include/cantera/extensions/PythonExtensionManager.h b/include/cantera/extensions/PythonExtensionManager.h index a29d0c2fd5..2da9033f80 100644 --- a/include/cantera/extensions/PythonExtensionManager.h +++ b/include/cantera/extensions/PythonExtensionManager.h @@ -20,6 +20,8 @@ namespace Cantera //! documentation for //! `@extension` //! in the Python documentation for more information. +//! +//! @since New in Cantera 3.0 class PythonExtensionManager : public ExtensionManager { public: diff --git a/include/cantera/kinetics/ReactionRateDelegator.h b/include/cantera/kinetics/ReactionRateDelegator.h index bd68b4b449..12e9b6af91 100644 --- a/include/cantera/kinetics/ReactionRateDelegator.h +++ b/include/cantera/kinetics/ReactionRateDelegator.h @@ -14,6 +14,8 @@ namespace Cantera { //! Delegate methods of the ReactionRate class to external functions +//! +//! @since New in Cantera 3.0 class ReactionRateDelegator : public Delegator, public ReactionRate { public: diff --git a/interfaces/cython/cantera/delegator.pyx b/interfaces/cython/cantera/delegator.pyx index c793b75319..5107b12069 100644 --- a/interfaces/cython/cantera/delegator.pyx +++ b/interfaces/cython/cantera/delegator.pyx @@ -372,6 +372,8 @@ def extension(*, name): Loading this input file from any Cantera user interface would cause Cantera to load the ``my_cool_module.py`` module and register the ``CoolRate`` class to handle reactions whose ``type`` in the YAML file is set to ``cool-rate``. + + .. versionadded:: 3.0 """ def decorator(cls): if issubclass(cls, ExtensibleRate): diff --git a/interfaces/cython/cantera/reaction.pyx b/interfaces/cython/cantera/reaction.pyx index 40f3ecc786..834e112b1b 100644 --- a/interfaces/cython/cantera/reaction.pyx +++ b/interfaces/cython/cantera/reaction.pyx @@ -744,6 +744,8 @@ cdef class ExtensibleRate(ReactionRate): **Warning:** The delegatable methods defined here are an experimental part of the Cantera API and may change without notice. + + .. versionadded:: 3.0 """ _reaction_rate_type = "extensible" diff --git a/samples/python/kinetics/custom_reactions.py b/samples/python/kinetics/custom_reactions.py index 5eaa16781f..45f806ffb1 100644 --- a/samples/python/kinetics/custom_reactions.py +++ b/samples/python/kinetics/custom_reactions.py @@ -3,7 +3,7 @@ For benchmark purposes, an ignition test is run to compare simulation times. -Requires: cantera >= 2.6.0 +Requires: cantera >= 3.0.0 Keywords: kinetics, benchmarking, user-defined model """ diff --git a/src/base/application.h b/src/base/application.h index bf5039a86d..883156639e 100644 --- a/src/base/application.h +++ b/src/base/application.h @@ -290,6 +290,7 @@ class Application //! parameter depends on the specific extension interface. For example, for //! Python extensions, this is the name of the Python module containing the //! models. + //! @since New in Cantera 3.0 void loadExtension(const std::string& extType, const std::string& name); #ifdef _WIN32 From d7efc72fd40c6d8714af3ac666d9cc682f5d0d99 Mon Sep 17 00:00:00 2001 From: Ray Speth Date: Fri, 9 Sep 2022 21:45:07 -0400 Subject: [PATCH 26/26] Make Python extensions compatible with venv --- .../extensions/PythonExtensionManager.h | 3 ++ src/extensions/PythonExtensionManager.cpp | 38 +++++++++++++++---- 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/include/cantera/extensions/PythonExtensionManager.h b/include/cantera/extensions/PythonExtensionManager.h index 2da9033f80..646e60cbf2 100644 --- a/include/cantera/extensions/PythonExtensionManager.h +++ b/include/cantera/extensions/PythonExtensionManager.h @@ -31,6 +31,9 @@ class PythonExtensionManager : public ExtensionManager //! Function called from Cython to register an ExtensibleRate implementation static void registerPythonRateBuilder(const std::string& moduleName, const std::string& className, const std::string& rateName); + +private: + static bool s_imported; }; } diff --git a/src/extensions/PythonExtensionManager.cpp b/src/extensions/PythonExtensionManager.cpp index 9cbaf44c99..5520445008 100644 --- a/src/extensions/PythonExtensionManager.cpp +++ b/src/extensions/PythonExtensionManager.cpp @@ -10,6 +10,7 @@ #include "pythonExtensions.h" // generated by Cython #include +#include #ifdef _WIN32 #include @@ -67,20 +68,40 @@ std::string getPythonExceptionInfo() namespace Cantera { +bool PythonExtensionManager::s_imported = false; + PythonExtensionManager::PythonExtensionManager() { if (!Py_IsInitialized()) { - #if defined(CT_PYTHONHOME) && defined(_WIN32) - const char* old_pythonhome = getenv("PYTHONHOME"); - if (old_pythonhome == nullptr || old_pythonhome[0] == '\0') { - std::string pythonhome = "PYTHONHOME="; - pythonhome += CT_PYTHONHOME; - _putenv(pythonhome.c_str()); - } - #endif + // Update the path to include the virtual environment, if one is active + const char* venv_path = getenv("VIRTUAL_ENV"); + if (venv_path != nullptr) { + #ifdef _WIN32 + string suffix = "\\Scripts\\python.exe"; + #else + string suffix = "/bin/python"; + #endif + string path(venv_path); + path += suffix; + wstring wpath = wstring_convert>().from_bytes(path); + Py_SetProgramName(wpath.c_str()); + } else { + #if defined(CT_PYTHONHOME) && defined(_WIN32) + const char* old_pythonhome = getenv("PYTHONHOME"); + if (old_pythonhome == nullptr || old_pythonhome[0] == '\0') { + string pythonhome = "PYTHONHOME="; + pythonhome += CT_PYTHONHOME; + _putenv(pythonhome.c_str()); + } + #endif + } Py_Initialize(); } + if (s_imported) { + return; + } + // PEP 489 Multi-phase initialization // The 'pythonExtensions' Cython module defines some functions that are used @@ -130,6 +151,7 @@ PythonExtensionManager::PythonExtensionManager() } Py_DECREF(spec); Py_DECREF(pyModule); + s_imported = true; } void PythonExtensionManager::registerRateBuilders(const string& extensionName)