gh-35774: Modularization fixes for fast_callable interpreters

    
<!-- Please provide a concise, informative and self-explanatory title.
-->
<!-- Don't put issue numbers in the title. Put it in the Description
below. -->
<!-- For example, instead of "Fixes #12345", use "Add a new method to
multiply two integers" -->

### 📚 Description

<!-- Describe your changes here in detail. -->
The code generator in `sage_setup.autogen.interpreters` can now build a
subset of interpreters. This is used in #35095 to only build the
interpreters for `Element` and `object` in the sagemath-categories
distribution.

At runtime, `sage.ext.fast_callable` no longer fails when an interpreter
cannot be imported but falls back to the `Element` interpreter if
necessary.

<!-- Why is this change required? What problem does it solve? -->
<!-- If this PR resolves an open issue, please link to it here. For
example "Fixes #12345". -->
- Part of: #29705
- Cherry-picked from: #35095

<!-- If your change requires a documentation PR, please link it
appropriately. -->

### 📝 Checklist

<!-- Put an `x` in all the boxes that apply. It should be `[x]` not `[x
]`. -->

- [x] The title is concise, informative, and self-explanatory.
- [x] The description explains in detail what this PR is about.
- [ ] I have linked a relevant issue or discussion.
- [ ] I have created tests covering the changes.
- [ ] I have updated the documentation accordingly.

### ⌛ Dependencies

<!-- List all open PRs that this PR logically depends on
- #12345: short description why this is a dependency
- #34567: ...
-->

<!-- If you're unsure about any of these, don't hesitate to ask. We're
here to help! -->
    
URL: #35774
Reported by: Matthias Köppe
Reviewer(s): Matthias Köppe, Michael Orlitzky

src/sage/ext/fast_callable.pyx

@@ -465,41 +465,89 @@ def fast_callable(x, domain=None, vars=None,
         etb = ExpressionTreeBuilder(vars=vars, domain=domain)
         et = x._fast_callable_(etb)
 
-    if isinstance(domain, sage.rings.abc.RealField):
-        from sage.ext.interpreters.wrapper_rr import Wrapper_rr as builder
-        str = InstructionStream(sage.ext.interpreters.wrapper_rr.metadata,
-                                len(vars),
-                                domain)
-
-    elif isinstance(domain, sage.rings.abc.ComplexField):
-        from sage.ext.interpreters.wrapper_cc import Wrapper_cc as builder
-        str = InstructionStream(sage.ext.interpreters.wrapper_cc.metadata,
-                                len(vars),
-                                domain)
-
-    elif isinstance(domain, sage.rings.abc.RealDoubleField) or domain is float:
-        from sage.ext.interpreters.wrapper_rdf import Wrapper_rdf as builder
-        str = InstructionStream(sage.ext.interpreters.wrapper_rdf.metadata,
-                                len(vars),
-                                domain)
-    elif isinstance(domain, sage.rings.abc.ComplexDoubleField):
-        from sage.ext.interpreters.wrapper_cdf import Wrapper_cdf as builder
-        str = InstructionStream(sage.ext.interpreters.wrapper_cdf.metadata,
-                                len(vars),
-                                domain)
-    elif domain is None:
-        from sage.ext.interpreters.wrapper_py import Wrapper_py as builder
-        str = InstructionStream(sage.ext.interpreters.wrapper_py.metadata,
-                                len(vars))
-    else:
-        from sage.ext.interpreters.wrapper_el import Wrapper_el as builder
-        str = InstructionStream(sage.ext.interpreters.wrapper_el.metadata,
-                                len(vars),
-                                domain)
+    builder, str = _builder_and_stream(vars=vars, domain=domain)
+
     generate_code(et, str)
     str.instr('return')
     return builder(str.get_current())
 
+
+def _builder_and_stream(vars, domain):
+    r"""
+    Return a builder and a stream.
+
+    This is an internal function used only once, by :func:`fast_callable`.
+
+    INPUT:
+
+    - ``vars`` -- a sequence of variable names
+
+    - ``domain`` -- a Sage parent or Python type or ``None``; if non-``None``,
+      all arithmetic is done in that domain
+
+    OUTPUT: A :class:`Wrapper`, an class:`InstructionStream`
+
+    EXAMPLES::
+
+        sage: from sage.ext.fast_callable import _builder_and_stream
+        sage: _builder_and_stream(["x", "y"], ZZ)
+        (<class 'sage.ext.interpreters.wrapper_el.Wrapper_el'>,
+         <sage.ext.fast_callable.InstructionStream object at 0x...>)
+        sage: _builder_and_stream(["x", "y"], RR)                                       # optional - sage.rings.real_mpfr
+        (<class 'sage.ext.interpreters.wrapper_rr.Wrapper_rr'>,
+         <sage.ext.fast_callable.InstructionStream object at 0x...>)
+
+    Modularized test with sagemath-categories after :issue:`35095`, which has
+    (a basic version of) ``RDF`` but not the specialized interpreter for it.
+    In this case, the function falls back to using the :class:`Element`
+    interpreter::
+
+        sage: domain = RDF
+        sage: from sage.structure.element import Element as domain                      # optional - sage.modules
+        sage: _builder_and_stream(["x", "y"], domain)
+        (<class 'sage.ext.interpreters.wrapper_el.Wrapper_el'>,
+         <sage.ext.fast_callable.InstructionStream object at 0x...>)
+    """
+    if isinstance(domain, sage.rings.abc.RealField):
+        try:
+            from sage.ext.interpreters.wrapper_rr import metadata, Wrapper_rr as builder
+        except ImportError:
+            pass
+        else:
+            return builder, InstructionStream(metadata, len(vars), domain)
+
+    if isinstance(domain, sage.rings.abc.ComplexField):
+        try:
+            from sage.ext.interpreters.wrapper_cc import metadata, Wrapper_cc as builder
+        except ImportError:
+            pass
+        else:
+            return builder, InstructionStream(metadata, len(vars), domain)
+
+    if isinstance(domain, sage.rings.abc.RealDoubleField) or domain is float:
+        try:
+            from sage.ext.interpreters.wrapper_rdf import metadata, Wrapper_rdf as builder
+        except ImportError:
+            pass
+        else:
+            return builder, InstructionStream(metadata, len(vars), domain)
+
+    if isinstance(domain, sage.rings.abc.ComplexDoubleField):
+        try:
+            from sage.ext.interpreters.wrapper_cdf import metadata, Wrapper_cdf as builder
+        except ImportError:
+            pass
+        else:
+            return builder, InstructionStream(metadata, len(vars), domain)
+
+    if domain is None:
+        from sage.ext.interpreters.wrapper_py import metadata, Wrapper_py as builder
+        return builder, InstructionStream(metadata, len(vars))
+
+    from sage.ext.interpreters.wrapper_el import metadata, Wrapper_el as builder
+    return builder, InstructionStream(metadata, len(vars), domain)
+
+
 def function_name(fn):
     r"""
     Given a function, return a string giving a name for the function.

src/sage_setup/autogen/interpreters/__init__.py

@@ -119,25 +119,10 @@
 from .instructions import *
 from .memory import *
 from .specs.base import *
-from .specs.cdf import *
-from .specs.element import *
-from .specs.python import *
-from .specs.rdf import *
-from .specs.rr import *
-from .specs.cc import *
 from .storage import *
 from .utils import *
 
 
-# Gather up a list of all interpreter classes imported into this module
-# A better way might be to recursively iterate InterpreterSpec.__subclasses__
-# or to use a registry, but this is fine for now.
-_INTERPRETERS = sorted(filter(lambda c: (isinstance(c, type) and
-                                         issubclass(c, InterpreterSpec) and
-                                         c.name),
-                              globals().values()),
-                       key=lambda c: c.name)
-
 # Tuple of (filename_root, extension, method) where filename_root is the
 # root of the filename to be joined with "_<interpreter_name>".ext and
 # method is the name of a get_ method on InterpreterGenerator that returns
@@ -157,6 +142,7 @@ def build_interp(interp_spec, dir):
     EXAMPLES::
 
         sage: from sage_setup.autogen.interpreters import *
+        sage: from sage_setup.autogen.interpreters.specs.rdf import RDFInterpreter
         sage: testdir = tmp_dir()
         sage: rdf_interp = RDFInterpreter()
         sage: build_interp(rdf_interp, testdir)
@@ -174,12 +160,28 @@ def build_interp(interp_spec, dir):
         write_if_changed(path, method())
 
 
-def rebuild(dirname, force=False):
+def rebuild(dirname, force=False, interpreters=None, distribution=None):
     r"""
     Check whether the interpreter and wrapper sources have been written
     since the last time this module was changed.  If not, write them.
 
-    EXAMPLES::
+    INPUT:
+
+    - ``dirname`` -- name of the target directory for the generated sources
+
+    - ``force`` -- boolean (default ``False``); if ``True``, ignore timestamps
+      and regenerate the sources unconditionally
+
+    - ``interpreters`` -- an iterable of strings, or ``None`` (the default,
+      which means all interpreters); which interpreters to generate
+
+    - ``distribution`` -- a string (the distribution name such as
+      ``"sagemath-categories"``)  or ``None`` (the default, which means
+      the monolithic Sage library)
+
+    EXAMPLES:
+
+    Monolithic build::
 
         sage: from sage_setup.autogen.interpreters import *
         sage: testdir = tmp_dir()
@@ -189,11 +191,36 @@ def rebuild(dirname, force=False):
         sage: with open(testdir + '/wrapper_el.pyx') as f:
         ....:     f.readline()
         '# Automatically generated by ...\n'
+
+    Modularized build::
+
+        sage: testdir = tmp_dir()
+        sage: rebuild(testdir, interpreters=['Element', 'Python'],
+        ....:         distribution='sagemath-categories')
+        Building interpreters for fast_callable
+        -> First build of interpreters
+        sage: with open(testdir + '/all__sagemath_categories.py') as f:
+        ....:     f.readline()
+        '# Automatically generated by ...'
     """
     # This line will show up in "sage -b" (once per upgrade, not every time
     # you run it).
     print("Building interpreters for fast_callable")
 
+    if interpreters is None:
+        interpreters = ['CDF', 'Element', 'Python', 'RDF', 'RR', 'CC']
+
+    from importlib import import_module
+
+    _INTERPRETERS = [getattr(import_module('sage_setup.autogen.interpreters.specs.' + interpreter.lower()),
+                             interpreter + 'Interpreter')
+                     for interpreter in interpreters]
+
+    if distribution is None:
+        all_py = 'all.py'
+    else:
+        all_py = f'all__{distribution.replace("-", "_")}.py'
+
     try:
         os.makedirs(dirname)
     except OSError:
@@ -213,7 +240,7 @@ class NeedToRebuild(Exception):
     try:
         if force:
             raise NeedToRebuild("-> Force rebuilding interpreters")
-        gen_file = os.path.join(dirname, 'all.py')
+        gen_file = os.path.join(dirname, all_py)
         if not os.path.isfile(gen_file):
             raise NeedToRebuild("-> First build of interpreters")
 
@@ -235,5 +262,5 @@ class NeedToRebuild(Exception):
     for interp in _INTERPRETERS:
         build_interp(interp(), dirname)
 
-    with open(os.path.join(dirname, 'all.py'), 'w') as f:
+    with open(os.path.join(dirname, all_py), 'w') as f:
         f.write("# " + AUTOGEN_WARN)

src/sage_setup/autogen/interpreters/generator.py

@@ -43,6 +43,7 @@ def __init__(self, spec):
         EXAMPLES::
 
             sage: from sage_setup.autogen.interpreters import *
+            sage: from sage_setup.autogen.interpreters.specs.rdf import RDFInterpreter
             sage: interp = RDFInterpreter()
             sage: gen = InterpreterGenerator(interp)
             sage: gen._spec is interp
@@ -72,6 +73,7 @@ def gen_code(self, instr_desc, write):
         EXAMPLES::
 
             sage: from sage_setup.autogen.interpreters import *
+            sage: from sage_setup.autogen.interpreters.specs.rdf import RDFInterpreter
             sage: interp = RDFInterpreter()
             sage: gen = InterpreterGenerator(interp)
             sage: from io import StringIO
@@ -218,6 +220,7 @@ def func_header(self, cython=False):
         EXAMPLES::
 
             sage: from sage_setup.autogen.interpreters import *
+            sage: from sage_setup.autogen.interpreters.specs.element import ElementInterpreter
             sage: interp = ElementInterpreter()
             sage: gen = InterpreterGenerator(interp)
             sage: print(gen.func_header())
@@ -260,6 +263,7 @@ def write_interpreter(self, write):
         EXAMPLES::
 
             sage: from sage_setup.autogen.interpreters import *
+            sage: from sage_setup.autogen.interpreters.specs.rdf import RDFInterpreter
             sage: interp = RDFInterpreter()
             sage: gen = InterpreterGenerator(interp)
             sage: from io import StringIO
@@ -307,6 +311,7 @@ def write_wrapper(self, write):
         EXAMPLES::
 
             sage: from sage_setup.autogen.interpreters import *
+            sage: from sage_setup.autogen.interpreters.specs.rdf import RDFInterpreter
             sage: interp = RDFInterpreter()
             sage: gen = InterpreterGenerator(interp)
             sage: from io import StringIO
@@ -476,6 +481,7 @@ def write_pxd(self, write):
         EXAMPLES::
 
             sage: from sage_setup.autogen.interpreters import *
+            sage: from sage_setup.autogen.interpreters.specs.rdf import RDFInterpreter
             sage: interp = RDFInterpreter()
             sage: gen = InterpreterGenerator(interp)
             sage: from io import StringIO
@@ -527,6 +533,9 @@ def get_interpreter(self):
         First we get the InterpreterSpec for several interpreters::
 
             sage: from sage_setup.autogen.interpreters import *
+            sage: from sage_setup.autogen.interpreters.specs.rdf import RDFInterpreter
+            sage: from sage_setup.autogen.interpreters.specs.rr import RRInterpreter
+            sage: from sage_setup.autogen.interpreters.specs.element import ElementInterpreter
             sage: rdf_spec = RDFInterpreter()
             sage: rr_spec = RRInterpreter()
             sage: el_spec = ElementInterpreter()
@@ -649,6 +658,9 @@ def get_wrapper(self):
         First we get the InterpreterSpec for several interpreters::
 
             sage: from sage_setup.autogen.interpreters import *
+            sage: from sage_setup.autogen.interpreters.specs.rdf import RDFInterpreter
+            sage: from sage_setup.autogen.interpreters.specs.rr import RRInterpreter
+            sage: from sage_setup.autogen.interpreters.specs.element import ElementInterpreter
             sage: rdf_spec = RDFInterpreter()
             sage: rr_spec = RRInterpreter()
             sage: el_spec = ElementInterpreter()
@@ -972,6 +984,9 @@ def get_pxd(self):
         First we get the InterpreterSpec for several interpreters::
 
             sage: from sage_setup.autogen.interpreters import *
+            sage: from sage_setup.autogen.interpreters.specs.rdf import RDFInterpreter
+            sage: from sage_setup.autogen.interpreters.specs.rr import RRInterpreter
+            sage: from sage_setup.autogen.interpreters.specs.element import ElementInterpreter
             sage: rdf_spec = RDFInterpreter()
             sage: rr_spec = RRInterpreter()
             sage: el_spec = ElementInterpreter()

src/sage_setup/autogen/interpreters/instructions.py

@@ -187,6 +187,7 @@ class InstrSpec(object):
     EXAMPLES::
 
         sage: from sage_setup.autogen.interpreters import *
+        sage: from sage_setup.autogen.interpreters.specs.rdf import RDFInterpreter
         sage: pg = RDFInterpreter().pg
         sage: InstrSpec('add', pg('SS','S'), code='o0 = i0+i1;')
         add: SS->S = 'o0 = i0+i1;'
@@ -213,7 +214,7 @@ def __init__(self, name, io, code=None, uses_error_handler=False,
         EXAMPLES::
 
             sage: from sage_setup.autogen.interpreters import *
-
+            sage: from sage_setup.autogen.interpreters.specs.rdf import RDFInterpreter
             sage: pg = RDFInterpreter().pg
             sage: InstrSpec('add', pg('SS','S'), code='o0 = i0+i1;')
             add: SS->S = 'o0 = i0+i1;'
@@ -288,6 +289,7 @@ def __repr__(self):
         EXAMPLES::
 
             sage: from sage_setup.autogen.interpreters import *
+            sage: from sage_setup.autogen.interpreters.specs.rdf import RDFInterpreter
             sage: pg = RDFInterpreter().pg
             sage: InstrSpec('add', pg('SS','S'), code='o0 = i0+i1;')
             add: SS->S = 'o0 = i0+i1;'
@@ -310,6 +312,7 @@ def instr_infix(name, io, op):
     EXAMPLES::
 
         sage: from sage_setup.autogen.interpreters import *
+        sage: from sage_setup.autogen.interpreters.specs.rdf import RDFInterpreter
         sage: pg = RDFInterpreter().pg
         sage: instr_infix('mul', pg('SS', 'S'), '*')
         mul: SS->S = 'o0 = i0 * i1;'
@@ -325,6 +328,7 @@ def instr_funcall_2args(name, io, op):
     EXAMPLES::
 
         sage: from sage_setup.autogen.interpreters import *
+        sage: from sage_setup.autogen.interpreters.specs.rdf import RDFInterpreter
         sage: pg = RDFInterpreter().pg
         sage: instr_funcall_2args('atan2', pg('SS', 'S'), 'atan2')
         atan2: SS->S = 'o0 = atan2(i0, i1);'
@@ -340,6 +344,7 @@ def instr_unary(name, io, op):
     EXAMPLES::
 
         sage: from sage_setup.autogen.interpreters import *
+        sage: from sage_setup.autogen.interpreters.specs.rdf import RDFInterpreter
         sage: pg = RDFInterpreter().pg
         sage: instr_unary('sin', pg('S','S'), 'sin(i0)')
         sin: S->S = 'o0 = sin(i0);'
@@ -357,6 +362,7 @@ def instr_funcall_2args_mpfr(name, io, op):
     EXAMPLES::
 
         sage: from sage_setup.autogen.interpreters import *
+        sage: from sage_setup.autogen.interpreters.specs.rr import RRInterpreter
         sage: pg = RRInterpreter().pg
         sage: instr_funcall_2args_mpfr('add', pg('SS','S'), 'mpfr_add')
         add: SS->S = 'mpfr_add(o0, i0, i1, MPFR_RNDN);'
@@ -372,6 +378,7 @@ def instr_funcall_1arg_mpfr(name, io, op):
     EXAMPLES::
 
         sage: from sage_setup.autogen.interpreters import *
+        sage: from sage_setup.autogen.interpreters.specs.rr import RRInterpreter
         sage: pg = RRInterpreter().pg
         sage: instr_funcall_1arg_mpfr('exp', pg('S','S'), 'mpfr_exp')
         exp: S->S = 'mpfr_exp(o0, i0, MPFR_RNDN);'
@@ -386,6 +393,7 @@ def instr_funcall_2args_mpc(name, io, op):
     EXAMPLES::
 
         sage: from sage_setup.autogen.interpreters import *
+        sage: from sage_setup.autogen.interpreters.specs.cc import CCInterpreter
         sage: pg = CCInterpreter().pg
         sage: instr_funcall_2args_mpc('add', pg('SS','S'), 'mpc_add')
         add: SS->S = 'mpc_add(o0, i0, i1, MPC_RNDNN);'
@@ -400,6 +408,7 @@ def instr_funcall_1arg_mpc(name, io, op):
     EXAMPLES::
 
         sage: from sage_setup.autogen.interpreters import *
+        sage: from sage_setup.autogen.interpreters.specs.cc import CCInterpreter
         sage: pg = CCInterpreter().pg
         sage: instr_funcall_1arg_mpc('exp', pg('S','S'), 'mpc_exp')
         exp: S->S = 'mpc_exp(o0, i0, MPC_RNDNN);'