diff --git a/src/bin/sage-runtests b/src/bin/sage-runtests index 6b92c980cfb..4131161647d 100755 --- a/src/bin/sage-runtests +++ b/src/bin/sage-runtests @@ -52,6 +52,10 @@ if __name__ == "__main__": 'if set to "all", then all tests will be run; ' 'use "!FEATURE" to disable tests marked "# optional - FEATURE". ' 'Note that "!" needs to be quoted or escaped in the shell.') + parser.add_argument("--hide", metavar="FEATURES", default="", + help='run tests pretending that the software listed in FEATURES (separated by commas) is not installed; ' + 'if "all" is listed, will also hide features corresponding to all optional or experimental packages; ' + 'if "optional" is listed, will also hide features corresponding to optional packages.') parser.add_argument("--randorder", type=int, metavar="SEED", help="randomize order of tests") parser.add_argument("--random-seed", dest="random_seed", type=int, metavar="SEED", help="random seed (integer) for fuzzing doctests", default=os.environ.get("SAGE_DOCTEST_RANDOM_SEED")) diff --git a/src/sage/doctest/control.py b/src/sage/doctest/control.py index 450d9b607da..802f185d363 100644 --- a/src/sage/doctest/control.py +++ b/src/sage/doctest/control.py @@ -57,7 +57,6 @@ except ImportError: pass - class DocTestDefaults(SageObject): """ This class is used for doctesting the Sage doctest module. @@ -137,6 +136,7 @@ def __init__(self, **kwds): # displaying user-defined optional tags and we don't want to see # the auto_optional_tags there. self.optional = set(['sage']) | auto_optional_tags + self.hide = '' # > 0: always run GC before every test # < 0: disable GC @@ -401,6 +401,27 @@ def __init__(self, options, args): if options.verbose: options.show_skipped = True + options.hidden_features = set() + if isinstance(options.hide, str): + if not len(options.hide): + options.hide = set([]) + else: + s = options.hide.lower() + options.hide = set(s.split(',')) + for h in options.hide: + if not optionaltag_regex.search(h): + raise ValueError('invalid optional tag {!r}'.format(h)) + if 'all' in options.hide: + options.hide.discard('all') + from sage.features.all import all_features + feature_names = set([f.name for f in all_features() if not f.is_standard()]) + options.hide = options.hide.union(feature_names) + if 'optional' in options.hide: + options.hide.discard('optional') + from sage.features.all import all_features + feature_names = set([f.name for f in all_features() if f.is_optional()]) + options.hide = options.hide.union(feature_names) + options.disabled_optional = set() if isinstance(options.optional, str): s = options.optional.lower() @@ -417,6 +438,8 @@ def __init__(self, options, args): options.optional.discard('optional') from sage.misc.package import list_packages for pkg in list_packages('optional', local=True).values(): + if pkg.name in options.hide: + continue if pkg.is_installed() and pkg.installed_version == pkg.remote_version: options.optional.add(pkg.name) @@ -1329,6 +1352,49 @@ def run(self): Features detected... 0 + We test the ``--hide`` option (:trac:`34185`):: + + sage: from sage.doctest.control import test_hide + sage: filename = tmp_filename(ext='.py') + sage: with open(filename, 'w') as f: + ....: f.write(test_hide) + ....: f.close() + 729 + sage: DF = DocTestDefaults(hide='buckygen,all') + sage: DC = DocTestController(DF, [filename]) + sage: DC.run() + Running doctests with ID ... + Using --optional=sage... + Features to be detected: ... + Doctesting 1 file. + sage -t ....py + [4 tests, ... s] + ---------------------------------------------------------------------- + All tests passed! + ---------------------------------------------------------------------- + Total time for all tests: ... seconds + cpu time: ... seconds + cumulative wall time: ... seconds + Features detected... + 0 + + sage: DF = DocTestDefaults(hide='benzene,optional') + sage: DC = DocTestController(DF, [filename]) + sage: DC.run() + Running doctests with ID ... + Using --optional=sage + Features to be detected: ... + Doctesting 1 file. + sage -t ....py + [4 tests, ... s] + ---------------------------------------------------------------------- + All tests passed! + ---------------------------------------------------------------------- + Total time for all tests: ... seconds + cpu time: ... seconds + cumulative wall time: ... seconds + Features detected... + 0 """ opt = self.options L = (opt.gdb, opt.lldb, opt.valgrind, opt.massif, opt.cachegrind, opt.omega) @@ -1369,6 +1435,21 @@ def run(self): self.log("Using --optional=" + self._optional_tags_string()) available_software._allow_external = self.options.optional is True or 'external' in self.options.optional + + for h in self.options.hide: + try: + i = available_software._indices[h] + except KeyError: + pass + else: + f = available_software._features[i] + if f.is_present(): + f.hide() + self.options.hidden_features.add(f) + for g in f.joined_features(): + if g.name in self.options.optional: + self.options.optional.discard(g.name) + for o in self.options.disabled_optional: try: i = available_software._indices[o] @@ -1378,12 +1459,17 @@ def run(self): available_software._seen[i] = -1 self.log("Features to be detected: " + ','.join(available_software.detectable())) + if self.options.hidden_features: + self.log("Hidden features: " + ','.join([f.name for f in self.options.hidden_features])) self.add_files() self.expand_files_into_sources() self.filter_sources() self.sort_sources() self.run_doctests() + for f in self.options.hidden_features: + f.unhide() + self.log("Features detected for doctesting: " + ','.join(available_software.seen())) self.cleanup() @@ -1464,3 +1550,34 @@ def stringify(x): if not save_dtmode and IP is not None: IP.run_line_magic('colors', old_color) IP.config.TerminalInteractiveShell.colors = old_config_color + + +############################################################################### +# Declaration of doctest strings +############################################################################### + +test_hide=r"""r{quotmark} +{prompt}: next(graphs.fullerenes(20)) +Traceback (most recent call last): + ... +FeatureNotPresentError: buckygen is not available. +... +{prompt}: next(graphs.fullerenes(20)) # optional buckygen +Graph on 20 vertices + +{prompt}: len(list(graphs.fusenes(2))) +Traceback (most recent call last): + ... +FeatureNotPresentError: benzene is not available. +... +{prompt}: len(list(graphs.fusenes(2))) # optional benzene +1 +{prompt}: from sage.matrix.matrix_space import get_matrix_class +{prompt}: get_matrix_class(GF(25,'x'), 4, 4, False, 'meataxe') +Failed lazy import: +sage.matrix.matrix_gfpn_dense is not available. +... +{prompt}: get_matrix_class(GF(25,'x'), 4, 4, False, 'meataxe') # optional meataxe + +{quotmark} +""".format(quotmark='"""', prompt='sage') # using prompt to hide these lines from _test_enough_doctests diff --git a/src/sage/features/__init__.py b/src/sage/features/__init__.py index 5f5f9affc5f..30ae1701784 100644 --- a/src/sage/features/__init__.py +++ b/src/sage/features/__init__.py @@ -87,8 +87,10 @@ def __call__(cls, *args, **kwds): else: return type.__call__(cls, *args, **kwds) + _trivial_unique_representation_cache = dict() + class TrivialUniqueRepresentation(metaclass=TrivialClasscallMetaClass): r""" A trivial version of :class:`UniqueRepresentation` without Cython dependencies. @@ -105,6 +107,7 @@ def __classcall__(cls, *args, **options): cached = _trivial_unique_representation_cache[key] = type.__call__(cls, *args, **options) return cached + class Feature(TrivialUniqueRepresentation): r""" A feature of the runtime environment @@ -150,6 +153,7 @@ def __init__(self, name, spkg=None, url=None, description=None): self._cache_is_present = None self._cache_resolution = None + self._hidden = False def is_present(self): r""" @@ -186,6 +190,8 @@ def is_present(self): sage: TestFeature("other").is_present() FeatureTestResult('other', True) """ + if self._hidden: + return FeatureTestResult(self, False, reason="Feature `{name}` is hidden.".format(name=self.name)) # We do not use @cached_method here because we wish to use # Feature early in the build system of sagelib. if self._cache_is_present is None: @@ -238,6 +244,27 @@ def __repr__(self): description = f'{self.name!r}: {self.description}' if self.description else f'{self.name!r}' return f'Feature({description})' + def _spkg_type(self): + r""" + Return the type of the SPKG corresponding to this feature. + + EXAMPLES:: + + sage: from sage.features.databases import DatabaseCremona + sage: DatabaseCremona()._spkg_type() + 'optional' + + OUTPUT: + + The type as a string in ``('base', 'standard', 'optional', 'experimental')``. + If no SPKG corresponds to this feature ``None`` is returned. + """ + from sage.misc.package import _spkg_type + spkg = self.spkg + if not spkg: + spkg = self.name + return _spkg_type(spkg) + def resolution(self): r""" Return a suggestion on how to make :meth:`is_present` pass if it did not @@ -253,6 +280,8 @@ def resolution(self): sage: Executable(name="CSDP", spkg="csdp", executable="theta", url="https://github.com/dimpase/csdp").resolution() # optional - sage_spkg '...To install CSDP...you can try to run...sage -i csdp...Further installation instructions might be available at https://github.com/dimpase/csdp.' """ + if self._hidden: + return "Use method `unhide` to make it available again." if self._cache_resolution is not None: return self._cache_resolution lines = [] @@ -264,6 +293,117 @@ def resolution(self): self._cache_resolution = "\n".join(lines) return self._cache_resolution + def joined_features(self): + r""" + Return a list of features that ``self`` is the join of. + + OUTPUT: + + A (possibly empty) list of instances of :class:`Feature`. + + EXAMPLES:: + + sage: from sage.features.graphviz import Graphviz + sage: Graphviz().joined_features() + [Feature('dot'), Feature('neato'), Feature('twopi')] + sage: from sage.features.sagemath import sage__rings__function_field + sage: sage__rings__function_field().joined_features() + [Feature('sage.rings.function_field.function_field_polymod'), + Feature('sage.libs.singular'), + Feature('sage.libs.singular.singular'), + Feature('sage.interfaces.singular')] + sage: from sage.features.interfaces import Mathematica + sage: Mathematica().joined_features() + [] + """ + from sage.features.join_feature import JoinFeature + res = [] + if isinstance(self, JoinFeature): + for f in self._features: + res += [f] + f.joined_features() + return res + + def is_standard(self): + r""" + Return whether this feature corresponds to a standard SPKG. + + EXAMPLES:: + + sage: from sage.features.databases import DatabaseCremona, DatabaseConwayPolynomials + sage: DatabaseCremona().is_standard() + False + sage: DatabaseConwayPolynomials().is_standard() + True + """ + if self.name.startswith('sage.'): + return True + return self._spkg_type() == 'standard' + + def is_optional(self): + r""" + Return whether this feature corresponds to an optional SPKG. + + EXAMPLES:: + + sage: from sage.features.databases import DatabaseCremona, DatabaseConwayPolynomials + sage: DatabaseCremona().is_optional() + True + sage: DatabaseConwayPolynomials().is_optional() + False + """ + return self._spkg_type() == 'optional' + + def hide(self): + r""" + Hide this feature. For example this is used when the doctest option + ``--hide`` is set. Setting an installed feature as hidden pretends + that it is not available. To revert this use :meth:`unhide`. + + EXAMPLES: + + Benzene is an optional SPKG. The following test fails if it is hidden or + not installed. Thus, in the second invocation the optional tag is needed:: + + sage: from sage.features.graph_generators import Benzene + sage: Benzene().hide() + sage: len(list(graphs.fusenes(2))) + Traceback (most recent call last): + ... + FeatureNotPresentError: benzene is not available. + Feature `benzene` is hidden. + Use method `unhide` to make it available again. + + sage: Benzene().unhide() + sage: len(list(graphs.fusenes(2))) # optional benzene + 1 + """ + self._hidden = True + + def unhide(self): + r""" + Revert what :meth:`hide` does. + + EXAMPLES: + + Polycyclic is a standard GAP package since 4.10 (see :trac:`26856`). The + following test just fails if it is hidden. Thus, in the second + invocation no optional tag is needed:: + + sage: from sage.features.gap import GapPackage + sage: Polycyclic = GapPackage("polycyclic", spkg="gap_packages") + sage: Polycyclic.hide() + sage: libgap(AbelianGroup(3, [0,3,4], names="abc")) + Traceback (most recent call last): + ... + FeatureNotPresentError: gap_package_polycyclic is not available. + Feature `gap_package_polycyclic` is hidden. + Use method `unhide` to make it available again. + + sage: Polycyclic.unhide() + sage: libgap(AbelianGroup(3, [0,3,4], names="abc")) + Pcp-group with orders [ 0, 3, 4 ] + """ + self._hidden = False class FeatureNotPresentError(RuntimeError): r""" @@ -682,7 +822,9 @@ def absolute_filename(self) -> str: A :class:`FeatureNotPresentError` is raised if the file cannot be found:: sage: from sage.features import StaticFile - sage: StaticFile(name="no_such_file", filename="KaT1aihu", search_path=(), spkg="some_spkg", url="http://rand.om").absolute_filename() # optional - sage_spkg + sage: StaticFile(name="no_such_file", filename="KaT1aihu",\ + search_path=(), spkg="some_spkg",\ + url="http://rand.om").absolute_filename() # optional - sage_spkg Traceback (most recent call last): ... FeatureNotPresentError: no_such_file is not available. @@ -694,9 +836,8 @@ def absolute_filename(self) -> str: path = os.path.join(directory, self.filename) if os.path.isfile(path) or os.path.isdir(path): return os.path.abspath(path) - raise FeatureNotPresentError(self, - reason="{filename!r} not found in any of {search_path}".format(filename=self.filename, search_path=self.search_path), - resolution=self.resolution()) + reason = "{filename!r} not found in any of {search_path}".format(filename=self.filename, search_path=self.search_path) + raise FeatureNotPresentError(self, reason=reason, resolution=self.resolution()) class CythonFeature(Feature): diff --git a/src/sage/features/join_feature.py b/src/sage/features/join_feature.py index d02ad669833..36e8e557546 100644 --- a/src/sage/features/join_feature.py +++ b/src/sage/features/join_feature.py @@ -46,6 +46,7 @@ class JoinFeature(Feature): sage: F.is_present() FeatureTestResult('xxyyyy', False) """ + def __init__(self, name, features, spkg=None, url=None, description=None): """ TESTS: @@ -116,3 +117,50 @@ def is_functional(self): if not test: return test return FeatureTestResult(self, True) + + def hide(self): + r""" + Hide this feature and all its joined features. + + EXAMPLES:: + + sage: from sage.features.sagemath import sage__groups + sage: f = sage__groups() + sage: f.hide() + sage: f._features[0].is_present() + FeatureTestResult('sage.groups.perm_gps.permgroup', False) + + sage: f.require() + Traceback (most recent call last): + ... + FeatureNotPresentError: sage.groups is not available. + Feature `sage.groups` is hidden. + Use method `unhide` to make it available again. + """ + for f in self._features: + f.hide() + super(JoinFeature, self).hide() + + def unhide(self): + r""" + Revert what :meth:`hide` does. + + EXAMPLES:: + + sage: from sage.features.sagemath import sage__groups + sage: f = sage__groups() + sage: f.hide() + sage: f.is_present() + FeatureTestResult('sage.groups', False) + sage: f._features[0].is_present() + FeatureTestResult('sage.groups.perm_gps.permgroup', False) + + sage: f.unhide() + sage: f.is_present() # optional sage.groups + FeatureTestResult('sage.groups', True) + sage: f._features[0].is_present() # optional sage.groups + FeatureTestResult('sage.groups.perm_gps.permgroup', True) + """ + for f in self._features: + f.unhide() + super(JoinFeature, self).unhide() diff --git a/src/sage/misc/lazy_import.pyx b/src/sage/misc/lazy_import.pyx index d4a6c12ed5a..0d4c4830643 100644 --- a/src/sage/misc/lazy_import.pyx +++ b/src/sage/misc/lazy_import.pyx @@ -257,15 +257,20 @@ cdef class LazyImport(): if finish_startup_called: warn(f"Option ``at_startup=True`` for lazy import {self._name} not needed anymore") + feature = self._feature try: self._object = getattr(__import__(self._module, {}, {}, [self._name]), self._name) except ImportError as e: - if self._feature: + if feature: # Avoid warnings from static type checkers by explicitly importing FeatureNotPresentError. from sage.features import FeatureNotPresentError - raise FeatureNotPresentError(self._feature, reason=f'Importing {self._name} failed: {e}') + raise FeatureNotPresentError(feature, reason=f'Importing {self._name} failed: {e}') raise + if feature: + # for the case that the feature is hidden + feature.require() + if self._deprecation is not None: from sage.misc.superseded import deprecation_cython as deprecation try: diff --git a/src/sage/misc/package.py b/src/sage/misc/package.py index d0b5808e215..f6c7cccc113 100644 --- a/src/sage/misc/package.py +++ b/src/sage/misc/package.py @@ -124,6 +124,38 @@ def pip_remote_version(pkg, pypi_url=DEFAULT_PYPI, ignore_URLError=False): stable_releases = [v for v in info['releases'] if 'a' not in v and 'b' not in v] return max(stable_releases) +def _spkg_type(name): + r""" + Return the type of the Sage package with the given name. + + INPUT: + + - ``name`` -- string giving the subdirectory name of the package under + ``SAGE_PKGS`` + + EXAMPLES:: + + sage: from sage.misc.package import _spkg_type + sage: _spkg_type('pip') + 'standard' + + OUTPUT: + + The type as a string in ``('base', 'standard', 'optional', 'experimental')``. + If no ``SPKG`` exists with the given name ``None`` is returned. + """ + spkg_type = None + from sage.env import SAGE_PKGS + try: + f = open(os.path.join(SAGE_PKGS, name, "type")) + except IOError: + # Probably an empty directory => ignore + return None + + with f: + spkg_type = f.read().strip() + return spkg_type + def pip_installed_packages(normalization=None): r""" @@ -310,15 +342,10 @@ def list_packages(*pkg_types: str, pkg_sources: List[str] = ['normal', 'pip', 's for p in lp: - try: - f = open(os.path.join(SAGE_PKGS, p, "type")) - except IOError: - # Probably an empty directory => ignore + typ = _spkg_type(p) + if not typ: continue - with f: - typ = f.read().strip() - if os.path.isfile(os.path.join(SAGE_PKGS, p, "requirements.txt")): src = 'pip' elif os.path.isfile(os.path.join(SAGE_PKGS, p, "checksums.ini")):