From f51f29a09bce023836a01c1d866eeecf26a078fa Mon Sep 17 00:00:00 2001 From: Sebastian Oehms Date: Fri, 15 Jul 2022 09:03:58 +0200 Subject: [PATCH 1/3] 34185: initial version --- src/bin/sage-runtests | 4 ++ src/sage/doctest/control.py | 110 +++++++++++++++++++++++++++++++- src/sage/features/__init__.py | 115 ++++++++++++++++++++++++++++++++++ 3 files changed, 228 insertions(+), 1 deletion(-) diff --git a/src/bin/sage-runtests b/src/bin/sage-runtests index 75e2119c99a..809aedf5724 100755 --- a/src/bin/sage-runtests +++ b/src/bin/sage-runtests @@ -52,6 +52,10 @@ if __name__ == "__main__": 'if set to "all", then all tests will be run; ' 'use "!FEATURE" to disable tests marked "# optional - FEATURE". ' 'Note that "!" needs to be quoted or escaped in the shell.') + parser.add_argument("--hide", metavar="FEATURES", default="", + help='run tests pretending that the software listed in FEATURES (separated by commas) is not installed; ' + 'if "all" is listed, will also hide features corresponding to all non standard packages; ' + 'if "optional" is listed, will also hide features corresponding to optional packages.') parser.add_argument("--randorder", type=int, metavar="SEED", help="randomize order of tests") parser.add_argument("--random-seed", dest="random_seed", type=int, metavar="SEED", help="random seed (integer) for fuzzing doctests", default=os.environ.get("SAGE_DOCTEST_RANDOM_SEED")) diff --git a/src/sage/doctest/control.py b/src/sage/doctest/control.py index beccbeacf1f..8714fddc99f 100644 --- a/src/sage/doctest/control.py +++ b/src/sage/doctest/control.py @@ -56,7 +56,6 @@ except ImportError: pass - class DocTestDefaults(SageObject): """ This class is used for doctesting the Sage doctest module. @@ -135,6 +134,7 @@ def __init__(self, **kwds): # displaying user-defined optional tags and we don't want to see # the auto_optional_tags there. self.optional = set(['sage']) | auto_optional_tags + self.hide = '' # > 0: always run GC before every test # < 0: disable GC @@ -398,6 +398,28 @@ def __init__(self, options, args): if options.verbose: options.show_skipped = True + options.hidden_features = set() + if isinstance(options.hide, str): + if not len(options.hide): + options.hide = set([]) + else: + s = options.hide.lower() + options.hide = set(s.split(',')) + for h in options.hide: + if not optionaltag_regex.search(h): + raise ValueError('invalid optional tag {!r}'.format(h)) + if 'all' in options.hide: + options.hide.discard('all') + from sage.features.all import all_features + feature_names = set([f.name for f in all_features() if not f.is_standard()]) + options.hide = options.hide.union(feature_names) + if 'optional' in options.hide: + options.hide.discard('optional') + from sage.features.all import all_features + feature_names = set([f.name for f in all_features() if f.is_optional()]) + options.hide = options.hide.union(feature_names) + + options.disabled_optional = set() if isinstance(options.optional, str): s = options.optional.lower() @@ -414,6 +436,8 @@ def __init__(self, options, args): options.optional.discard('optional') from sage.misc.package import list_packages for pkg in list_packages('optional', local=True).values(): + if pkg.name in options.hide: + continue if pkg.is_installed() and pkg.installed_version == pkg.remote_version: options.optional.add(pkg.name) @@ -1320,6 +1344,49 @@ def run(self): Features detected... 0 + We test the ``--hide`` option (:trac:`34185`): + + sage: from sage.doctest.control import test_hide + sage: filename = tmp_filename(ext='.py') + sage: with open(filename, 'w') as f: + ....: f.write(test_hide) + ....: f.close() + 402 + sage: DF = DocTestDefaults(hide='buckygen,all') + sage: DC = DocTestController(DF, [filename]) + sage: DC.run() + Running doctests with ID ... + Using --optional=sage + Features to be detected: ... + Doctesting 1 file. + sage -t ....py + [2 tests, ... s] + ---------------------------------------------------------------------- + All tests passed! + ---------------------------------------------------------------------- + Total time for all tests: ... seconds + cpu time: ... seconds + cumulative wall time: ... seconds + Features detected... + 0 + + sage: DF = DocTestDefaults(hide='benzene,optional') + sage: DC = DocTestController(DF, [filename]) + sage: DC.run() + Running doctests with ID ... + Using --optional=sage + Features to be detected: ... + Doctesting 1 file. + sage -t ....py + [2 tests, ... s] + ---------------------------------------------------------------------- + All tests passed! + ---------------------------------------------------------------------- + Total time for all tests: ... seconds + cpu time: ... seconds + cumulative wall time: ... seconds + Features detected... + 0 """ opt = self.options L = (opt.gdb, opt.valgrind, opt.massif, opt.cachegrind, opt.omega) @@ -1360,6 +1427,18 @@ def run(self): self.log("Using --optional=" + self._optional_tags_string()) available_software._allow_external = self.options.optional is True or 'external' in self.options.optional + + for h in self.options.hide: + try: + i = available_software._indices[h] + except KeyError: + pass + else: + f = available_software._features[i] + if f.is_present(): + f.hide() + self.options.hidden_features.add(f) + for o in self.options.disabled_optional: try: i = available_software._indices[o] @@ -1369,12 +1448,17 @@ def run(self): available_software._seen[i] = -1 self.log("Features to be detected: " + ','.join(available_software.detectable())) + if self.options.hidden_features: + self.log("Hidden features: " + ','.join([f.name for f in self.options.hidden_features])) self.add_files() self.expand_files_into_sources() self.filter_sources() self.sort_sources() self.run_doctests() + for f in self.options.hidden_features: + f.unhide() + self.log("Features detected for doctesting: " + ','.join(available_software.seen())) self.cleanup() @@ -1455,3 +1539,27 @@ def stringify(x): if not save_dtmode and IP is not None: IP.run_line_magic('colors', old_color) IP.config.TerminalInteractiveShell.colors = old_config_color + + +############################################################################### +# Declaration of doctest strings +############################################################################### + +test_hide=r"""{} +sage: next(graphs.fullerenes(20)) +Traceback (most recent call last): + ... +FeatureNotPresentError: buckygen is not available. +... +sage: next(graphs.fullerenes(20)) # optional buckygen +Graph on 20 vertices + +sage: len(list(graphs.fusenes(2))) +Traceback (most recent call last): + ... +FeatureNotPresentError: benzene is not available. +... +sage: len(list(graphs.fusenes(2))) # optional benzene +1 +{} +""".format('r"""', '"""') diff --git a/src/sage/features/__init__.py b/src/sage/features/__init__.py index fd899aa4770..bbe5dd1bd34 100644 --- a/src/sage/features/__init__.py +++ b/src/sage/features/__init__.py @@ -137,6 +137,7 @@ def __init__(self, name, spkg=None, url=None, description=None): self._cache_is_present = None self._cache_resolution = None + self._hidden = False def is_present(self): r""" @@ -173,6 +174,8 @@ def is_present(self): sage: TestFeature("other").is_present() FeatureTestResult('other', True) """ + if self._hidden: + return FeatureTestResult(self, False, reason="Feature `{name}` is hidden.".format(name=self.name)) # We do not use @cached_method here because we wish to use # Feature early in the build system of sagelib. if self._cache_is_present is None: @@ -225,6 +228,33 @@ def __repr__(self): description = f'{self.name!r}: {self.description}' if self.description else f'{self.name!r}' return f'Feature({description})' + def _spkg_type(self): + r""" + Return the type of the SPKG corresponding to this feature. + + EXAMPLES:: + + sage: from sage.features.databases import DatabaseCremona + sage: DatabaseCremona()._spkg_type() + 'optional' + + OUTPUT: + + The type as a string in ``('base', 'standard', 'optional', 'experimental')``. + If no SPKG corresponds to this feature ``None`` is returned. + """ + spkg_type = None + from sage.env import SAGE_PKGS + try: + f = open(os.path.join(SAGE_PKGS, self.name, "type")) + except IOError: + # Probably an empty directory => ignore + return None + + with f: + spkg_type = f.read().strip() + return spkg_type + def resolution(self): r""" Return a suggestion on how to make :meth:`is_present` pass if it did not @@ -240,6 +270,8 @@ def resolution(self): sage: Executable(name="CSDP", spkg="csdp", executable="theta", url="https://github.com/dimpase/csdp").resolution() # optional - sage_spkg '...To install CSDP...you can try to run...sage -i csdp...Further installation instructions might be available at https://github.com/dimpase/csdp.' """ + if self._hidden: + return "Use method `unhide` to make it available again." if self._cache_resolution is not None: return self._cache_resolution lines = [] @@ -251,6 +283,89 @@ def resolution(self): self._cache_resolution = "\n".join(lines) return self._cache_resolution + def is_standard(self): + r""" + Return whether this feature corresponds to a standard SPKG. + + EXAMPLES:: + + sage: from sage.features.databases import DatabaseCremona, DatabaseConwayPolynomials + sage: DatabaseCremona().is_standard() + False + sage: DatabaseConwayPolynomials().is_standard() + True + """ + if self.name.startswith('sage.'): + return True + return self._spkg_type() == 'standard' + + def is_optional(self): + r""" + Return whether this feature corresponds to an optional SPKG. + + EXAMPLES:: + + sage: from sage.features.databases import DatabaseCremona, DatabaseConwayPolynomials + sage: DatabaseCremona().is_optional() + True + sage: DatabaseConwayPolynomials().is_optional() + False + """ + return self._spkg_type() == 'optional' + + def hide(self): + r""" + Hide this feature. For example this is used when the doctest option + ``--hide``is set. Setting an installed feature as hidden pretends + that it is not available. To revert this use :meth:`unhide`. + + EXAMPLES: + + Benzene is an optional SPKG. The following test fails if it is hidden or + not installed. Thus, in the second invocation the optional tag is needed:: + + sage: from sage.features.graph_generators import Benzene + sage: Benzene().hide() + sage: len(list(graphs.fusenes(2))) + Traceback (most recent call last): + ... + FeatureNotPresentError: benzene is not available. + Feature `benzene` is hidden. + Use method `unhide` to make it available again. + + sage: Benzene().unhide() + sage: len(list(graphs.fusenes(2))) # optional benzene + 1 + """ + self._hidden = True + + def unhide(self): + r""" + Revert what :meth:`hide` does. + + EXAMPLES: + + Polycyclic is a standard GAP package since 4.10 (see :trac:`26856`). The + following test just fails if it is hidden. Thus, in the second + invocation no optional tag is needed:: + + sage: from sage.features.gap import GapPackage + sage: Polycyclic = GapPackage("polycyclic", spkg="gap_packages") + sage: Polycyclic.hide() + sage: libgap(AbelianGroup(3, [0,3,4], names="abc")) + Traceback (most recent call last): + ... + FeatureNotPresentError: gap_package_polycyclic is not available. + Feature `gap_package_polycyclic` is hidden. + Use method `unhide` to make it available again. + + sage: Polycyclic.unhide() + sage: libgap(AbelianGroup(3, [0,3,4], names="abc")) + Pcp-group with orders [ 0, 3, 4 ] + """ + self._hidden = False + + class FeatureNotPresentError(RuntimeError): From 3071dbc20e76459a23282001311b6fee3cd897c3 Mon Sep 17 00:00:00 2001 From: Sebastian Oehms Date: Mon, 18 Jul 2022 18:01:12 +0200 Subject: [PATCH 2/3] 34185: correction according to review --- src/sage/features/__init__.py | 12 ++-------- src/sage/misc/package.py | 41 +++++++++++++++++++++++++++++------ 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/src/sage/features/__init__.py b/src/sage/features/__init__.py index bbe5dd1bd34..5887c1f26dd 100644 --- a/src/sage/features/__init__.py +++ b/src/sage/features/__init__.py @@ -243,17 +243,9 @@ def _spkg_type(self): The type as a string in ``('base', 'standard', 'optional', 'experimental')``. If no SPKG corresponds to this feature ``None`` is returned. """ + from sage.misc.package import _spkg_type + return _spkg_type(self.name) spkg_type = None - from sage.env import SAGE_PKGS - try: - f = open(os.path.join(SAGE_PKGS, self.name, "type")) - except IOError: - # Probably an empty directory => ignore - return None - - with f: - spkg_type = f.read().strip() - return spkg_type def resolution(self): r""" diff --git a/src/sage/misc/package.py b/src/sage/misc/package.py index 849ee71f3fb..a37bb95db3f 100644 --- a/src/sage/misc/package.py +++ b/src/sage/misc/package.py @@ -125,6 +125,38 @@ def pip_remote_version(pkg, pypi_url=DEFAULT_PYPI, ignore_URLError=False): stable_releases = [v for v in info['releases'] if 'a' not in v and 'b' not in v] return max(stable_releases) +def _spkg_type(name): + r""" + Return the type of the Sage package with the given name. + + INPUT: + + - ``name`` -- string giving the subdirectory name of the package under + ``SAGE_PKGS`` + + EXAMPLES:: + + sage: from sage.misc.package import _spkg_type + sage: _spkg_type('pip') + 'standard' + + OUTPUT: + + The type as a string in ``('base', 'standard', 'optional', 'experimental')``. + If no ``SPKG`` exists with the given name ``None`` is returned. + """ + spkg_type = None + from sage.env import SAGE_PKGS + try: + f = open(os.path.join(SAGE_PKGS, name, "type")) + except IOError: + # Probably an empty directory => ignore + return None + + with f: + spkg_type = f.read().strip() + return spkg_type + def pip_installed_packages(normalization=None): r""" @@ -311,15 +343,10 @@ def list_packages(*pkg_types: str, pkg_sources: List[str] = ['normal', 'pip', 's for p in lp: - try: - f = open(os.path.join(SAGE_PKGS, p, "type")) - except IOError: - # Probably an empty directory => ignore + typ = _spkg_type(p) + if not typ: continue - with f: - typ = f.read().strip() - if os.path.isfile(os.path.join(SAGE_PKGS, p, "requirements.txt")): src = 'pip' elif os.path.isfile(os.path.join(SAGE_PKGS, p, "checksums.ini")): From b4b562c56206b83fb23f259e6cba4f75304b43b9 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Mon, 25 Jul 2022 08:39:20 +0200 Subject: [PATCH 3/3] 34185: take care of joined features --- src/sage/doctest/control.py | 16 +++++++++-- src/sage/features/__init__.py | 22 +++++++++++++++ src/sage/features/join_feature.py | 47 +++++++++++++++++++++++++++++++ src/sage/misc/lazy_import.pyx | 7 ++++- 4 files changed, 88 insertions(+), 4 deletions(-) diff --git a/src/sage/doctest/control.py b/src/sage/doctest/control.py index 0d78b961d86..535c71557b1 100644 --- a/src/sage/doctest/control.py +++ b/src/sage/doctest/control.py @@ -1351,7 +1351,7 @@ def run(self): sage: with open(filename, 'w') as f: ....: f.write(test_hide) ....: f.close() - 402 + 729 sage: DF = DocTestDefaults(hide='buckygen,all') sage: DC = DocTestController(DF, [filename]) sage: DC.run() @@ -1360,7 +1360,7 @@ def run(self): Features to be detected: ... Doctesting 1 file. sage -t ....py - [2 tests, ... s] + [4 tests, ... s] ---------------------------------------------------------------------- All tests passed! ---------------------------------------------------------------------- @@ -1378,7 +1378,7 @@ def run(self): Features to be detected: ... Doctesting 1 file. sage -t ....py - [2 tests, ... s] + [4 tests, ... s] ---------------------------------------------------------------------- All tests passed! ---------------------------------------------------------------------- @@ -1438,6 +1438,9 @@ def run(self): if f.is_present(): f.hide() self.options.hidden_features.add(f) + for g in f.joined_features(): + if g.name in self.options.optional: + self.options.optional.discard(g.name) for o in self.options.disabled_optional: try: @@ -1561,5 +1564,12 @@ def stringify(x): ... sage: len(list(graphs.fusenes(2))) # optional benzene 1 +sage: from sage.matrix.matrix_space import get_matrix_class +sage: get_matrix_class(GF(25,'x'), 4, 4, False, 'meataxe') +Failed lazy import: +sage.matrix.matrix_gfpn_dense is not available. +... +sage: get_matrix_class(GF(25,'x'), 4, 4, False, 'meataxe') # optional meataxe + {} """.format('r"""', '"""') diff --git a/src/sage/features/__init__.py b/src/sage/features/__init__.py index 5887c1f26dd..1750908c74b 100644 --- a/src/sage/features/__init__.py +++ b/src/sage/features/__init__.py @@ -275,6 +275,28 @@ def resolution(self): self._cache_resolution = "\n".join(lines) return self._cache_resolution + def joined_features(self): + r""" + Return a list of features joined with ``self``. + + OUTPUT: + + A (possibly empty) list of instances of :class:`Feature`. + + EXAMPLES:: + + sage: from sage.features.graphviz import Graphviz + sage: Graphviz().joined_features() + [Feature('dot'), Feature('neato'), Feature('twopi')] + sage: from sage.features.interfaces import Mathematica + sage: Mathematica().joined_features() + [] + """ + from sage.features.join_feature import JoinFeature + if isinstance(self, JoinFeature): + return self._features + return [] + def is_standard(self): r""" Return whether this feature corresponds to a standard SPKG. diff --git a/src/sage/features/join_feature.py b/src/sage/features/join_feature.py index b29241eb462..218246190c4 100644 --- a/src/sage/features/join_feature.py +++ b/src/sage/features/join_feature.py @@ -94,3 +94,50 @@ def is_functional(self): if not test: return test return FeatureTestResult(self, True) + + def hide(self): + r""" + Hide this feature and all its joined features. + + EXAMPLES: + + sage: from sage.features.sagemath import sage__groups + sage: f = sage__groups() + sage: f.hide() + sage: f._features[0].is_present() + FeatureTestResult('sage.groups.perm_gps.permgroup', False) + + sage: f.require() + Traceback (most recent call last): + ... + FeatureNotPresentError: sage.groups is not available. + Feature `sage.groups` is hidden. + Use method `unhide` to make it available again. + """ + for f in self._features: + f.hide() + super(JoinFeature, self).hide() + + def unhide(self): + r""" + Hide this feature and all its joined features. + + EXAMPLES: + + sage: from sage.features.sagemath import sage__groups + sage: f = sage__groups() + sage: f.hide() + sage: f.is_present() + FeatureTestResult('sage.groups', False) + sage: f._features[0].is_present() + FeatureTestResult('sage.groups.perm_gps.permgroup', False) + + sage: f.unhide() + sage: f.is_present() # optional sage.groups + FeatureTestResult('sage.groups', True) + sage: f._features[0].is_present() # optional sage.groups + FeatureTestResult('sage.groups.perm_gps.permgroup', True) + """ + for f in self._features: + f.unhide() + super(JoinFeature, self).unhide() diff --git a/src/sage/misc/lazy_import.pyx b/src/sage/misc/lazy_import.pyx index 97055afb01a..bf41b4b9fca 100644 --- a/src/sage/misc/lazy_import.pyx +++ b/src/sage/misc/lazy_import.pyx @@ -248,13 +248,18 @@ cdef class LazyImport(): if finish_startup_called: warn(f"Option ``at_startup=True`` for lazy import {self._name} not needed anymore") + feature = self._feature try: self._object = getattr(__import__(self._module, {}, {}, [self._name]), self._name) except ImportError as e: - if self._feature: + if feature: raise FeatureNotPresentError(self._feature, reason=f'Importing {self._name} failed: {e}') raise + if feature: + # for the case that the feature is hidden + feature.require() + name = self._as_name if self._deprecation is not None: from sage.misc.superseded import deprecation_cython as deprecation