From d7ff532b3519c2904b9c61dfd07ad542c639017b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Sat, 11 Aug 2018 04:00:43 +0200 Subject: [PATCH 01/14] A proof of concept of airspeed velocity integration --- .gitignore | 5 ++ asv.conf.json | 12 ++++ src/sage/benchmark/__init__.py | 128 +++++++++++++++++++++++++++++++++ src/sage/doctest/forker.py | 4 +- 4 files changed, 148 insertions(+), 1 deletion(-) create mode 100644 asv.conf.json create mode 100644 src/sage/benchmark/__init__.py diff --git a/.gitignore b/.gitignore index 101d349690c..3e4e0d5a12b 100644 --- a/.gitignore +++ b/.gitignore @@ -81,3 +81,8 @@ $RECYCLE.BIN/ ########### .ipynb_checkpoints Untitled*.ipynb + +##################### +# airspeed velocity # +##################### +/.asv diff --git a/asv.conf.json b/asv.conf.json new file mode 100644 index 00000000000..71aef8c43e9 --- /dev/null +++ b/asv.conf.json @@ -0,0 +1,12 @@ +{ + "version": 1, + "project": "sage", + "project_url": "https://sagemath.org", + "repo": ".", + "plugins": ["sage_asv"], + "environment_type": "sage", + "env_dir": ".asv/env", + "results_dir": ".asv/results", + "html_dir": ".asv/html", + "benchmark_dir": "src/sage/benchmark/" +} diff --git a/src/sage/benchmark/__init__.py b/src/sage/benchmark/__init__.py new file mode 100644 index 00000000000..0bced064c06 --- /dev/null +++ b/src/sage/benchmark/__init__.py @@ -0,0 +1,128 @@ +from sage.all import * + +from sage.doctest.control import DocTestDefaults, DocTestController +from sage.doctest.forker import SageDocTestRunner, DocTestTask +from sage.doctest.parsing import parse_optional_tags + +import timeit +import doctest + +DEFAULTS = DocTestDefaults() +DEFAULTS.serial = True +DEFAULTS.long = True + +PREFIX = 'track__' + +def myglob(path, pattern): + # python 2 does not have support for ** in glob patterns + import fnmatch + import os + + matches = [] + for root, dirnames, filenames in os.walk(path): + for filename in fnmatch.filter(filenames, pattern): + matches.append(os.path.join(root, filename)) + return matches + +class BenchmarkMetaclass(type): + _dir = None + + def _run(cls, fname=[SAGE_SRC + '/sage/']): + old_runner = DocTestTask.runner + DocTestTask.runner = BenchmarkRunner + try: + DocTestController(DEFAULTS, fname).run() + finally: + DocTestTask.runner = old_runner + + def __dir__(cls): + if cls._dir is None: + cls._dir = set() + BenchmarkRunner._dir = cls._dir + cls._run() + return list(cls._dir) + + def __getattr__(cls, name): + if not name.startswith(PREFIX): + raise AttributeError + cls.create_timer(name) + return getattr(cls, name) + + def create_timer(cls, name): + try: + type.__getattr__(cls, name) + except AttributeError: + def time_doctest(self): + BenchmarkRunner._selected = name + BenchmarkRunner._time = 0 + cls._run(myglob(os.path.join(SAGE_SRC,'sage'), BenchmarkRunner.decode(name)+"*.*")) + return BenchmarkRunner._time + + time_doctest.__name__ = name + setattr(cls, name, time_doctest) + +class Benchmarks(object): + __metaclass__ = BenchmarkMetaclass + + def __getattr__(self, name): + if not name.startswith(PREFIX): + raise AttributeError + type(self).create_timer(name) + return getattr(self, name) + +class BenchmarkRunner(SageDocTestRunner): + _selected = None + _dir = set() + _time = None + + @classmethod + def encode(cls, prefix, filename, name, digest): + module = os.path.splitext(os.path.basename(filename))[0] + method = name.split('.')[-1] + return PREFIX+module+"__"+method+"__"+digest + + @classmethod + def decode(cls, name): + # This is not the full file name but only the bit up to the first _. + # But that's good enough as we later seach for this with a glob pattern + return name.split("__")[1] + + def run(self, test, clear_globs=True, *args, **kwargs): + self._do_not_run_tests = True + super(BenchmarkRunner, self).run(test, *args, clear_globs=False, **kwargs) + + name = BenchmarkRunner.encode(PREFIX, test.filename, test.name, self.running_doctest_digest.hexdigest()) + if name not in BenchmarkRunner._dir: + for example in test.examples: + if isinstance(example, doctest.Example): + if "long time" in parse_optional_tags(example.source): + BenchmarkRunner._dir.add(name) + break + + self._do_not_run_tests = False + if type(self)._selected is not None: + if name == type(self)._selected: + pass + else: + self._do_not_run_tests = True + if not self._do_not_run_tests: + super(BenchmarkRunner, self).run(test, *args, clear_globs=clear_globs, **kwargs) + return + + def compile_and_execute(self, example, compiler, globs, *args, **kwargs): + if self._do_not_run_tests: + compiler = lambda example: compile("print(%s.encode('utf8'))"%(repr(example.want),), "want.py", "single") + super(BenchmarkRunner, self).compile_and_execute(example, compiler, globs, *args, **kwargs) + else: + compiled = compiler(example) + # TODO: According to https://asv.readthedocs.io/en/latest/writing_benchmarks.html, time.process_time would be better here but it's not available in Python 2 + start = timeit.default_timer() + exec(compiled, globs) + end = timeit.default_timer() + type(self)._time += end-start + +class BenchmarkRunningRunner(SageDocTestRunner): + def run(self, test, *args, **kwargs): + print(test,BenchmarkRunningRunner._test) + if test == BenchmarkRunningRunner._test: + super(BenchmarkRunningRunner).run(test, *args, **kwargs) diff --git a/src/sage/doctest/forker.py b/src/sage/doctest/forker.py index c67a62f7d6f..2e5c9c42063 100644 --- a/src/sage/doctest/forker.py +++ b/src/sage/doctest/forker.py @@ -2290,6 +2290,8 @@ class DocTestTask(object): ['cputime', 'err', 'failures', 'optionals', 'walltime'] """ + runner = SageDocTestRunner + if six.PY2: extra_globals = {} else: @@ -2369,7 +2371,7 @@ def __call__(self, options, outtmpfile=None, msgfile=None, result_queue=None): """ result = None try: - runner = SageDocTestRunner( + runner = DocTestTask.runner( SageOutputChecker(), verbose=options.verbose, outtmpfile=outtmpfile, From 8883d8db3edd3817e8e02470d42210fed918afa8 Mon Sep 17 00:00:00 2001 From: David Roe Date: Thu, 9 Feb 2023 18:36:59 +0100 Subject: [PATCH 02/14] Add asv_stats_path option to sage -t --- src/bin/sage-runtests | 2 ++ src/doc/en/developer/doctesting.rst | 4 ++++ src/sage/doctest/control.py | 30 ++++++++++++++++++++++++++ src/sage/doctest/forker.py | 33 +++++++++++++++++++++++++---- src/sage/doctest/reporting.py | 29 +++++++++++++++++-------- 5 files changed, 85 insertions(+), 13 deletions(-) diff --git a/src/bin/sage-runtests b/src/bin/sage-runtests index 81dff8d5bf3..2a2792a8a46 100755 --- a/src/bin/sage-runtests +++ b/src/bin/sage-runtests @@ -97,6 +97,8 @@ if __name__ == "__main__": help="path to a json dictionary for timings and failure status for each file from previous runs; it will be updated in this run") parser.add_argument("--baseline_stats_path", "--baseline-stats-path", default=None, help="path to a json dictionary for timings and failure status for each file, to be used as a baseline; it will not be updated") + parser.add_argument("--asv_stats_path", "--asv-stats-path", default=None, + help="path to a json dictionary for storing individual doctest timings for use with airspeed velocity speed regression testing") class GCAction(argparse.Action): def __call__(self, parser, namespace, values, option_string=None): diff --git a/src/doc/en/developer/doctesting.rst b/src/doc/en/developer/doctesting.rst index 9619460c30f..a8e79f98b9f 100644 --- a/src/doc/en/developer/doctesting.rst +++ b/src/doc/en/developer/doctesting.rst @@ -1338,3 +1338,7 @@ utilized most efficiently):: Sorting sources by runtime so that slower doctests are run first.... Doctesting 2067 files using 2 threads. ... + +To give a json file for recording the timings for each doctest, use the +``--asv_stats_path`` flag. These statistics can be collected by the airspeed +velocity code to generate interactive speed regression webpages. diff --git a/src/sage/doctest/control.py b/src/sage/doctest/control.py index aa1268a6a27..d6ded325d0a 100644 --- a/src/sage/doctest/control.py +++ b/src/sage/doctest/control.py @@ -130,6 +130,7 @@ def __init__(self, **kwds): self.show_skipped = False self.target_walltime = -1 self.baseline_stats_path = None + self.asv_stats_path = None # sage-runtests contains more optional tags. Technically, adding # auto_optional_tags here is redundant, since that is added @@ -399,6 +400,7 @@ def __init__(self, options, args): options.nthreads = 1 if options.verbose: options.show_skipped = True + options.use_asv = (options.asv_stats_path is not None) options.disabled_optional = set() if isinstance(options.optional, str): @@ -478,6 +480,7 @@ def __init__(self, options, args): self.baseline_stats = {} if options.baseline_stats_path: self.load_baseline_stats(options.baseline_stats_path) + self.asv_stats = {} self._init_warn_long() if self.options.random_seed is None: @@ -692,6 +695,30 @@ def save_stats(self, filename): with atomic_write(filename) as stats_file: json.dump(self.stats, stats_file) + def save_asv_stats(self, filename): + """ + Save individual doctest stats from the most recent run as a JSON file, + for use in speed regression testing. + + WARNING: This function overwrites the file. + + EXAMPLES:: + + sage: from sage.doctest.control import DocTestDefaults, DocTestController + sage: DC = DocTestController(DocTestDefaults(), []) + sage: DC.asv_stats['sage.doctest.control'] = {'walltime':1.0r} + sage: filename = tmp_filename() + sage: DC.save_avs_stats(filename) + sage: import json + sage: with open(filename) as f: + ....: D = json.load(f) + sage: D['sage.doctest.control'] + {'walltime': 1.0} + """ + from sage.misc.temporary_file import atomic_write + with atomic_write(filename) as stats_file: + json.dump(self.asv_stats, stats_file) + def log(self, s, end="\n"): """ Log the string ``s + end`` (where ``end`` is a newline by default) @@ -1110,6 +1137,9 @@ def cleanup(self, final=True): """ self.stats.update(self.reporter.stats) self.save_stats(self.options.stats_path) + if self.options.asv_stats_path: + self.asv_stats.update(self.reporter.asv_stats) + self.save_asv_stats(self.options.asv_stats_path) # Close the logfile if final and self.logfile is not None: self.logfile.close() diff --git a/src/sage/doctest/forker.py b/src/sage/doctest/forker.py index 961facf3c50..bd1f9c0295b 100644 --- a/src/sage/doctest/forker.py +++ b/src/sage/doctest/forker.py @@ -505,6 +505,8 @@ def __init__(self, *args, **kwds): - ``verbose`` -- boolean, determines whether verbose printing is enabled. + - ``sage_options`` -- dictionary with Sage options + - ``optionflags`` -- Controls the comparison with the expected output. See :mod:`testmod` for more information. @@ -790,6 +792,13 @@ def compiler(example): # Restore the option flags (in case they were modified) self.optionflags = original_optionflags + if self.options.use_asv: + if failures == 0: + # We record the timing for this test for speed regression purposes + test.walltime = sum(example.walltime for example in test.examples) + else: + test.walltime = -1 + # Record and return the number of failures and tries. self._DocTestRunner__record_outcome(test, failures, tries) self.total_walltime_skips += walltime_skips @@ -1533,14 +1542,16 @@ def report_unexpected_exception(self, out, test, example, exc_info): self._fakeout.start_spoofing() return returnval - def update_results(self, D): + def update_results(self, D, doctests): """ When returning results we pick out the results of interest since many attributes are not pickleable. INPUT: - - ``D`` -- a dictionary to update with cputime and walltime + - ``D`` -- a dictionary to update with cputime and walltime (and asv_results if that option set) + + - ``doctests`` -- a list of doctests OUTPUT: @@ -1564,7 +1575,7 @@ def update_results(self, D): TestResults(failed=0, attempted=4) sage: T.stop().annotate(DTR) sage: D = DictAsObject({'cputime':[],'walltime':[],'err':None}) - sage: DTR.update_results(D) + sage: DTR.update_results(D, doctests) 0 sage: sorted(list(D.items())) [('cputime', [...]), ('err', None), ('failures', 0), ('tests', 4), ('walltime', [...]), ('walltime_skips', 0)] @@ -1576,6 +1587,20 @@ def update_results(self, D): D[key].append(self.__dict__[key]) D['tests'] = self.total_performed_tests D['walltime_skips'] = self.total_walltime_skips + if self.options.use_asv: + + def asv_label(test): + """ + This information is hashed by asv to determine continuity for speed regressions + """ + name = test.name + code = "\n".join([example.source for example in test.examples]) + return f"{name}|{code}" + + D['asv_times'] = {} + for test in doctests: + if test.walltime > 0: # -1 indicates a failure, 0 means nothing was actually run + D['asv_times'][asv_label(test)] = test.walltime if hasattr(self, 'failures'): D['failures'] = self.failures return self.failures @@ -2488,7 +2513,7 @@ def __call__(self, options, outtmpfile=None, msgfile=None, result_queue=None): for it in range(N): doctests, extras = self._run(runner, options, results) runner.summarize(options.verbose) - if runner.update_results(results): + if runner.update_results(results, doctests): break if extras['tab']: diff --git a/src/sage/doctest/reporting.py b/src/sage/doctest/reporting.py index 51c6ea55626..17690ba5e5f 100644 --- a/src/sage/doctest/reporting.py +++ b/src/sage/doctest/reporting.py @@ -113,6 +113,7 @@ def __init__(self, controller): self.postscript = dict(lines=[], cputime=0, walltime=0) self.sources_completed = 0 self.stats = {} + self.asv_stats = {} self.error_status = 0 def were_doctests_with_optional_tag_run(self, tag): @@ -381,6 +382,7 @@ def report(self, source, timeout, return_code, results, output, pid=None): postscript = self.postscript stats = self.stats basename = source.basename + options = self.controller.options if self.controller.baseline_stats: the_baseline_stats = self.controller.baseline_stats.get(basename, {}) else: @@ -500,6 +502,15 @@ def report(self, source, timeout, return_code, results, output, pid=None): stats[basename] = dict(failed=True, walltime=wall) else: stats[basename] = dict(walltime=wall) + if options.use_asv: + asv = [] + for label, time in result_dict.asv_times.items(): + name, code = label.split("|", 1) + flags = list(options.optional) + if options.long: + flags.append("long") + asv.append([name, flags, code, time]) + self.asv_stats[basename] = asv postscript['cputime'] += cpu postscript['walltime'] += wall @@ -510,29 +521,29 @@ def report(self, source, timeout, return_code, results, output, pid=None): for tag in sorted(optionals): nskipped = optionals[tag] if tag == "long time": - if not self.controller.options.long: - if self.controller.options.show_skipped: + if not options.long: + if options.show_skipped: log(" %s not run"%(count_noun(nskipped, "long test"))) elif tag == "not tested": - if self.controller.options.show_skipped: + if options.show_skipped: log(" %s not run"%(count_noun(nskipped, "not tested test"))) elif tag == "not implemented": - if self.controller.options.show_skipped: + if options.show_skipped: log(" %s for not implemented functionality not run"%(count_noun(nskipped, "test"))) else: if not self.were_doctests_with_optional_tag_run(tag): if tag == "bug": - if self.controller.options.show_skipped: + if options.show_skipped: log(" %s not run due to known bugs"%(count_noun(nskipped, "test"))) elif tag == "": - if self.controller.options.show_skipped: + if options.show_skipped: log(" %s not run"%(count_noun(nskipped, "unlabeled test"))) else: - if self.controller.options.show_skipped: + if options.show_skipped: log(" %s not run"%(count_noun(nskipped, tag + " test"))) nskipped = result_dict.walltime_skips - if self.controller.options.show_skipped: + if options.show_skipped: log(" %s not run because we ran out of time"%(count_noun(nskipped, "test"))) if nskipped != 0: @@ -547,7 +558,7 @@ def report(self, source, timeout, return_code, results, output, pid=None): total = "%d%% of tests run"%(round(100*ntests_run/float(ntests_run + nskipped))) else: total = count_noun(ntests, "test") - if not (self.controller.options.only_errors and not f): + if not (options.only_errors and not f): log(" [%s, %s%.2f s]" % (total, "%s, "%(count_noun(f, "failure")) if f else "", wall)) self.sources_completed += 1 From beed61685a96bfdb56e3ad6c3c97edc83b2c6743 Mon Sep 17 00:00:00 2001 From: David Roe Date: Thu, 9 Feb 2023 18:43:36 +0100 Subject: [PATCH 03/14] Remove doubled newlines --- src/sage/doctest/forker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sage/doctest/forker.py b/src/sage/doctest/forker.py index bd1f9c0295b..46ec3cabd69 100644 --- a/src/sage/doctest/forker.py +++ b/src/sage/doctest/forker.py @@ -1594,7 +1594,7 @@ def asv_label(test): This information is hashed by asv to determine continuity for speed regressions """ name = test.name - code = "\n".join([example.source for example in test.examples]) + code = "".join([example.source for example in test.examples]) return f"{name}|{code}" D['asv_times'] = {} From 8ffe8a9cd3d7afeb82ff23c2a7e4dd7f16511ccd Mon Sep 17 00:00:00 2001 From: David Roe Date: Thu, 9 Feb 2023 22:26:42 +0100 Subject: [PATCH 04/14] Save optional tags on each example --- src/sage/doctest/forker.py | 12 +++++++++++- src/sage/doctest/parsing.py | 1 + src/sage/doctest/reporting.py | 23 ++++++++++++++--------- 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/src/sage/doctest/forker.py b/src/sage/doctest/forker.py index 46ec3cabd69..a3bbd8d6ec9 100644 --- a/src/sage/doctest/forker.py +++ b/src/sage/doctest/forker.py @@ -1595,7 +1595,17 @@ def asv_label(test): """ name = test.name code = "".join([example.source for example in test.examples]) - return f"{name}|{code}" + opts = set() + for example in test.examples: + # The added sig_on_count() test has no tags + if hasattr(example, "optional_tags"): + opts = opts.union(example.optional_tags) + olist = [] + if "long" in opts: + olist = ["long"] + opts.remove("long") + olist.extend(sorted(opts)) + return f"{name}|{','.join(olist)}|{code}" D['asv_times'] = {} for test in doctests: diff --git a/src/sage/doctest/parsing.py b/src/sage/doctest/parsing.py index 26d793b96bf..e4cc04f4b21 100644 --- a/src/sage/doctest/parsing.py +++ b/src/sage/doctest/parsing.py @@ -633,6 +633,7 @@ def parse(self, string, *args): for item in res: if isinstance(item, doctest.Example): optional_tags = parse_optional_tags(item.source) + item.optional_tags = optional_tags # for asv if optional_tags: for tag in optional_tags: self.optionals[tag] += 1 diff --git a/src/sage/doctest/reporting.py b/src/sage/doctest/reporting.py index 17690ba5e5f..40d55d802cc 100644 --- a/src/sage/doctest/reporting.py +++ b/src/sage/doctest/reporting.py @@ -502,15 +502,20 @@ def report(self, source, timeout, return_code, results, output, pid=None): stats[basename] = dict(failed=True, walltime=wall) else: stats[basename] = dict(walltime=wall) - if options.use_asv: - asv = [] - for label, time in result_dict.asv_times.items(): - name, code = label.split("|", 1) - flags = list(options.optional) - if options.long: - flags.append("long") - asv.append([name, flags, code, time]) - self.asv_stats[basename] = asv + if options.use_asv: + asv = [] + for label, time in result_dict.asv_times.items(): + name, opts, code = label.split("|", 2) + if name.startswith(basename): + name = name[len(basename):] + if name.startswith("."): + name = name[1:] + if opts: + opts = opts.split(",") + else: + opts = [] + asv.append([name, opts, code, time]) + self.asv_stats[basename] = asv postscript['cputime'] += cpu postscript['walltime'] += wall From e9751afdc3c5255170658f6dc48a5e90ae36da70 Mon Sep 17 00:00:00 2001 From: David Roe Date: Thu, 9 Feb 2023 22:45:19 +0100 Subject: [PATCH 05/14] Better recovery of function names when arguments wrap --- src/sage/doctest/sources.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/sage/doctest/sources.py b/src/sage/doctest/sources.py index 6321e73a5e8..c078e5508b3 100644 --- a/src/sage/doctest/sources.py +++ b/src/sage/doctest/sources.py @@ -1100,14 +1100,16 @@ def starting_docstring(self, line): # quotematch wasn't None, but then we mishandle classes # that don't have a docstring. if not self.code_wrapping and self.last_indent >= 0 and indent > self.last_indent: - name = name_regex.match(self.last_line) + name = name_regex.match(self.last_line.replace("\n", "")) if name: name = name.groups()[0] self.qualified_name[indent] = name elif quotematch: self.qualified_name[indent] = '?' self._update_quotetype(line) - if line[indent] != '#' and not self.code_wrapping: + if self.code_wrapping: + self.last_line += line + elif line[indent] != '#': self.last_line, self.last_indent = line, indent self.code_wrapping = not (self.paren_count == self.bracket_count == self.curly_count == 0) return quotematch From 169fd19545d6d3f48c4ea008cf49b8c300525a00 Mon Sep 17 00:00:00 2001 From: David Roe Date: Thu, 9 Feb 2023 22:56:29 +0100 Subject: [PATCH 06/14] Use sage source --- src/sage/doctest/forker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sage/doctest/forker.py b/src/sage/doctest/forker.py index a3bbd8d6ec9..157c4885cb6 100644 --- a/src/sage/doctest/forker.py +++ b/src/sage/doctest/forker.py @@ -1594,7 +1594,7 @@ def asv_label(test): This information is hashed by asv to determine continuity for speed regressions """ name = test.name - code = "".join([example.source for example in test.examples]) + code = "".join([example.sage_source for example in test.examples]) opts = set() for example in test.examples: # The added sig_on_count() test has no tags From d441f2f022a0141bdc324fa650de5193e86b3cdf Mon Sep 17 00:00:00 2001 From: David Roe Date: Thu, 9 Feb 2023 23:10:58 +0100 Subject: [PATCH 07/14] Fix some doctests --- src/sage/doctest/control.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sage/doctest/control.py b/src/sage/doctest/control.py index d6ded325d0a..4fc6084a96f 100644 --- a/src/sage/doctest/control.py +++ b/src/sage/doctest/control.py @@ -131,6 +131,7 @@ def __init__(self, **kwds): self.target_walltime = -1 self.baseline_stats_path = None self.asv_stats_path = None + self.use_asv = False # sage-runtests contains more optional tags. Technically, adding # auto_optional_tags here is redundant, since that is added From 9d43f55493d49483016c396b246162c38b818665 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Fri, 10 Feb 2023 00:41:41 +0200 Subject: [PATCH 08/14] Implement generation of benchmarks from previous doctest run statistics --- asv.conf.json | 12 +- src/sage/benchmark/__init__.py | 297 +++++++++++++++++++-------------- 2 files changed, 178 insertions(+), 131 deletions(-) diff --git a/asv.conf.json b/asv.conf.json index 71aef8c43e9..abf30504242 100644 --- a/asv.conf.json +++ b/asv.conf.json @@ -1,10 +1,16 @@ { "version": 1, - "project": "sage", + "project": "sagemath", "project_url": "https://sagemath.org", "repo": ".", - "plugins": ["sage_asv"], - "environment_type": "sage", + "environment_type": "conda", + "matrix": {}, + "conda_channels": ["conda-forge", "default"], + "build_command": [ + "echo 'Rebuilding sage for benchmarks is not supported yet. Use `asv run --environment existing --quick` in an environment with a built SageMath instead. Add --set-commit-hash `git rev-parse HEAD` to store timings in the database in .asv/. Note that o actually make this work, the entire installation workflow with conda would need to be added to asv.conf.json and the matrix there populated.'" + ], + "install_command": [], + "uninstall_command": [], "env_dir": ".asv/env", "results_dir": ".asv/results", "html_dir": ".asv/html", diff --git a/src/sage/benchmark/__init__.py b/src/sage/benchmark/__init__.py index 0bced064c06..393df768fd7 100644 --- a/src/sage/benchmark/__init__.py +++ b/src/sage/benchmark/__init__.py @@ -1,128 +1,169 @@ -from sage.all import * - -from sage.doctest.control import DocTestDefaults, DocTestController -from sage.doctest.forker import SageDocTestRunner, DocTestTask -from sage.doctest.parsing import parse_optional_tags - -import timeit -import doctest - -DEFAULTS = DocTestDefaults() -DEFAULTS.serial = True -DEFAULTS.long = True - -PREFIX = 'track__' - -def myglob(path, pattern): - # python 2 does not have support for ** in glob patterns - import fnmatch - import os - - matches = [] - for root, dirnames, filenames in os.walk(path): - for filename in fnmatch.filter(filenames, pattern): - matches.append(os.path.join(root, filename)) - return matches - -class BenchmarkMetaclass(type): - _dir = None - - def _run(cls, fname=[SAGE_SRC + '/sage/']): - old_runner = DocTestTask.runner - DocTestTask.runner = BenchmarkRunner - try: - DocTestController(DEFAULTS, fname).run() - finally: - DocTestTask.runner = old_runner - - def __dir__(cls): - if cls._dir is None: - cls._dir = set() - BenchmarkRunner._dir = cls._dir - cls._run() - return list(cls._dir) - - def __getattr__(cls, name): - if not name.startswith(PREFIX): - raise AttributeError - cls.create_timer(name) - return getattr(cls, name) - - def create_timer(cls, name): - try: - type.__getattr__(cls, name) - except AttributeError: - def time_doctest(self): - BenchmarkRunner._selected = name - BenchmarkRunner._time = 0 - cls._run(myglob(os.path.join(SAGE_SRC,'sage'), BenchmarkRunner.decode(name)+"*.*")) - return BenchmarkRunner._time - - time_doctest.__name__ = name - setattr(cls, name, time_doctest) - -class Benchmarks(object): - __metaclass__ = BenchmarkMetaclass - - def __getattr__(self, name): - if not name.startswith(PREFIX): - raise AttributeError - type(self).create_timer(name) - return getattr(self, name) - -class BenchmarkRunner(SageDocTestRunner): - _selected = None - _dir = set() - _time = None - - @classmethod - def encode(cls, prefix, filename, name, digest): - module = os.path.splitext(os.path.basename(filename))[0] - method = name.split('.')[-1] - return PREFIX+module+"__"+method+"__"+digest - - @classmethod - def decode(cls, name): - # This is not the full file name but only the bit up to the first _. - # But that's good enough as we later seach for this with a glob pattern - return name.split("__")[1] - - def run(self, test, clear_globs=True, *args, **kwargs): - self._do_not_run_tests = True - super(BenchmarkRunner, self).run(test, *args, clear_globs=False, **kwargs) - - name = BenchmarkRunner.encode(PREFIX, test.filename, test.name, self.running_doctest_digest.hexdigest()) - if name not in BenchmarkRunner._dir: - for example in test.examples: - if isinstance(example, doctest.Example): - if "long time" in parse_optional_tags(example.source): - BenchmarkRunner._dir.add(name) - break - - self._do_not_run_tests = False - if type(self)._selected is not None: - if name == type(self)._selected: - pass - else: - self._do_not_run_tests = True - if not self._do_not_run_tests: - super(BenchmarkRunner, self).run(test, *args, clear_globs=clear_globs, **kwargs) - return - - def compile_and_execute(self, example, compiler, globs, *args, **kwargs): - if self._do_not_run_tests: - compiler = lambda example: compile("print(%s.encode('utf8'))"%(repr(example.want),), "want.py", "single") - super(BenchmarkRunner, self).compile_and_execute(example, compiler, globs, *args, **kwargs) - else: - compiled = compiler(example) - # TODO: According to https://asv.readthedocs.io/en/latest/writing_benchmarks.html, time.process_time would be better here but it's not available in Python 2 - start = timeit.default_timer() - exec(compiled, globs) - end = timeit.default_timer() - type(self)._time += end-start - -class BenchmarkRunningRunner(SageDocTestRunner): - def run(self, test, *args, **kwargs): - print(test,BenchmarkRunningRunner._test) - if test == BenchmarkRunningRunner._test: - super(BenchmarkRunningRunner).run(test, *args, **kwargs) +r""" +Benchmarks for SageMath + +This module searches for doctest timings created with ``sage -t +--asv_stats_path=$SAGE_ASV_STATS`` if the ``$SAGE_ASV_STATS`` environment +variable is set. For each timing, it dynamically creates a method in a class in +this module to report that timing to airspeed velocity as configured in +``asv.conf.json``. + +EXAMPLES: + +Since this variable is usually not set, this module does nothing:: + + sage: import sage.benchmark + sage: dir(sage.benchmark) + ['__builtins__', + '__cached__', + '__doc__', + '__file__', + '__loader__', + '__name__', + '__package__', + '__path__', + '__spec__', + 'create_benchmark_class', + 'create_track_method', + 'create_trackers_from_doctest_stats', + 'json', + 'os'] + +""" +# **************************************************************************** +# Copyright (C) 2023 Julian Rueth +# +# Distributed under the terms of the GNU General Public License (GPL) +# as published by the Free Software Foundation; either version 2 of +# the License, or (at your option) any later version. +# +# https://www.gnu.org/licenses/ +# **************************************************************************** + +import json +import os + + +def create_trackers_from_doctest_stats(): + r""" + Create asv-compatible benchmark classes that contain a `track_…` method + for each doctest block reported in the log produced by ``sage -t + --asv_stats_path=$SAGE_ASV_STATS`` + + We export these wrappers in this module. These get picked up by asv which + will then run these as "benchmarks". Instead of running the doctests to + benchmark them, we are just reporting the timings that we collected in a + previous doctest run instead. + + EXAMPLES: + + When ``SAGE_ASV_STATS`` is not set, this creates an empty dict:: + + sage: from sage.benchmark import create_trackers_from_doctest_stats + sage: create_trackers_from_doctest_stats() + {} + + """ + stats_file = os.environ.get("SAGE_ASV_STATS", None) + if not stats_file: + return {} + + runs_by_module = json.load(open(stats_file)) + + return { + module: create_benchmark_class(module, runs) + for (module, runs) in runs_by_module.items() + } + + +def create_benchmark_class(name, runs): + r""" + Return an ASV compatible benchmark class called `name` that pretends to + perform the ``runs`` representing all the doctests of a module. + + EXAMPLES:: + + sage: from sage.benchmark import create_benchmark_class + sage: benchmark = create_benchmark_class("sage.rings.padics.padic_generic", [ + ....: ["", [], "pass", 1.337], + ....: ["pAdicGeneric.some_elements", ["long", "meataxe"], "pass\npass", 13.37] + ....: ]) + sage: dir(benchmark) + [... + 'track_[sage.rings.padics.padic_generic]', + 'track_pAdicGeneric.some_elements[long,meataxe]'] + + """ + class Benchmark: + pass + + # asv groups entries by top-level package name, so everything goes into "sage". + # To break this logic, we replace "." with a One Dot Leader character that + # looks similar. + Benchmark.__name__ = name.replace(".", "․") + + for identifier, flags, source, time in runs: + method = "track_" + (identifier or f"[{name}]") + if flags: + method += "[" + ",".join(flags) + "]" + + if hasattr(Benchmark, method): + raise NotImplementedError(f"cannot merge duplicate results for {identifier} with {flags}") + + setattr(Benchmark, method, create_track_method(name, identifier, flags, source, time)) + + return Benchmark + + +def create_track_method(module, identifier, flags, source, time): + r""" + Return a function that can be added as a method of a benchmark class. + + The method returns tracking results that will be picked up by asv as timing + benchmark results. + + EXAMPLES:: + + sage: from sage.benchmark import create_track_method + sage: method = create_track_method("sage.rings.padics.padic_generic", "", [], "pass", 1.337) + sage: method.pretty_name + '[sage.rings.padics.padic_generic]' + sage: method.pretty_source + 'pass' + sage: method(None) + 1337.00000000000 + + :: + + sage: method = create_track_method("sage.rings.padics.padic_generic", "pAdicGeneric.some_elements", ["long", "meataxe"], "pass\npass", 13.37) + sage: method.pretty_name + 'pAdicGeneric.some_elements' + sage: print(method.pretty_source) + # with long, meataxe + pass + pass + sage: method(None) + 13370.0000000000 + + """ + def method(self): + return time * 1e3 + + method.unit = "ms" + + # We do not need to run this "benchmark" more than once; we are just + # reporting the same static data every time. + method.repeat = 1 + method.number = 1 + method.min_run_count = 1 + + # The doctest of a module itself has no identifier, so we write the module name instead. + method.pretty_name = identifier or f"[{module}]" + + method.pretty_source=source + if flags: + method.pretty_source = f"# with {', '.join(flags)}\n" + method.pretty_source + + return method + + +locals().update(create_trackers_from_doctest_stats()) From 9fb7e1de434769385381334583f35ada21b54db7 Mon Sep 17 00:00:00 2001 From: David Roe Date: Fri, 10 Feb 2023 13:43:06 +0100 Subject: [PATCH 09/14] Try to fix the mysterious errors --- src/sage/doctest/forker.py | 14 ++++++++------ src/sage/doctest/parsing.py | 2 +- src/sage/doctest/reporting.py | 4 ++++ 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/sage/doctest/forker.py b/src/sage/doctest/forker.py index 157c4885cb6..4a646a5fc4d 100644 --- a/src/sage/doctest/forker.py +++ b/src/sage/doctest/forker.py @@ -51,6 +51,7 @@ from queue import Empty import gc import IPython.lib.pretty +import base64 import sage.misc.randstate as randstate from sage.misc.misc import walltime @@ -795,9 +796,9 @@ def compiler(example): if self.options.use_asv: if failures == 0: # We record the timing for this test for speed regression purposes - test.walltime = sum(example.walltime for example in test.examples) + test._sage_walltime = sum(example.walltime for example in test.examples) else: - test.walltime = -1 + test._sage_walltime = -1 # Record and return the number of failures and tries. self._DocTestRunner__record_outcome(test, failures, tries) @@ -1599,18 +1600,19 @@ def asv_label(test): for example in test.examples: # The added sig_on_count() test has no tags if hasattr(example, "optional_tags"): - opts = opts.union(example.optional_tags) + opts = opts.union(example._sage_optional_tags) olist = [] if "long" in opts: olist = ["long"] opts.remove("long") olist.extend(sorted(opts)) - return f"{name}|{','.join(olist)}|{code}" + label = "%s|%s|%s" % (name, ','.join(olist), code) + return base64.b64encode(label.encode('utf-8')).decode('ascii') D['asv_times'] = {} for test in doctests: - if test.walltime > 0: # -1 indicates a failure, 0 means nothing was actually run - D['asv_times'][asv_label(test)] = test.walltime + if test._sage_walltime > 0: # -1 indicates a failure, 0 means nothing was actually run + D['asv_times'][asv_label(test)] = test._sage_walltime if hasattr(self, 'failures'): D['failures'] = self.failures return self.failures diff --git a/src/sage/doctest/parsing.py b/src/sage/doctest/parsing.py index e4cc04f4b21..4af6338f97b 100644 --- a/src/sage/doctest/parsing.py +++ b/src/sage/doctest/parsing.py @@ -633,7 +633,7 @@ def parse(self, string, *args): for item in res: if isinstance(item, doctest.Example): optional_tags = parse_optional_tags(item.source) - item.optional_tags = optional_tags # for asv + item._sage_optional_tags = optional_tags # for asv if optional_tags: for tag in optional_tags: self.optionals[tag] += 1 diff --git a/src/sage/doctest/reporting.py b/src/sage/doctest/reporting.py index 40d55d802cc..700c01f0617 100644 --- a/src/sage/doctest/reporting.py +++ b/src/sage/doctest/reporting.py @@ -38,6 +38,7 @@ from sys import stdout from signal import (SIGABRT, SIGALRM, SIGBUS, SIGFPE, SIGHUP, SIGILL, SIGINT, SIGKILL, SIGPIPE, SIGQUIT, SIGSEGV, SIGTERM) +import base64 from sage.structure.sage_object import SageObject from sage.doctest.util import count_noun from sage.doctest.sources import DictAsObject @@ -505,6 +506,9 @@ def report(self, source, timeout, return_code, results, output, pid=None): if options.use_asv: asv = [] for label, time in result_dict.asv_times.items(): + # We encoded the label in weird ways + label = base64.b64decode(label.encode('ascii')).decode('utf-8') + #print(label) name, opts, code = label.split("|", 2) if name.startswith(basename): name = name[len(basename):] From 8ea099f84de469565cf8dcb94ba7a6756e75f941 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Fri, 10 Feb 2023 15:51:45 +0200 Subject: [PATCH 10/14] Revert "Try to fix the mysterious errors" This reverts commit 9fb7e1de434769385381334583f35ada21b54db7. --- src/sage/doctest/forker.py | 14 ++++++-------- src/sage/doctest/parsing.py | 2 +- src/sage/doctest/reporting.py | 4 ---- 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/src/sage/doctest/forker.py b/src/sage/doctest/forker.py index 4a646a5fc4d..157c4885cb6 100644 --- a/src/sage/doctest/forker.py +++ b/src/sage/doctest/forker.py @@ -51,7 +51,6 @@ from queue import Empty import gc import IPython.lib.pretty -import base64 import sage.misc.randstate as randstate from sage.misc.misc import walltime @@ -796,9 +795,9 @@ def compiler(example): if self.options.use_asv: if failures == 0: # We record the timing for this test for speed regression purposes - test._sage_walltime = sum(example.walltime for example in test.examples) + test.walltime = sum(example.walltime for example in test.examples) else: - test._sage_walltime = -1 + test.walltime = -1 # Record and return the number of failures and tries. self._DocTestRunner__record_outcome(test, failures, tries) @@ -1600,19 +1599,18 @@ def asv_label(test): for example in test.examples: # The added sig_on_count() test has no tags if hasattr(example, "optional_tags"): - opts = opts.union(example._sage_optional_tags) + opts = opts.union(example.optional_tags) olist = [] if "long" in opts: olist = ["long"] opts.remove("long") olist.extend(sorted(opts)) - label = "%s|%s|%s" % (name, ','.join(olist), code) - return base64.b64encode(label.encode('utf-8')).decode('ascii') + return f"{name}|{','.join(olist)}|{code}" D['asv_times'] = {} for test in doctests: - if test._sage_walltime > 0: # -1 indicates a failure, 0 means nothing was actually run - D['asv_times'][asv_label(test)] = test._sage_walltime + if test.walltime > 0: # -1 indicates a failure, 0 means nothing was actually run + D['asv_times'][asv_label(test)] = test.walltime if hasattr(self, 'failures'): D['failures'] = self.failures return self.failures diff --git a/src/sage/doctest/parsing.py b/src/sage/doctest/parsing.py index 4af6338f97b..e4cc04f4b21 100644 --- a/src/sage/doctest/parsing.py +++ b/src/sage/doctest/parsing.py @@ -633,7 +633,7 @@ def parse(self, string, *args): for item in res: if isinstance(item, doctest.Example): optional_tags = parse_optional_tags(item.source) - item._sage_optional_tags = optional_tags # for asv + item.optional_tags = optional_tags # for asv if optional_tags: for tag in optional_tags: self.optionals[tag] += 1 diff --git a/src/sage/doctest/reporting.py b/src/sage/doctest/reporting.py index 700c01f0617..40d55d802cc 100644 --- a/src/sage/doctest/reporting.py +++ b/src/sage/doctest/reporting.py @@ -38,7 +38,6 @@ from sys import stdout from signal import (SIGABRT, SIGALRM, SIGBUS, SIGFPE, SIGHUP, SIGILL, SIGINT, SIGKILL, SIGPIPE, SIGQUIT, SIGSEGV, SIGTERM) -import base64 from sage.structure.sage_object import SageObject from sage.doctest.util import count_noun from sage.doctest.sources import DictAsObject @@ -506,9 +505,6 @@ def report(self, source, timeout, return_code, results, output, pid=None): if options.use_asv: asv = [] for label, time in result_dict.asv_times.items(): - # We encoded the label in weird ways - label = base64.b64decode(label.encode('ascii')).decode('utf-8') - #print(label) name, opts, code = label.split("|", 2) if name.startswith(basename): name = name[len(basename):] From e58617b44f94178a02e3744f0451b7e35ea40518 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Fri, 10 Feb 2023 19:23:02 +0200 Subject: [PATCH 11/14] Read and write from tmpfile to not block queue it appears that you cannot send arbitrary amounts of data through a message queue in Python, see https://stackoverflow.com/questions/10028809/maximum-size-for-multiprocessing-queue-item. The queue will block pretty early. So we take load off the queue by using the local file system for some of the transfers. We still need to delete the files which is not done in this proof of concept yet. --- src/sage/doctest/forker.py | 9 ++++++++- src/sage/doctest/reporting.py | 23 ++++++++++++----------- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/src/sage/doctest/forker.py b/src/sage/doctest/forker.py index 157c4885cb6..ecbfc8ad6e4 100644 --- a/src/sage/doctest/forker.py +++ b/src/sage/doctest/forker.py @@ -1605,7 +1605,14 @@ def asv_label(test): olist = ["long"] opts.remove("long") olist.extend(sorted(opts)) - return f"{name}|{','.join(olist)}|{code}" + + import tempfile + tmp, fname = tempfile.mkstemp() + import os + tmp = os.fdopen(tmp, 'w') + tmp.write(f"{name}|{','.join(olist)}|{code}") + tmp.close() + return fname D['asv_times'] = {} for test in doctests: diff --git a/src/sage/doctest/reporting.py b/src/sage/doctest/reporting.py index 40d55d802cc..5f5a43036d1 100644 --- a/src/sage/doctest/reporting.py +++ b/src/sage/doctest/reporting.py @@ -504,17 +504,18 @@ def report(self, source, timeout, return_code, results, output, pid=None): stats[basename] = dict(walltime=wall) if options.use_asv: asv = [] - for label, time in result_dict.asv_times.items(): - name, opts, code = label.split("|", 2) - if name.startswith(basename): - name = name[len(basename):] - if name.startswith("."): - name = name[1:] - if opts: - opts = opts.split(",") - else: - opts = [] - asv.append([name, opts, code, time]) + for fname, time in result_dict.asv_times.items(): + with open(fname) as label: + name, opts, code = label.read().split("|", 2) + if name.startswith(basename): + name = name[len(basename):] + if name.startswith("."): + name = name[1:] + if opts: + opts = opts.split(",") + else: + opts = [] + asv.append([name, opts, code, time]) self.asv_stats[basename] = asv postscript['cputime'] += cpu postscript['walltime'] += wall From 6645eed975b45aff7a3578b1cc24e5c695c274bb Mon Sep 17 00:00:00 2001 From: David Roe Date: Fri, 10 Feb 2023 19:10:42 +0100 Subject: [PATCH 12/14] Try to improve name detection --- src/sage/doctest/sources.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/sage/doctest/sources.py b/src/sage/doctest/sources.py index c078e5508b3..573bb2a491e 100644 --- a/src/sage/doctest/sources.py +++ b/src/sage/doctest/sources.py @@ -35,7 +35,8 @@ # Python file parsing triple_quotes = re.compile(r"\s*[rRuU]*((''')|(\"\"\"))") -name_regex = re.compile(r".*\s(\w+)([(].*)?:") +paren_name_regex = re.compile(r".*\s(\w+)\s*[(].*:") +noparen_name_regex = re.compile(r".*\s(\w+)\s*:") # LaTeX file parsing begin_verb = re.compile(r"\s*\\begin{verbatim}") @@ -1100,7 +1101,10 @@ def starting_docstring(self, line): # quotematch wasn't None, but then we mishandle classes # that don't have a docstring. if not self.code_wrapping and self.last_indent >= 0 and indent > self.last_indent: - name = name_regex.match(self.last_line.replace("\n", "")) + if "(" in self.last_line: + name = paren_name_regex.match(self.last_line.replace("\n", "")) + else: + name = noparen_name_regex.match(self.last_line.replace("\n", "")) if name: name = name.groups()[0] self.qualified_name[indent] = name From 129a16899e0e47737a53684eb60184954dbc524f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Sat, 11 Feb 2023 11:44:26 +0200 Subject: [PATCH 13/14] Resolve slowness in benchmark consumption by using a module name that can be resolved by get_benchmark_from_name() in asv.benchmark --- src/sage/benchmark/__init__.py | 169 ------------------------------- src/sage/benchmark/docstring.py | 171 ++++++++++++++++++++++++++++++++ 2 files changed, 171 insertions(+), 169 deletions(-) create mode 100644 src/sage/benchmark/docstring.py diff --git a/src/sage/benchmark/__init__.py b/src/sage/benchmark/__init__.py index 393df768fd7..e69de29bb2d 100644 --- a/src/sage/benchmark/__init__.py +++ b/src/sage/benchmark/__init__.py @@ -1,169 +0,0 @@ -r""" -Benchmarks for SageMath - -This module searches for doctest timings created with ``sage -t ---asv_stats_path=$SAGE_ASV_STATS`` if the ``$SAGE_ASV_STATS`` environment -variable is set. For each timing, it dynamically creates a method in a class in -this module to report that timing to airspeed velocity as configured in -``asv.conf.json``. - -EXAMPLES: - -Since this variable is usually not set, this module does nothing:: - - sage: import sage.benchmark - sage: dir(sage.benchmark) - ['__builtins__', - '__cached__', - '__doc__', - '__file__', - '__loader__', - '__name__', - '__package__', - '__path__', - '__spec__', - 'create_benchmark_class', - 'create_track_method', - 'create_trackers_from_doctest_stats', - 'json', - 'os'] - -""" -# **************************************************************************** -# Copyright (C) 2023 Julian Rueth -# -# Distributed under the terms of the GNU General Public License (GPL) -# as published by the Free Software Foundation; either version 2 of -# the License, or (at your option) any later version. -# -# https://www.gnu.org/licenses/ -# **************************************************************************** - -import json -import os - - -def create_trackers_from_doctest_stats(): - r""" - Create asv-compatible benchmark classes that contain a `track_…` method - for each doctest block reported in the log produced by ``sage -t - --asv_stats_path=$SAGE_ASV_STATS`` - - We export these wrappers in this module. These get picked up by asv which - will then run these as "benchmarks". Instead of running the doctests to - benchmark them, we are just reporting the timings that we collected in a - previous doctest run instead. - - EXAMPLES: - - When ``SAGE_ASV_STATS`` is not set, this creates an empty dict:: - - sage: from sage.benchmark import create_trackers_from_doctest_stats - sage: create_trackers_from_doctest_stats() - {} - - """ - stats_file = os.environ.get("SAGE_ASV_STATS", None) - if not stats_file: - return {} - - runs_by_module = json.load(open(stats_file)) - - return { - module: create_benchmark_class(module, runs) - for (module, runs) in runs_by_module.items() - } - - -def create_benchmark_class(name, runs): - r""" - Return an ASV compatible benchmark class called `name` that pretends to - perform the ``runs`` representing all the doctests of a module. - - EXAMPLES:: - - sage: from sage.benchmark import create_benchmark_class - sage: benchmark = create_benchmark_class("sage.rings.padics.padic_generic", [ - ....: ["", [], "pass", 1.337], - ....: ["pAdicGeneric.some_elements", ["long", "meataxe"], "pass\npass", 13.37] - ....: ]) - sage: dir(benchmark) - [... - 'track_[sage.rings.padics.padic_generic]', - 'track_pAdicGeneric.some_elements[long,meataxe]'] - - """ - class Benchmark: - pass - - # asv groups entries by top-level package name, so everything goes into "sage". - # To break this logic, we replace "." with a One Dot Leader character that - # looks similar. - Benchmark.__name__ = name.replace(".", "․") - - for identifier, flags, source, time in runs: - method = "track_" + (identifier or f"[{name}]") - if flags: - method += "[" + ",".join(flags) + "]" - - if hasattr(Benchmark, method): - raise NotImplementedError(f"cannot merge duplicate results for {identifier} with {flags}") - - setattr(Benchmark, method, create_track_method(name, identifier, flags, source, time)) - - return Benchmark - - -def create_track_method(module, identifier, flags, source, time): - r""" - Return a function that can be added as a method of a benchmark class. - - The method returns tracking results that will be picked up by asv as timing - benchmark results. - - EXAMPLES:: - - sage: from sage.benchmark import create_track_method - sage: method = create_track_method("sage.rings.padics.padic_generic", "", [], "pass", 1.337) - sage: method.pretty_name - '[sage.rings.padics.padic_generic]' - sage: method.pretty_source - 'pass' - sage: method(None) - 1337.00000000000 - - :: - - sage: method = create_track_method("sage.rings.padics.padic_generic", "pAdicGeneric.some_elements", ["long", "meataxe"], "pass\npass", 13.37) - sage: method.pretty_name - 'pAdicGeneric.some_elements' - sage: print(method.pretty_source) - # with long, meataxe - pass - pass - sage: method(None) - 13370.0000000000 - - """ - def method(self): - return time * 1e3 - - method.unit = "ms" - - # We do not need to run this "benchmark" more than once; we are just - # reporting the same static data every time. - method.repeat = 1 - method.number = 1 - method.min_run_count = 1 - - # The doctest of a module itself has no identifier, so we write the module name instead. - method.pretty_name = identifier or f"[{module}]" - - method.pretty_source=source - if flags: - method.pretty_source = f"# with {', '.join(flags)}\n" + method.pretty_source - - return method - - -locals().update(create_trackers_from_doctest_stats()) diff --git a/src/sage/benchmark/docstring.py b/src/sage/benchmark/docstring.py new file mode 100644 index 00000000000..66f228b37f7 --- /dev/null +++ b/src/sage/benchmark/docstring.py @@ -0,0 +1,171 @@ +r""" +Benchmarks for SageMath + +This module searches for doctest timings created with ``sage -t +--asv_stats_path=$SAGE_ASV_STATS`` if the ``$SAGE_ASV_STATS`` environment +variable is set. For each timing, it dynamically creates a method in a class in +this module to report that timing to airspeed velocity as configured in +``asv.conf.json``. + +EXAMPLES: + +Since this variable is usually not set, this module does nothing:: + + sage: import sage.benchmark + sage: dir(sage.benchmark) + ['__builtins__', + '__cached__', + '__doc__', + '__file__', + '__loader__', + '__name__', + '__package__', + '__path__', + '__spec__', + 'create_benchmark_class', + 'create_track_method', + 'create_trackers_from_doctest_stats', + 'json', + 'os'] + +""" +# **************************************************************************** +# Copyright (C) 2023 Julian Rueth +# +# Distributed under the terms of the GNU General Public License (GPL) +# as published by the Free Software Foundation; either version 2 of +# the License, or (at your option) any later version. +# +# https://www.gnu.org/licenses/ +# **************************************************************************** + +import json +import os + + +def create_trackers_from_doctest_stats(): + r""" + Create asv-compatible benchmark classes that contain a `track_…` method + for each doctest block reported in the log produced by ``sage -t + --asv_stats_path=$SAGE_ASV_STATS`` + + We export these wrappers in this module. These get picked up by asv which + will then run these as "benchmarks". Instead of running the doctests to + benchmark them, we are just reporting the timings that we collected in a + previous doctest run instead. + + EXAMPLES: + + When ``SAGE_ASV_STATS`` is not set, this creates an empty dict:: + + sage: from sage.benchmark import create_trackers_from_doctest_stats + sage: create_trackers_from_doctest_stats() + {} + + """ + stats_file = os.environ.get("SAGE_ASV_STATS", None) + if not stats_file: + return {} + + runs_by_module = json.load(open(stats_file)) + + return { + module: create_benchmark_class(module, runs) + for (module, runs) in runs_by_module.items() + } + + +def create_benchmark_class(name, runs): + r""" + Return an ASV compatible benchmark class called `name` that pretends to + perform the ``runs`` representing all the doctests of a module. + + EXAMPLES:: + + sage: from sage.benchmark import create_benchmark_class + sage: benchmark = create_benchmark_class("sage.rings.padics.padic_generic", [ + ....: ["", [], "pass", 1.337], + ....: ["pAdicGeneric.some_elements", ["long", "meataxe"], "pass\npass", 13.37] + ....: ]) + sage: dir(benchmark) + [... + 'track_[sage.rings.padics.padic_generic]', + 'track_pAdicGeneric.some_elements[long,meataxe]'] + + """ + class Benchmark: + pass + + # asv groups entries by top-level package name, so everything goes into "sage". + # To break this logic, we replace "." with a One Dot Leader character that + # looks similar. + Benchmark.__name__ = name.replace(".", "․") + + for identifier, flags, source, time in runs: + method = "track_" + (identifier or f"[{name}]") + if flags: + method += "[" + ",".join(flags) + "]" + + if hasattr(Benchmark, method): + # raise NotImplementedError(f"cannot merge duplicate results for {identifier} with {flags}") + print("duplicate results") + continue + + setattr(Benchmark, method, create_track_method(name, identifier, flags, source, time)) + + return Benchmark + + +def create_track_method(module, identifier, flags, source, time): + r""" + Return a function that can be added as a method of a benchmark class. + + The method returns tracking results that will be picked up by asv as timing + benchmark results. + + EXAMPLES:: + + sage: from sage.benchmark import create_track_method + sage: method = create_track_method("sage.rings.padics.padic_generic", "", [], "pass", 1.337) + sage: method.pretty_name + '[sage.rings.padics.padic_generic]' + sage: method.pretty_source + 'pass' + sage: method(None) + 1337.00000000000 + + :: + + sage: method = create_track_method("sage.rings.padics.padic_generic", "pAdicGeneric.some_elements", ["long", "meataxe"], "pass\npass", 13.37) + sage: method.pretty_name + 'pAdicGeneric.some_elements' + sage: print(method.pretty_source) + # with long, meataxe + pass + pass + sage: method(None) + 13370.0000000000 + + """ + def method(self): + return time * 1e3 + + method.unit = "ms" + + # We do not need to run this "benchmark" more than once; we are just + # reporting the same static data every time. + method.repeat = 1 + method.number = 1 + method.min_run_count = 1 + + # The doctest of a module itself has no identifier, so we write the module name instead. + method.pretty_name = identifier or f"[{module}]" + + method.pretty_source=source + if flags: + method.pretty_source = f"# with {', '.join(flags)}\n" + method.pretty_source + + return method + + +locals().update(create_trackers_from_doctest_stats()) From ab5414cdcd80e2ded3a5f9b97b98ae77e1c2d71e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Sat, 11 Feb 2023 12:27:43 +0200 Subject: [PATCH 14/14] Fix slowness when consuming asv results by making sure that a module.class.method lookup works, i.e., by not having any . in the wrong places. --- src/sage/benchmark/docstring.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/sage/benchmark/docstring.py b/src/sage/benchmark/docstring.py index 66f228b37f7..5c873d9faa2 100644 --- a/src/sage/benchmark/docstring.py +++ b/src/sage/benchmark/docstring.py @@ -42,6 +42,10 @@ import json import os +# TODO: Explain here what this is for and put the replacement logic in a central space +# TODO: Can we load a plugin that undoes this replacement on the client? +DOT = "․" + def create_trackers_from_doctest_stats(): r""" @@ -70,7 +74,7 @@ def create_trackers_from_doctest_stats(): runs_by_module = json.load(open(stats_file)) return { - module: create_benchmark_class(module, runs) + module.replace(".", DOT): create_benchmark_class(module, runs) for (module, runs) in runs_by_module.items() } @@ -99,10 +103,10 @@ class Benchmark: # asv groups entries by top-level package name, so everything goes into "sage". # To break this logic, we replace "." with a One Dot Leader character that # looks similar. - Benchmark.__name__ = name.replace(".", "․") + Benchmark.__name__ = name.replace(".", DOT) for identifier, flags, source, time in runs: - method = "track_" + (identifier or f"[{name}]") + method = "track_" + (identifier or f"[{name}]").replace(".", DOT) if flags: method += "[" + ",".join(flags) + "]"