From 2359dde041bfe52674ce23afe7691beeb69e2994 Mon Sep 17 00:00:00 2001 From: joncrall Date: Sun, 3 Apr 2022 21:53:15 -0400 Subject: [PATCH 01/38] Fix error reporting bug --- CHANGELOG.md | 10 ++++++++-- xdoctest/__init__.py | 2 +- xdoctest/doctest_example.py | 4 +++- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f701caf..d480e276 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,12 +4,18 @@ We are currently working on porting this changelog to the specifications in [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## Version 1.0.0 - Unreleased +## Version 1.0.1 - Unreleased + +### Fixed + +* Corner case bug in error reporting + + +## Version 1.0.0 - Released 2022-03-25 ### Added * Support for Python 3.10 - ### Fixed * Warning in pytest8 * Spelling errors in documentation diff --git a/xdoctest/__init__.py b/xdoctest/__init__.py index 6eab6a31..c6bae586 100644 --- a/xdoctest/__init__.py +++ b/xdoctest/__init__.py @@ -314,7 +314,7 @@ def fib(n): mkinit xdoctest --nomods ''' -__version__ = '1.0.0' +__version__ = '1.0.1' # Expose only select submodules diff --git a/xdoctest/doctest_example.py b/xdoctest/doctest_example.py index 166cc742..33eefdc7 100644 --- a/xdoctest/doctest_example.py +++ b/xdoctest/doctest_example.py @@ -243,6 +243,8 @@ def __init__(self, docsrc, modpath=None, callname=None, num=0, self.failed_part = None self.warn_list = None + self._partfilename = None + self.logged_evals = OrderedDict() self.logged_stdout = OrderedDict() self._unmatched_stdout = [] @@ -1082,7 +1084,7 @@ def overwrite_lineno(linepart): # raise Exception('foo') # continue - if self._partfilename in line: + if self._partfilename is not None and self._partfilename in line: # Intercept the line corresponding to the doctest tbparts = line.split(',') tb_lineno = int(tbparts[-2].strip().split()[1]) From e75fa9987435df24406914f0455cd3ca717dc325 Mon Sep 17 00:00:00 2001 From: joncrall Date: Thu, 5 May 2022 23:48:06 -0400 Subject: [PATCH 02/38] wip --- xdoctest/directive.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/xdoctest/directive.py b/xdoctest/directive.py index d105b4c4..881f82b4 100644 --- a/xdoctest/directive.py +++ b/xdoctest/directive.py @@ -67,6 +67,13 @@ * Environment variables, via: ``env:==``, (e.g. ``# xdoctest +REQUIRES(env:MYENVIRON==1)``) +TODO +---- + +- [ ] Directive for Python version: e.g. xdoctest: +REQUIRES(Python>=3.7) +- [ ] Directive for module version: e.g. xdoctest: +REQUIRES(module:rich>=1.0) + + CommandLine: python -m xdoctest.directive __doc__ From 3c4aee79c5075a778bcdde36ed7a387eb35b724f Mon Sep 17 00:00:00 2001 From: joncrall Date: Thu, 9 Jun 2022 10:45:53 -0400 Subject: [PATCH 03/38] Make deprecations robust, add global state for debugging --- CHANGELOG.md | 6 ++- xdoctest/core.py | 61 ++++++++++++++++----------- xdoctest/directive.py | 16 ++++++- xdoctest/doctest_example.py | 31 +++++++------- xdoctest/global_state.py | 19 +++++++++ xdoctest/parser.py | 47 ++++++++++----------- xdoctest/runner.py | 6 +-- xdoctest/static_analysis.py | 11 +++-- xdoctest/utils/util_deprecation.py | 67 ++++++++++++++++++++++++++++++ xdoctest/utils/util_stream.py | 10 +++-- 10 files changed, 195 insertions(+), 79 deletions(-) create mode 100644 xdoctest/global_state.py create mode 100644 xdoctest/utils/util_deprecation.py diff --git a/CHANGELOG.md b/CHANGELOG.md index d480e276..6e54ba75 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,9 +7,13 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ## Version 1.0.1 - Unreleased ### Fixed - * Corner case bug in error reporting +### Changed +* Moved some globals into a new module called `global_state` and allowed + environs to enable debug print statements. +* Added `util_deprecation` module to robustly mark features as deprecated. + ## Version 1.0.0 - Released 2022-03-25 diff --git a/xdoctest/core.py b/xdoctest/core.py index 0525c448..d9e04321 100644 --- a/xdoctest/core.py +++ b/xdoctest/core.py @@ -39,9 +39,7 @@ from xdoctest import utils from xdoctest.docstr import docscrape_google from xdoctest.utils import util_import - - -DEBUG = '--debug' in sys.argv +from xdoctest import global_state DOCTEST_STYLES = [ @@ -147,7 +145,7 @@ def doctest_from_parts(parts, num, curr_offset): example._parts = parts return example - if DEBUG: + if global_state.DEBUG_CORE: print('Parsing docstring for callname={} in modpath={}'.format( callname, modpath)) @@ -292,7 +290,7 @@ def parse_auto_docstr_examples(docstr, *args, **kwargs): First try to parse google style, but if no tests are found use freeform style. """ - if DEBUG: + if global_state.DEBUG_CORE: print('Automatic style is trying google parsing') n_found = 0 @@ -306,7 +304,7 @@ def parse_auto_docstr_examples(docstr, *args, **kwargs): # no google style tests were found, parse in freeform if n_found == 0: - if DEBUG: + if global_state.DEBUG_CORE: print('Automatic style is trying freeform parsing') for example in parse_freeform_docstr_examples(docstr, *args, **kwargs): yield example @@ -363,7 +361,7 @@ def parse_docstr_examples(docstr, callname=None, modpath=None, lineno=1, 1 >>> examples = list(parse_docstr_examples(docstr, fpath='foo.txt')) """ - if DEBUG: + if global_state.DEBUG_CORE: print('Parsing docstring examples for ' 'callname={} in modpath={}'.format(callname, modpath)) if style == 'freeform': @@ -379,7 +377,7 @@ def parse_docstr_examples(docstr, callname=None, modpath=None, lineno=1, raise KeyError('Unknown style={}. Valid styles are {}'.format( style, DOCTEST_STYLES)) - if DEBUG: + if global_state.DEBUG_CORE: print('parser = {!r}'.format(parser)) n_parsed = 0 @@ -389,7 +387,7 @@ def parse_docstr_examples(docstr, callname=None, modpath=None, lineno=1, n_parsed += 1 yield example except Exception as ex: - if DEBUG: + if global_state.DEBUG_CORE: print('Caught an error when parsing') msg = ('Cannot scrape callname={} in modpath={} line={}.\n' 'Caused by: {}\n') @@ -420,7 +418,7 @@ def parse_docstr_examples(docstr, callname=None, modpath=None, lineno=1, pass else: raise - if DEBUG: + if global_state.DEBUG_CORE: print('Finished parsing {} examples'.format(n_parsed)) @@ -474,7 +472,7 @@ def package_calldefs(pkg_identifier, exclude=[], ignore_syntax_errors=True, >>> assert util_import.modpath_to_modname(modpath) == pkg_identifier >>> assert 'package_calldefs' in calldefs """ - if DEBUG: + if global_state.DEBUG_CORE: print('Find package calldefs: pkg_identifier = {!r}'.format(pkg_identifier)) if isinstance(pkg_identifier, types.ModuleType): @@ -531,14 +529,22 @@ def parse_calldefs(module_identifier, analysis='auto'): """ # backwards compatibility hacks if '--allow-xdoc-dynamic' in sys.argv: - warnings.warn( - '--allow-xdoc-dynamic is deprecated and will be removed in ' - 'the future use --analysis=auto instead', DeprecationWarning) + from xdoctest.utils import util_deprecation + util_deprecation.schedule_deprecation3( + modname='xdoctest', + name='--allow-xdoc-dynamic', type='CLI flag', + migration='use --analysis=auto instead', + deprecate='1.0.0', error='1.1.0', remove='1.2.0' + ) analysis = 'auto' if '--xdoc-force-dynamic' in sys.argv: - warnings.warn( - '--xdoc-force-dynamic is deprecated and will be removed in ' - 'the future use --analysis=dynamic instead', DeprecationWarning) + from xdoctest.utils import util_deprecation + util_deprecation.schedule_deprecation3( + modname='xdoctest', + name='--xdoc-force-dynamic', type='CLI flag', + migration='use --analysis=dynamic instead', + deprecate='1.0.0', error='1.1.0', remove='1.2.0' + ) analysis = 'dynamic' if isinstance(module_identifier, types.ModuleType): @@ -567,7 +573,7 @@ def parse_calldefs(module_identifier, analysis='auto'): else: raise KeyError(analysis) - if DEBUG: + if global_state.DEBUG_CORE: print('About to parse calldefs with do_dynamic={}'.format(do_dynamic)) calldefs = None @@ -587,6 +593,9 @@ def parse_calldefs(module_identifier, analysis='auto'): else: calldefs = static_analysis.parse_static_calldefs(fpath=module_identifier) + if global_state.DEBUG_CORE: + print('Found {} calldefs'.format(len(calldefs))) + return calldefs @@ -661,12 +670,16 @@ def parse_doctestables(module_identifier, exclude=[], style='auto', docstr = calldef.docstr if calldef.docstr is not None: lineno = calldef.doclineno - for example in parse_docstr_examples(docstr, callname=callname, - modpath=modpath, - lineno=lineno, - style=style, - parser_kw=parser_kw): - yield example + example_gen = parse_docstr_examples( + docstr, callname=callname, modpath=modpath, lineno=lineno, + style=style, parser_kw=parser_kw) + if global_state.DEBUG_CORE: # nocover + for example in example_gen: + print(' * Yield example={}'.format(example)) + yield example + else: + for example in example_gen: + yield example if __name__ == '__main__': diff --git a/xdoctest/directive.py b/xdoctest/directive.py index 881f82b4..152a1158 100644 --- a/xdoctest/directive.py +++ b/xdoctest/directive.py @@ -437,7 +437,13 @@ def __nice__(self): return '{}{}'.format(prefix, self.name) def _unpack_args(self, num): - warnings.warn('Deprecated and will be removed', DeprecationWarning) + from xdoctest.utils import util_deprecation + util_deprecation.schedule_deprecation3( + modname='xdoctest', + name='Directive._unpack_args', type='method', + migration='there is no need to use this', + deprecate='1.0.0', error='1.1.0', remove='1.2.0' + ) nargs = self.args if len(nargs) != 1: raise TypeError( @@ -446,7 +452,13 @@ def _unpack_args(self, num): return self.args def effect(self, argv=None, environ=None): - warnings.warn('Deprecated use effects', DeprecationWarning) + from xdoctest.utils import util_deprecation + util_deprecation.schedule_deprecation3( + modname='xdoctest', + name='Directive.effect', type='method', + migration='Use Directive.effects instead', + deprecate='1.0.0', error='1.1.0', remove='1.2.0' + ) effects = self.effects(argv=argv, environ=environ) if len(effects) > 1: raise Exception('Old method cannot handle multiple effects') diff --git a/xdoctest/doctest_example.py b/xdoctest/doctest_example.py index 33eefdc7..09095881 100644 --- a/xdoctest/doctest_example.py +++ b/xdoctest/doctest_example.py @@ -19,15 +19,10 @@ from xdoctest import checker from xdoctest import exceptions -# I believe the original reason for this hack was fixed in 3.9rc (The CI will -# tell us otherwise if this is incorrect) -# from distutils.version import LooseVersion -# EVAL_MIGHT_RETURN_COROUTINE = LooseVersion(sys.version.split(' ')[0]) >= LooseVersion('3.9.0') -# EVAL_MIGHT_RETURN_COROUTINE = False - __devnotes__ = """ TODO: - [ ] Rename DocTest to Doctest? + - [ ] I dont like having "example" as a suffix to this modname, can we rename? """ @@ -279,7 +274,8 @@ def is_disabled(self, pytest=False): """ Checks for comment directives on the first line of the doctest - A doctest is disabled if it starts with any of the following patterns + A doctest is force-disabled if it starts with any of the following + patterns * ``>>> # DISABLE_DOCTEST`` * ``>>> # SCRIPT`` @@ -289,6 +285,17 @@ def is_disabled(self, pytest=False): And if running in pytest, you can also use * ``>>> import pytest; pytest.skip()`` + + Note: + modern versions of xdoctest contain directives like + `# xdoctest: +SKIP`, which are a better way to do this. + + TODO: + Robustly deprecate these non-standard ways of disabling a doctest. + Generate a warning for several versions if they are used, and + indicate what the replacement strategy is. Then raise an error for + several more versions before finally removing this code. + """ disable_patterns = [ r'>>>\s*#\s*DISABLE', @@ -641,16 +648,6 @@ def run(self, verbose=None, on_error=None): if part.compile_mode == 'eval': # print('test_globals = {}'.format(sorted(test_globals.keys()))) got_eval = eval(code, test_globals) - # if EVAL_MIGHT_RETURN_COROUTINE: - # import types - # if isinstance(got_eval, types.CoroutineType): - # # In 3.9-rc (2020-mar-31) it looks like - # # eval sometimes returns coroutines. I - # # found no docs on this. Not sure if it - # # will be mainlined, but this seems to - # # fix it. - # import asyncio - # got_eval = asyncio.run(got_eval) else: exec(code, test_globals) diff --git a/xdoctest/global_state.py b/xdoctest/global_state.py new file mode 100644 index 00000000..d5d13704 --- /dev/null +++ b/xdoctest/global_state.py @@ -0,0 +1,19 @@ +""" +Global state initialized at import time. +Used for hidden arguments and developer features. +""" +import os +import sys + + +def _boolean_environ(key): + value = os.environ.get(key, '').lower() + TRUTHY_ENVIRONS = {'true', 'on', 'yes', '1'} + return value in TRUTHY_ENVIRONS + + +DEBUG = _boolean_environ('XDOCTEST_DEBUG') or '--debug' in sys.argv + +DEBUG_PARSER = DEBUG or _boolean_environ('XDOCTEST_DEBUG_PARSER') +DEBUG_CORE = DEBUG or _boolean_environ('XDOCTEST_DEBUG_CORE') +DEBUG_RUNNER = DEBUG or _boolean_environ('XDOCTEST_DEBUG_RUNNER') diff --git a/xdoctest/parser.py b/xdoctest/parser.py index 040c6efe..d0f336a4 100644 --- a/xdoctest/parser.py +++ b/xdoctest/parser.py @@ -45,10 +45,9 @@ from xdoctest import exceptions from xdoctest import doctest_part from xdoctest import static_analysis as static +from xdoctest import global_state -DEBUG = '--debug' in sys.argv - INDENT_RE = re.compile(r'^([ ]*)(?=\S)', re.MULTILINE) @@ -143,7 +142,7 @@ def parse(self, string, info=None): >>> assert len(doctest_parts) == 6 >>> len(doctest_parts) """ - if DEBUG > 1: + if global_state.DEBUG_PARSER > 1: print('\n===== PARSE ====') if sys.version_info.major == 2: # nocover string = utils.ensure_unicode(string) @@ -172,7 +171,7 @@ def parse(self, string, info=None): failpoint = '_group_labeled_lines' elif all_parts is None: failpoint = '_package_groups' - if DEBUG: + if global_state.DEBUG_PARSER: print('') print('!!! FAILED !!!') print('failpoint = {!r}'.format(failpoint)) @@ -199,12 +198,12 @@ def parse(self, string, info=None): raise exceptions.DoctestParseError( 'Failed to parse doctest in {}'.format(failpoint), string=string, info=info, orig_ex=orig_ex) - if DEBUG > 1: + if global_state.DEBUG_PARSER > 1: print('\n===== FINISHED PARSE ====') return all_parts def _package_groups(self, grouped_lines): - if DEBUG > 1: + if global_state.DEBUG_PARSER > 1: import ubelt as ub print('') print('grouped_lines = {}'.format(ub.repr2(grouped_lines, nl=2))) @@ -219,7 +218,7 @@ def _package_groups(self, grouped_lines): text_part = '\n'.join(chunk) yield text_part lineno += len(chunk) - if DEBUG > 1: + if global_state.DEBUG_PARSER > 1: print('') def _package_chunk(self, raw_source_lines, raw_want_lines, lineno=0): @@ -243,7 +242,7 @@ def _package_chunk(self, raw_source_lines, raw_want_lines, lineno=0): 'string' """ - if DEBUG > 1: + if global_state.DEBUG_PARSER > 1: print('') match = INDENT_RE.search(raw_source_lines[0]) line_indent = 0 if match is None else (match.end() - match.start()) @@ -256,11 +255,11 @@ def _package_chunk(self, raw_source_lines, raw_want_lines, lineno=0): exec_source_lines = [p[4:] for p in source_lines] - if DEBUG > 1: + if global_state.DEBUG_PARSER > 1: print(' * locate ps1 lines') # Find the line number of each standalone statement ps1_linenos, mode_hint = self._locate_ps1_linenos(source_lines) - if DEBUG > 1: + if global_state.DEBUG_PARSER > 1: print('mode_hint = {!r}'.format(mode_hint)) print(' * located ps1 lines') @@ -336,7 +335,7 @@ def slice_example(s1, s2, want_lines=None): assert mode_hint in {'eval', 'exec', 'single'} example.compile_mode = mode_hint - if DEBUG > 1: + if global_state.DEBUG_PARSER > 1: print('example.compile_mode = {!r}'.format(example.compile_mode)) print('') yield example @@ -351,7 +350,7 @@ def _group_labeled_lines(self, labeled_lines): lines. Executable parts are returned as a tuple of source lines and an optional "want" statement. """ - if DEBUG > 1: + if global_state.DEBUG_PARSER > 1: print('') # Now that lines have types, groups them. This could have done this # above, but functionality is split for readability. @@ -363,7 +362,7 @@ def _group_labeled_lines(self, labeled_lines): groups = [] current = [] state = None - if DEBUG > 4: + if global_state.DEBUG_PARSER > 4: print('labeled_lines = {!r}'.format(labeled_lines)) # Need to ensure that old-style continuations with want statements are @@ -380,7 +379,7 @@ def _group_labeled_lines(self, labeled_lines): if current: groups.append((state, current)) - if DEBUG > 4: + if global_state.DEBUG_PARSER > 4: print('groups = {!r}'.format(groups)) # need to merge consecutive dsrc groups without want statements @@ -429,7 +428,7 @@ def _group_labeled_lines(self, labeled_lines): if prev_source: grouped_lines.append((prev_source, '')) - if DEBUG > 1: # nocover + if global_state.DEBUG_PARSER > 1: # nocover print('') return grouped_lines @@ -727,7 +726,7 @@ def _label_docsrc_lines(self, string): for line_idx, line in line_iter: match = INDENT_RE.search(line) line_indent = 0 if match is None else (match.end() - match.start()) - if DEBUG: # nocover + if global_state.DEBUG_PARSER: # nocover print('Next line {}: {}'.format(line_idx, line)) print('state_indent = {!r}'.format(state_indent)) print('match = {!r}'.format(match)) @@ -796,10 +795,10 @@ def _label_docsrc_lines(self, string): if curr_state in {DSRC, DCNT}: # source parts may consume more than one line try: - if DEBUG: # nocover + if global_state.DEBUG_PARSER: # nocover print('completing source') for part, norm_line in _complete_source(line, state_indent, line_iter): - if DEBUG > 4: # nocover + if global_state.DEBUG_PARSER > 4: # nocover print('Append Completion Line:') print('part = {!r}'.format(part)) print('norm_line = {!r}'.format(norm_line)) @@ -811,7 +810,7 @@ def _label_docsrc_lines(self, string): except exceptions.IncompleteParseError: raise except SyntaxError: - if DEBUG: # nocover + if global_state.DEBUG_PARSER: # nocover print('