Coverage for /usr/local/lib/python3.7/site-packages/_pytest/doctest.py : 3%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1""" discover and run doctests in modules and test files."""
2import bdb
3import inspect
4import platform
5import sys
6import traceback
7import warnings
8from contextlib import contextmanager
9from typing import Dict
10from typing import List
11from typing import Optional
12from typing import Sequence
13from typing import Tuple
14from typing import Union
16import py
18import pytest
19from _pytest import outcomes
20from _pytest._code.code import ExceptionInfo
21from _pytest._code.code import ReprFileLocation
22from _pytest._code.code import TerminalRepr
23from _pytest.compat import safe_getattr
24from _pytest.compat import TYPE_CHECKING
25from _pytest.fixtures import FixtureRequest
26from _pytest.outcomes import Skipped
27from _pytest.python_api import approx
28from _pytest.warning_types import PytestWarning
30if TYPE_CHECKING:
31 import doctest
32 from typing import Type
34DOCTEST_REPORT_CHOICE_NONE = "none"
35DOCTEST_REPORT_CHOICE_CDIFF = "cdiff"
36DOCTEST_REPORT_CHOICE_NDIFF = "ndiff"
37DOCTEST_REPORT_CHOICE_UDIFF = "udiff"
38DOCTEST_REPORT_CHOICE_ONLY_FIRST_FAILURE = "only_first_failure"
40DOCTEST_REPORT_CHOICES = (
41 DOCTEST_REPORT_CHOICE_NONE,
42 DOCTEST_REPORT_CHOICE_CDIFF,
43 DOCTEST_REPORT_CHOICE_NDIFF,
44 DOCTEST_REPORT_CHOICE_UDIFF,
45 DOCTEST_REPORT_CHOICE_ONLY_FIRST_FAILURE,
46)
48# Lazy definition of runner class
49RUNNER_CLASS = None
50# Lazy definition of output checker class
51CHECKER_CLASS = None # type: Optional[Type[doctest.OutputChecker]]
54def pytest_addoption(parser):
55 parser.addini(
56 "doctest_optionflags",
57 "option flags for doctests",
58 type="args",
59 default=["ELLIPSIS"],
60 )
61 parser.addini(
62 "doctest_encoding", "encoding used for doctest files", default="utf-8"
63 )
64 group = parser.getgroup("collect")
65 group.addoption(
66 "--doctest-modules",
67 action="store_true",
68 default=False,
69 help="run doctests in all .py modules",
70 dest="doctestmodules",
71 )
72 group.addoption(
73 "--doctest-report",
74 type=str.lower,
75 default="udiff",
76 help="choose another output format for diffs on doctest failure",
77 choices=DOCTEST_REPORT_CHOICES,
78 dest="doctestreport",
79 )
80 group.addoption(
81 "--doctest-glob",
82 action="append",
83 default=[],
84 metavar="pat",
85 help="doctests file matching pattern, default: test*.txt",
86 dest="doctestglob",
87 )
88 group.addoption(
89 "--doctest-ignore-import-errors",
90 action="store_true",
91 default=False,
92 help="ignore doctest ImportErrors",
93 dest="doctest_ignore_import_errors",
94 )
95 group.addoption(
96 "--doctest-continue-on-failure",
97 action="store_true",
98 default=False,
99 help="for a given doctest, continue to run after the first failure",
100 dest="doctest_continue_on_failure",
101 )
104def pytest_unconfigure():
105 global RUNNER_CLASS
107 RUNNER_CLASS = None
110def pytest_collect_file(path, parent):
111 config = parent.config
112 if path.ext == ".py":
113 if config.option.doctestmodules and not _is_setup_py(config, path, parent):
114 return DoctestModule(path, parent)
115 elif _is_doctest(config, path, parent):
116 return DoctestTextfile(path, parent)
119def _is_setup_py(config, path, parent):
120 if path.basename != "setup.py":
121 return False
122 contents = path.read()
123 return "setuptools" in contents or "distutils" in contents
126def _is_doctest(config, path, parent):
127 if path.ext in (".txt", ".rst") and parent.session.isinitpath(path):
128 return True
129 globs = config.getoption("doctestglob") or ["test*.txt"]
130 for glob in globs:
131 if path.check(fnmatch=glob):
132 return True
133 return False
136class ReprFailDoctest(TerminalRepr):
137 def __init__(
138 self, reprlocation_lines: Sequence[Tuple[ReprFileLocation, Sequence[str]]]
139 ):
140 self.reprlocation_lines = reprlocation_lines
142 def toterminal(self, tw: py.io.TerminalWriter) -> None:
143 for reprlocation, lines in self.reprlocation_lines:
144 for line in lines:
145 tw.line(line)
146 reprlocation.toterminal(tw)
149class MultipleDoctestFailures(Exception):
150 def __init__(self, failures):
151 super().__init__()
152 self.failures = failures
155def _init_runner_class() -> "Type[doctest.DocTestRunner]":
156 import doctest
158 class PytestDoctestRunner(doctest.DebugRunner):
159 """
160 Runner to collect failures. Note that the out variable in this case is
161 a list instead of a stdout-like object
162 """
164 def __init__(
165 self, checker=None, verbose=None, optionflags=0, continue_on_failure=True
166 ):
167 doctest.DebugRunner.__init__(
168 self, checker=checker, verbose=verbose, optionflags=optionflags
169 )
170 self.continue_on_failure = continue_on_failure
172 def report_failure(self, out, test, example, got):
173 failure = doctest.DocTestFailure(test, example, got)
174 if self.continue_on_failure:
175 out.append(failure)
176 else:
177 raise failure
179 def report_unexpected_exception(self, out, test, example, exc_info):
180 if isinstance(exc_info[1], Skipped):
181 raise exc_info[1]
182 if isinstance(exc_info[1], bdb.BdbQuit):
183 outcomes.exit("Quitting debugger")
184 failure = doctest.UnexpectedException(test, example, exc_info)
185 if self.continue_on_failure:
186 out.append(failure)
187 else:
188 raise failure
190 return PytestDoctestRunner
193def _get_runner(
194 checker: Optional["doctest.OutputChecker"] = None,
195 verbose: Optional[bool] = None,
196 optionflags: int = 0,
197 continue_on_failure: bool = True,
198) -> "doctest.DocTestRunner":
199 # We need this in order to do a lazy import on doctest
200 global RUNNER_CLASS
201 if RUNNER_CLASS is None:
202 RUNNER_CLASS = _init_runner_class()
203 # Type ignored because the continue_on_failure argument is only defined on
204 # PytestDoctestRunner, which is lazily defined so can't be used as a type.
205 return RUNNER_CLASS( # type: ignore
206 checker=checker,
207 verbose=verbose,
208 optionflags=optionflags,
209 continue_on_failure=continue_on_failure,
210 )
213class DoctestItem(pytest.Item):
214 def __init__(self, name, parent, runner=None, dtest=None):
215 super().__init__(name, parent)
216 self.runner = runner
217 self.dtest = dtest
218 self.obj = None
219 self.fixture_request = None
221 def setup(self):
222 if self.dtest is not None:
223 self.fixture_request = _setup_fixtures(self)
224 globs = dict(getfixture=self.fixture_request.getfixturevalue)
225 for name, value in self.fixture_request.getfixturevalue(
226 "doctest_namespace"
227 ).items():
228 globs[name] = value
229 self.dtest.globs.update(globs)
231 def runtest(self):
232 _check_all_skipped(self.dtest)
233 self._disable_output_capturing_for_darwin()
234 failures = [] # type: List[doctest.DocTestFailure]
235 self.runner.run(self.dtest, out=failures)
236 if failures:
237 raise MultipleDoctestFailures(failures)
239 def _disable_output_capturing_for_darwin(self):
240 """
241 Disable output capturing. Otherwise, stdout is lost to doctest (#985)
242 """
243 if platform.system() != "Darwin":
244 return
245 capman = self.config.pluginmanager.getplugin("capturemanager")
246 if capman:
247 capman.suspend_global_capture(in_=True)
248 out, err = capman.read_global_capture()
249 sys.stdout.write(out)
250 sys.stderr.write(err)
252 def repr_failure(self, excinfo):
253 import doctest
255 failures = (
256 None
257 ) # type: Optional[List[Union[doctest.DocTestFailure, doctest.UnexpectedException]]]
258 if excinfo.errisinstance((doctest.DocTestFailure, doctest.UnexpectedException)):
259 failures = [excinfo.value]
260 elif excinfo.errisinstance(MultipleDoctestFailures):
261 failures = excinfo.value.failures
263 if failures is not None:
264 reprlocation_lines = []
265 for failure in failures:
266 example = failure.example
267 test = failure.test
268 filename = test.filename
269 if test.lineno is None:
270 lineno = None
271 else:
272 lineno = test.lineno + example.lineno + 1
273 message = type(failure).__name__
274 reprlocation = ReprFileLocation(filename, lineno, message)
275 checker = _get_checker()
276 report_choice = _get_report_choice(
277 self.config.getoption("doctestreport")
278 )
279 if lineno is not None:
280 assert failure.test.docstring is not None
281 lines = failure.test.docstring.splitlines(False)
282 # add line numbers to the left of the error message
283 assert test.lineno is not None
284 lines = [
285 "%03d %s" % (i + test.lineno + 1, x)
286 for (i, x) in enumerate(lines)
287 ]
288 # trim docstring error lines to 10
289 lines = lines[max(example.lineno - 9, 0) : example.lineno + 1]
290 else:
291 lines = [
292 "EXAMPLE LOCATION UNKNOWN, not showing all tests of that example"
293 ]
294 indent = ">>>"
295 for line in example.source.splitlines():
296 lines.append("??? {} {}".format(indent, line))
297 indent = "..."
298 if isinstance(failure, doctest.DocTestFailure):
299 lines += checker.output_difference(
300 example, failure.got, report_choice
301 ).split("\n")
302 else:
303 inner_excinfo = ExceptionInfo(failure.exc_info)
304 lines += ["UNEXPECTED EXCEPTION: %s" % repr(inner_excinfo.value)]
305 lines += traceback.format_exception(*failure.exc_info)
306 reprlocation_lines.append((reprlocation, lines))
307 return ReprFailDoctest(reprlocation_lines)
308 else:
309 return super().repr_failure(excinfo)
311 def reportinfo(self) -> Tuple[str, int, str]:
312 return self.fspath, self.dtest.lineno, "[doctest] %s" % self.name
315def _get_flag_lookup() -> Dict[str, int]:
316 import doctest
318 return dict(
319 DONT_ACCEPT_TRUE_FOR_1=doctest.DONT_ACCEPT_TRUE_FOR_1,
320 DONT_ACCEPT_BLANKLINE=doctest.DONT_ACCEPT_BLANKLINE,
321 NORMALIZE_WHITESPACE=doctest.NORMALIZE_WHITESPACE,
322 ELLIPSIS=doctest.ELLIPSIS,
323 IGNORE_EXCEPTION_DETAIL=doctest.IGNORE_EXCEPTION_DETAIL,
324 COMPARISON_FLAGS=doctest.COMPARISON_FLAGS,
325 ALLOW_UNICODE=_get_allow_unicode_flag(),
326 ALLOW_BYTES=_get_allow_bytes_flag(),
327 NUMBER=_get_number_flag(),
328 )
331def get_optionflags(parent):
332 optionflags_str = parent.config.getini("doctest_optionflags")
333 flag_lookup_table = _get_flag_lookup()
334 flag_acc = 0
335 for flag in optionflags_str:
336 flag_acc |= flag_lookup_table[flag]
337 return flag_acc
340def _get_continue_on_failure(config):
341 continue_on_failure = config.getvalue("doctest_continue_on_failure")
342 if continue_on_failure:
343 # We need to turn off this if we use pdb since we should stop at
344 # the first failure
345 if config.getvalue("usepdb"):
346 continue_on_failure = False
347 return continue_on_failure
350class DoctestTextfile(pytest.Module):
351 obj = None
353 def collect(self):
354 import doctest
356 # inspired by doctest.testfile; ideally we would use it directly,
357 # but it doesn't support passing a custom checker
358 encoding = self.config.getini("doctest_encoding")
359 text = self.fspath.read_text(encoding)
360 filename = str(self.fspath)
361 name = self.fspath.basename
362 globs = {"__name__": "__main__"}
364 optionflags = get_optionflags(self)
366 runner = _get_runner(
367 verbose=False,
368 optionflags=optionflags,
369 checker=_get_checker(),
370 continue_on_failure=_get_continue_on_failure(self.config),
371 )
373 parser = doctest.DocTestParser()
374 test = parser.get_doctest(text, globs, name, filename, 0)
375 if test.examples:
376 yield DoctestItem(test.name, self, runner, test)
379def _check_all_skipped(test):
380 """raises pytest.skip() if all examples in the given DocTest have the SKIP
381 option set.
382 """
383 import doctest
385 all_skipped = all(x.options.get(doctest.SKIP, False) for x in test.examples)
386 if all_skipped:
387 pytest.skip("all tests skipped by +SKIP option")
390def _is_mocked(obj):
391 """
392 returns if a object is possibly a mock object by checking the existence of a highly improbable attribute
393 """
394 return (
395 safe_getattr(obj, "pytest_mock_example_attribute_that_shouldnt_exist", None)
396 is not None
397 )
400@contextmanager
401def _patch_unwrap_mock_aware():
402 """
403 contextmanager which replaces ``inspect.unwrap`` with a version
404 that's aware of mock objects and doesn't recurse on them
405 """
406 real_unwrap = inspect.unwrap
408 def _mock_aware_unwrap(obj, stop=None):
409 try:
410 if stop is None or stop is _is_mocked:
411 return real_unwrap(obj, stop=_is_mocked)
412 return real_unwrap(obj, stop=lambda obj: _is_mocked(obj) or stop(obj))
413 except Exception as e:
414 warnings.warn(
415 "Got %r when unwrapping %r. This is usually caused "
416 "by a violation of Python's object protocol; see e.g. "
417 "https://github.com/pytest-dev/pytest/issues/5080" % (e, obj),
418 PytestWarning,
419 )
420 raise
422 inspect.unwrap = _mock_aware_unwrap
423 try:
424 yield
425 finally:
426 inspect.unwrap = real_unwrap
429class DoctestModule(pytest.Module):
430 def collect(self):
431 import doctest
433 class MockAwareDocTestFinder(doctest.DocTestFinder):
434 """
435 a hackish doctest finder that overrides stdlib internals to fix a stdlib bug
437 https://github.com/pytest-dev/pytest/issues/3456
438 https://bugs.python.org/issue25532
439 """
441 def _find_lineno(self, obj, source_lines):
442 """
443 Doctest code does not take into account `@property`, this is a hackish way to fix it.
445 https://bugs.python.org/issue17446
446 """
447 if isinstance(obj, property):
448 obj = getattr(obj, "fget", obj)
449 return doctest.DocTestFinder._find_lineno(self, obj, source_lines)
451 def _find(
452 self, tests, obj, name, module, source_lines, globs, seen
453 ) -> None:
454 if _is_mocked(obj):
455 return
456 with _patch_unwrap_mock_aware():
458 # Type ignored because this is a private function.
459 doctest.DocTestFinder._find( # type: ignore
460 self, tests, obj, name, module, source_lines, globs, seen
461 )
463 if self.fspath.basename == "conftest.py":
464 module = self.config.pluginmanager._importconftest(self.fspath)
465 else:
466 try:
467 module = self.fspath.pyimport()
468 except ImportError:
469 if self.config.getvalue("doctest_ignore_import_errors"):
470 pytest.skip("unable to import module %r" % self.fspath)
471 else:
472 raise
473 # uses internal doctest module parsing mechanism
474 finder = MockAwareDocTestFinder()
475 optionflags = get_optionflags(self)
476 runner = _get_runner(
477 verbose=False,
478 optionflags=optionflags,
479 checker=_get_checker(),
480 continue_on_failure=_get_continue_on_failure(self.config),
481 )
483 for test in finder.find(module, module.__name__):
484 if test.examples: # skip empty doctests
485 yield DoctestItem(test.name, self, runner, test)
488def _setup_fixtures(doctest_item):
489 """
490 Used by DoctestTextfile and DoctestItem to setup fixture information.
491 """
493 def func():
494 pass
496 doctest_item.funcargs = {}
497 fm = doctest_item.session._fixturemanager
498 doctest_item._fixtureinfo = fm.getfixtureinfo(
499 node=doctest_item, func=func, cls=None, funcargs=False
500 )
501 fixture_request = FixtureRequest(doctest_item)
502 fixture_request._fillfixtures()
503 return fixture_request
506def _init_checker_class() -> "Type[doctest.OutputChecker]":
507 import doctest
508 import re
510 class LiteralsOutputChecker(doctest.OutputChecker):
511 """
512 Based on doctest_nose_plugin.py from the nltk project
513 (https://github.com/nltk/nltk) and on the "numtest" doctest extension
514 by Sebastien Boisgerault (https://github.com/boisgera/numtest).
515 """
517 _unicode_literal_re = re.compile(r"(\W|^)[uU]([rR]?[\'\"])", re.UNICODE)
518 _bytes_literal_re = re.compile(r"(\W|^)[bB]([rR]?[\'\"])", re.UNICODE)
519 _number_re = re.compile(
520 r"""
521 (?P<number>
522 (?P<mantissa>
523 (?P<integer1> [+-]?\d*)\.(?P<fraction>\d+)
524 |
525 (?P<integer2> [+-]?\d+)\.
526 )
527 (?:
528 [Ee]
529 (?P<exponent1> [+-]?\d+)
530 )?
531 |
532 (?P<integer3> [+-]?\d+)
533 (?:
534 [Ee]
535 (?P<exponent2> [+-]?\d+)
536 )
537 )
538 """,
539 re.VERBOSE,
540 )
542 def check_output(self, want, got, optionflags):
543 if doctest.OutputChecker.check_output(self, want, got, optionflags):
544 return True
546 allow_unicode = optionflags & _get_allow_unicode_flag()
547 allow_bytes = optionflags & _get_allow_bytes_flag()
548 allow_number = optionflags & _get_number_flag()
550 if not allow_unicode and not allow_bytes and not allow_number:
551 return False
553 def remove_prefixes(regex, txt):
554 return re.sub(regex, r"\1\2", txt)
556 if allow_unicode:
557 want = remove_prefixes(self._unicode_literal_re, want)
558 got = remove_prefixes(self._unicode_literal_re, got)
560 if allow_bytes:
561 want = remove_prefixes(self._bytes_literal_re, want)
562 got = remove_prefixes(self._bytes_literal_re, got)
564 if allow_number:
565 got = self._remove_unwanted_precision(want, got)
567 return doctest.OutputChecker.check_output(self, want, got, optionflags)
569 def _remove_unwanted_precision(self, want, got):
570 wants = list(self._number_re.finditer(want))
571 gots = list(self._number_re.finditer(got))
572 if len(wants) != len(gots):
573 return got
574 offset = 0
575 for w, g in zip(wants, gots):
576 fraction = w.group("fraction")
577 exponent = w.group("exponent1")
578 if exponent is None:
579 exponent = w.group("exponent2")
580 if fraction is None:
581 precision = 0
582 else:
583 precision = len(fraction)
584 if exponent is not None:
585 precision -= int(exponent)
586 if float(w.group()) == approx(float(g.group()), abs=10 ** -precision):
587 # They're close enough. Replace the text we actually
588 # got with the text we want, so that it will match when we
589 # check the string literally.
590 got = (
591 got[: g.start() + offset] + w.group() + got[g.end() + offset :]
592 )
593 offset += w.end() - w.start() - (g.end() - g.start())
594 return got
596 return LiteralsOutputChecker
599def _get_checker() -> "doctest.OutputChecker":
600 """
601 Returns a doctest.OutputChecker subclass that supports some
602 additional options:
604 * ALLOW_UNICODE and ALLOW_BYTES options to ignore u'' and b''
605 prefixes (respectively) in string literals. Useful when the same
606 doctest should run in Python 2 and Python 3.
608 * NUMBER to ignore floating-point differences smaller than the
609 precision of the literal number in the doctest.
611 An inner class is used to avoid importing "doctest" at the module
612 level.
613 """
614 global CHECKER_CLASS
615 if CHECKER_CLASS is None:
616 CHECKER_CLASS = _init_checker_class()
617 return CHECKER_CLASS()
620def _get_allow_unicode_flag() -> int:
621 """
622 Registers and returns the ALLOW_UNICODE flag.
623 """
624 import doctest
626 return doctest.register_optionflag("ALLOW_UNICODE")
629def _get_allow_bytes_flag() -> int:
630 """
631 Registers and returns the ALLOW_BYTES flag.
632 """
633 import doctest
635 return doctest.register_optionflag("ALLOW_BYTES")
638def _get_number_flag() -> int:
639 """
640 Registers and returns the NUMBER flag.
641 """
642 import doctest
644 return doctest.register_optionflag("NUMBER")
647def _get_report_choice(key: str) -> int:
648 """
649 This function returns the actual `doctest` module flag value, we want to do it as late as possible to avoid
650 importing `doctest` and all its dependencies when parsing options, as it adds overhead and breaks tests.
651 """
652 import doctest
654 return {
655 DOCTEST_REPORT_CHOICE_UDIFF: doctest.REPORT_UDIFF,
656 DOCTEST_REPORT_CHOICE_CDIFF: doctest.REPORT_CDIFF,
657 DOCTEST_REPORT_CHOICE_NDIFF: doctest.REPORT_NDIFF,
658 DOCTEST_REPORT_CHOICE_ONLY_FIRST_FAILURE: doctest.REPORT_ONLY_FIRST_FAILURE,
659 DOCTEST_REPORT_CHOICE_NONE: 0,
660 }[key]
663@pytest.fixture(scope="session")
664def doctest_namespace():
665 """
666 Fixture that returns a :py:class:`dict` that will be injected into the namespace of doctests.
667 """
668 return dict()