Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1""" discover and run doctests in modules and test files.""" 

2import bdb 

3import inspect 

4import platform 

5import sys 

6import traceback 

7import warnings 

8from contextlib import contextmanager 

9from typing import Dict 

10from typing import List 

11from typing import Optional 

12from typing import Sequence 

13from typing import Tuple 

14from typing import Union 

15 

16import py 

17 

18import pytest 

19from _pytest import outcomes 

20from _pytest._code.code import ExceptionInfo 

21from _pytest._code.code import ReprFileLocation 

22from _pytest._code.code import TerminalRepr 

23from _pytest.compat import safe_getattr 

24from _pytest.compat import TYPE_CHECKING 

25from _pytest.fixtures import FixtureRequest 

26from _pytest.outcomes import Skipped 

27from _pytest.python_api import approx 

28from _pytest.warning_types import PytestWarning 

29 

30if TYPE_CHECKING: 

31 import doctest 

32 from typing import Type 

33 

34DOCTEST_REPORT_CHOICE_NONE = "none" 

35DOCTEST_REPORT_CHOICE_CDIFF = "cdiff" 

36DOCTEST_REPORT_CHOICE_NDIFF = "ndiff" 

37DOCTEST_REPORT_CHOICE_UDIFF = "udiff" 

38DOCTEST_REPORT_CHOICE_ONLY_FIRST_FAILURE = "only_first_failure" 

39 

40DOCTEST_REPORT_CHOICES = ( 

41 DOCTEST_REPORT_CHOICE_NONE, 

42 DOCTEST_REPORT_CHOICE_CDIFF, 

43 DOCTEST_REPORT_CHOICE_NDIFF, 

44 DOCTEST_REPORT_CHOICE_UDIFF, 

45 DOCTEST_REPORT_CHOICE_ONLY_FIRST_FAILURE, 

46) 

47 

48# Lazy definition of runner class 

49RUNNER_CLASS = None 

50# Lazy definition of output checker class 

51CHECKER_CLASS = None # type: Optional[Type[doctest.OutputChecker]] 

52 

53 

54def pytest_addoption(parser): 

55 parser.addini( 

56 "doctest_optionflags", 

57 "option flags for doctests", 

58 type="args", 

59 default=["ELLIPSIS"], 

60 ) 

61 parser.addini( 

62 "doctest_encoding", "encoding used for doctest files", default="utf-8" 

63 ) 

64 group = parser.getgroup("collect") 

65 group.addoption( 

66 "--doctest-modules", 

67 action="store_true", 

68 default=False, 

69 help="run doctests in all .py modules", 

70 dest="doctestmodules", 

71 ) 

72 group.addoption( 

73 "--doctest-report", 

74 type=str.lower, 

75 default="udiff", 

76 help="choose another output format for diffs on doctest failure", 

77 choices=DOCTEST_REPORT_CHOICES, 

78 dest="doctestreport", 

79 ) 

80 group.addoption( 

81 "--doctest-glob", 

82 action="append", 

83 default=[], 

84 metavar="pat", 

85 help="doctests file matching pattern, default: test*.txt", 

86 dest="doctestglob", 

87 ) 

88 group.addoption( 

89 "--doctest-ignore-import-errors", 

90 action="store_true", 

91 default=False, 

92 help="ignore doctest ImportErrors", 

93 dest="doctest_ignore_import_errors", 

94 ) 

95 group.addoption( 

96 "--doctest-continue-on-failure", 

97 action="store_true", 

98 default=False, 

99 help="for a given doctest, continue to run after the first failure", 

100 dest="doctest_continue_on_failure", 

101 ) 

102 

103 

104def pytest_unconfigure(): 

105 global RUNNER_CLASS 

106 

107 RUNNER_CLASS = None 

108 

109 

110def pytest_collect_file(path, parent): 

111 config = parent.config 

112 if path.ext == ".py": 

113 if config.option.doctestmodules and not _is_setup_py(config, path, parent): 

114 return DoctestModule(path, parent) 

115 elif _is_doctest(config, path, parent): 

116 return DoctestTextfile(path, parent) 

117 

118 

119def _is_setup_py(config, path, parent): 

120 if path.basename != "setup.py": 

121 return False 

122 contents = path.read() 

123 return "setuptools" in contents or "distutils" in contents 

124 

125 

126def _is_doctest(config, path, parent): 

127 if path.ext in (".txt", ".rst") and parent.session.isinitpath(path): 

128 return True 

129 globs = config.getoption("doctestglob") or ["test*.txt"] 

130 for glob in globs: 

131 if path.check(fnmatch=glob): 

132 return True 

133 return False 

134 

135 

136class ReprFailDoctest(TerminalRepr): 

137 def __init__( 

138 self, reprlocation_lines: Sequence[Tuple[ReprFileLocation, Sequence[str]]] 

139 ): 

140 self.reprlocation_lines = reprlocation_lines 

141 

142 def toterminal(self, tw: py.io.TerminalWriter) -> None: 

143 for reprlocation, lines in self.reprlocation_lines: 

144 for line in lines: 

145 tw.line(line) 

146 reprlocation.toterminal(tw) 

147 

148 

149class MultipleDoctestFailures(Exception): 

150 def __init__(self, failures): 

151 super().__init__() 

152 self.failures = failures 

153 

154 

155def _init_runner_class() -> "Type[doctest.DocTestRunner]": 

156 import doctest 

157 

158 class PytestDoctestRunner(doctest.DebugRunner): 

159 """ 

160 Runner to collect failures. Note that the out variable in this case is 

161 a list instead of a stdout-like object 

162 """ 

163 

164 def __init__( 

165 self, checker=None, verbose=None, optionflags=0, continue_on_failure=True 

166 ): 

167 doctest.DebugRunner.__init__( 

168 self, checker=checker, verbose=verbose, optionflags=optionflags 

169 ) 

170 self.continue_on_failure = continue_on_failure 

171 

172 def report_failure(self, out, test, example, got): 

173 failure = doctest.DocTestFailure(test, example, got) 

174 if self.continue_on_failure: 

175 out.append(failure) 

176 else: 

177 raise failure 

178 

179 def report_unexpected_exception(self, out, test, example, exc_info): 

180 if isinstance(exc_info[1], Skipped): 

181 raise exc_info[1] 

182 if isinstance(exc_info[1], bdb.BdbQuit): 

183 outcomes.exit("Quitting debugger") 

184 failure = doctest.UnexpectedException(test, example, exc_info) 

185 if self.continue_on_failure: 

186 out.append(failure) 

187 else: 

188 raise failure 

189 

190 return PytestDoctestRunner 

191 

192 

193def _get_runner( 

194 checker: Optional["doctest.OutputChecker"] = None, 

195 verbose: Optional[bool] = None, 

196 optionflags: int = 0, 

197 continue_on_failure: bool = True, 

198) -> "doctest.DocTestRunner": 

199 # We need this in order to do a lazy import on doctest 

200 global RUNNER_CLASS 

201 if RUNNER_CLASS is None: 

202 RUNNER_CLASS = _init_runner_class() 

203 # Type ignored because the continue_on_failure argument is only defined on 

204 # PytestDoctestRunner, which is lazily defined so can't be used as a type. 

205 return RUNNER_CLASS( # type: ignore 

206 checker=checker, 

207 verbose=verbose, 

208 optionflags=optionflags, 

209 continue_on_failure=continue_on_failure, 

210 ) 

211 

212 

213class DoctestItem(pytest.Item): 

214 def __init__(self, name, parent, runner=None, dtest=None): 

215 super().__init__(name, parent) 

216 self.runner = runner 

217 self.dtest = dtest 

218 self.obj = None 

219 self.fixture_request = None 

220 

221 def setup(self): 

222 if self.dtest is not None: 

223 self.fixture_request = _setup_fixtures(self) 

224 globs = dict(getfixture=self.fixture_request.getfixturevalue) 

225 for name, value in self.fixture_request.getfixturevalue( 

226 "doctest_namespace" 

227 ).items(): 

228 globs[name] = value 

229 self.dtest.globs.update(globs) 

230 

231 def runtest(self): 

232 _check_all_skipped(self.dtest) 

233 self._disable_output_capturing_for_darwin() 

234 failures = [] # type: List[doctest.DocTestFailure] 

235 self.runner.run(self.dtest, out=failures) 

236 if failures: 

237 raise MultipleDoctestFailures(failures) 

238 

239 def _disable_output_capturing_for_darwin(self): 

240 """ 

241 Disable output capturing. Otherwise, stdout is lost to doctest (#985) 

242 """ 

243 if platform.system() != "Darwin": 

244 return 

245 capman = self.config.pluginmanager.getplugin("capturemanager") 

246 if capman: 

247 capman.suspend_global_capture(in_=True) 

248 out, err = capman.read_global_capture() 

249 sys.stdout.write(out) 

250 sys.stderr.write(err) 

251 

252 def repr_failure(self, excinfo): 

253 import doctest 

254 

255 failures = ( 

256 None 

257 ) # type: Optional[List[Union[doctest.DocTestFailure, doctest.UnexpectedException]]] 

258 if excinfo.errisinstance((doctest.DocTestFailure, doctest.UnexpectedException)): 

259 failures = [excinfo.value] 

260 elif excinfo.errisinstance(MultipleDoctestFailures): 

261 failures = excinfo.value.failures 

262 

263 if failures is not None: 

264 reprlocation_lines = [] 

265 for failure in failures: 

266 example = failure.example 

267 test = failure.test 

268 filename = test.filename 

269 if test.lineno is None: 

270 lineno = None 

271 else: 

272 lineno = test.lineno + example.lineno + 1 

273 message = type(failure).__name__ 

274 reprlocation = ReprFileLocation(filename, lineno, message) 

275 checker = _get_checker() 

276 report_choice = _get_report_choice( 

277 self.config.getoption("doctestreport") 

278 ) 

279 if lineno is not None: 

280 assert failure.test.docstring is not None 

281 lines = failure.test.docstring.splitlines(False) 

282 # add line numbers to the left of the error message 

283 assert test.lineno is not None 

284 lines = [ 

285 "%03d %s" % (i + test.lineno + 1, x) 

286 for (i, x) in enumerate(lines) 

287 ] 

288 # trim docstring error lines to 10 

289 lines = lines[max(example.lineno - 9, 0) : example.lineno + 1] 

290 else: 

291 lines = [ 

292 "EXAMPLE LOCATION UNKNOWN, not showing all tests of that example" 

293 ] 

294 indent = ">>>" 

295 for line in example.source.splitlines(): 

296 lines.append("??? {} {}".format(indent, line)) 

297 indent = "..." 

298 if isinstance(failure, doctest.DocTestFailure): 

299 lines += checker.output_difference( 

300 example, failure.got, report_choice 

301 ).split("\n") 

302 else: 

303 inner_excinfo = ExceptionInfo(failure.exc_info) 

304 lines += ["UNEXPECTED EXCEPTION: %s" % repr(inner_excinfo.value)] 

305 lines += traceback.format_exception(*failure.exc_info) 

306 reprlocation_lines.append((reprlocation, lines)) 

307 return ReprFailDoctest(reprlocation_lines) 

308 else: 

309 return super().repr_failure(excinfo) 

310 

311 def reportinfo(self) -> Tuple[str, int, str]: 

312 return self.fspath, self.dtest.lineno, "[doctest] %s" % self.name 

313 

314 

315def _get_flag_lookup() -> Dict[str, int]: 

316 import doctest 

317 

318 return dict( 

319 DONT_ACCEPT_TRUE_FOR_1=doctest.DONT_ACCEPT_TRUE_FOR_1, 

320 DONT_ACCEPT_BLANKLINE=doctest.DONT_ACCEPT_BLANKLINE, 

321 NORMALIZE_WHITESPACE=doctest.NORMALIZE_WHITESPACE, 

322 ELLIPSIS=doctest.ELLIPSIS, 

323 IGNORE_EXCEPTION_DETAIL=doctest.IGNORE_EXCEPTION_DETAIL, 

324 COMPARISON_FLAGS=doctest.COMPARISON_FLAGS, 

325 ALLOW_UNICODE=_get_allow_unicode_flag(), 

326 ALLOW_BYTES=_get_allow_bytes_flag(), 

327 NUMBER=_get_number_flag(), 

328 ) 

329 

330 

331def get_optionflags(parent): 

332 optionflags_str = parent.config.getini("doctest_optionflags") 

333 flag_lookup_table = _get_flag_lookup() 

334 flag_acc = 0 

335 for flag in optionflags_str: 

336 flag_acc |= flag_lookup_table[flag] 

337 return flag_acc 

338 

339 

340def _get_continue_on_failure(config): 

341 continue_on_failure = config.getvalue("doctest_continue_on_failure") 

342 if continue_on_failure: 

343 # We need to turn off this if we use pdb since we should stop at 

344 # the first failure 

345 if config.getvalue("usepdb"): 

346 continue_on_failure = False 

347 return continue_on_failure 

348 

349 

350class DoctestTextfile(pytest.Module): 

351 obj = None 

352 

353 def collect(self): 

354 import doctest 

355 

356 # inspired by doctest.testfile; ideally we would use it directly, 

357 # but it doesn't support passing a custom checker 

358 encoding = self.config.getini("doctest_encoding") 

359 text = self.fspath.read_text(encoding) 

360 filename = str(self.fspath) 

361 name = self.fspath.basename 

362 globs = {"__name__": "__main__"} 

363 

364 optionflags = get_optionflags(self) 

365 

366 runner = _get_runner( 

367 verbose=False, 

368 optionflags=optionflags, 

369 checker=_get_checker(), 

370 continue_on_failure=_get_continue_on_failure(self.config), 

371 ) 

372 

373 parser = doctest.DocTestParser() 

374 test = parser.get_doctest(text, globs, name, filename, 0) 

375 if test.examples: 

376 yield DoctestItem(test.name, self, runner, test) 

377 

378 

379def _check_all_skipped(test): 

380 """raises pytest.skip() if all examples in the given DocTest have the SKIP 

381 option set. 

382 """ 

383 import doctest 

384 

385 all_skipped = all(x.options.get(doctest.SKIP, False) for x in test.examples) 

386 if all_skipped: 

387 pytest.skip("all tests skipped by +SKIP option") 

388 

389 

390def _is_mocked(obj): 

391 """ 

392 returns if a object is possibly a mock object by checking the existence of a highly improbable attribute 

393 """ 

394 return ( 

395 safe_getattr(obj, "pytest_mock_example_attribute_that_shouldnt_exist", None) 

396 is not None 

397 ) 

398 

399 

400@contextmanager 

401def _patch_unwrap_mock_aware(): 

402 """ 

403 contextmanager which replaces ``inspect.unwrap`` with a version 

404 that's aware of mock objects and doesn't recurse on them 

405 """ 

406 real_unwrap = inspect.unwrap 

407 

408 def _mock_aware_unwrap(obj, stop=None): 

409 try: 

410 if stop is None or stop is _is_mocked: 

411 return real_unwrap(obj, stop=_is_mocked) 

412 return real_unwrap(obj, stop=lambda obj: _is_mocked(obj) or stop(obj)) 

413 except Exception as e: 

414 warnings.warn( 

415 "Got %r when unwrapping %r. This is usually caused " 

416 "by a violation of Python's object protocol; see e.g. " 

417 "https://github.com/pytest-dev/pytest/issues/5080" % (e, obj), 

418 PytestWarning, 

419 ) 

420 raise 

421 

422 inspect.unwrap = _mock_aware_unwrap 

423 try: 

424 yield 

425 finally: 

426 inspect.unwrap = real_unwrap 

427 

428 

429class DoctestModule(pytest.Module): 

430 def collect(self): 

431 import doctest 

432 

433 class MockAwareDocTestFinder(doctest.DocTestFinder): 

434 """ 

435 a hackish doctest finder that overrides stdlib internals to fix a stdlib bug 

436 

437 https://github.com/pytest-dev/pytest/issues/3456 

438 https://bugs.python.org/issue25532 

439 """ 

440 

441 def _find_lineno(self, obj, source_lines): 

442 """ 

443 Doctest code does not take into account `@property`, this is a hackish way to fix it. 

444 

445 https://bugs.python.org/issue17446 

446 """ 

447 if isinstance(obj, property): 

448 obj = getattr(obj, "fget", obj) 

449 return doctest.DocTestFinder._find_lineno(self, obj, source_lines) 

450 

451 def _find( 

452 self, tests, obj, name, module, source_lines, globs, seen 

453 ) -> None: 

454 if _is_mocked(obj): 

455 return 

456 with _patch_unwrap_mock_aware(): 

457 

458 # Type ignored because this is a private function. 

459 doctest.DocTestFinder._find( # type: ignore 

460 self, tests, obj, name, module, source_lines, globs, seen 

461 ) 

462 

463 if self.fspath.basename == "conftest.py": 

464 module = self.config.pluginmanager._importconftest(self.fspath) 

465 else: 

466 try: 

467 module = self.fspath.pyimport() 

468 except ImportError: 

469 if self.config.getvalue("doctest_ignore_import_errors"): 

470 pytest.skip("unable to import module %r" % self.fspath) 

471 else: 

472 raise 

473 # uses internal doctest module parsing mechanism 

474 finder = MockAwareDocTestFinder() 

475 optionflags = get_optionflags(self) 

476 runner = _get_runner( 

477 verbose=False, 

478 optionflags=optionflags, 

479 checker=_get_checker(), 

480 continue_on_failure=_get_continue_on_failure(self.config), 

481 ) 

482 

483 for test in finder.find(module, module.__name__): 

484 if test.examples: # skip empty doctests 

485 yield DoctestItem(test.name, self, runner, test) 

486 

487 

488def _setup_fixtures(doctest_item): 

489 """ 

490 Used by DoctestTextfile and DoctestItem to setup fixture information. 

491 """ 

492 

493 def func(): 

494 pass 

495 

496 doctest_item.funcargs = {} 

497 fm = doctest_item.session._fixturemanager 

498 doctest_item._fixtureinfo = fm.getfixtureinfo( 

499 node=doctest_item, func=func, cls=None, funcargs=False 

500 ) 

501 fixture_request = FixtureRequest(doctest_item) 

502 fixture_request._fillfixtures() 

503 return fixture_request 

504 

505 

506def _init_checker_class() -> "Type[doctest.OutputChecker]": 

507 import doctest 

508 import re 

509 

510 class LiteralsOutputChecker(doctest.OutputChecker): 

511 """ 

512 Based on doctest_nose_plugin.py from the nltk project 

513 (https://github.com/nltk/nltk) and on the "numtest" doctest extension 

514 by Sebastien Boisgerault (https://github.com/boisgera/numtest). 

515 """ 

516 

517 _unicode_literal_re = re.compile(r"(\W|^)[uU]([rR]?[\'\"])", re.UNICODE) 

518 _bytes_literal_re = re.compile(r"(\W|^)[bB]([rR]?[\'\"])", re.UNICODE) 

519 _number_re = re.compile( 

520 r""" 

521 (?P<number> 

522 (?P<mantissa> 

523 (?P<integer1> [+-]?\d*)\.(?P<fraction>\d+) 

524 | 

525 (?P<integer2> [+-]?\d+)\. 

526 ) 

527 (?: 

528 [Ee] 

529 (?P<exponent1> [+-]?\d+) 

530 )? 

531 | 

532 (?P<integer3> [+-]?\d+) 

533 (?: 

534 [Ee] 

535 (?P<exponent2> [+-]?\d+) 

536 ) 

537 ) 

538 """, 

539 re.VERBOSE, 

540 ) 

541 

542 def check_output(self, want, got, optionflags): 

543 if doctest.OutputChecker.check_output(self, want, got, optionflags): 

544 return True 

545 

546 allow_unicode = optionflags & _get_allow_unicode_flag() 

547 allow_bytes = optionflags & _get_allow_bytes_flag() 

548 allow_number = optionflags & _get_number_flag() 

549 

550 if not allow_unicode and not allow_bytes and not allow_number: 

551 return False 

552 

553 def remove_prefixes(regex, txt): 

554 return re.sub(regex, r"\1\2", txt) 

555 

556 if allow_unicode: 

557 want = remove_prefixes(self._unicode_literal_re, want) 

558 got = remove_prefixes(self._unicode_literal_re, got) 

559 

560 if allow_bytes: 

561 want = remove_prefixes(self._bytes_literal_re, want) 

562 got = remove_prefixes(self._bytes_literal_re, got) 

563 

564 if allow_number: 

565 got = self._remove_unwanted_precision(want, got) 

566 

567 return doctest.OutputChecker.check_output(self, want, got, optionflags) 

568 

569 def _remove_unwanted_precision(self, want, got): 

570 wants = list(self._number_re.finditer(want)) 

571 gots = list(self._number_re.finditer(got)) 

572 if len(wants) != len(gots): 

573 return got 

574 offset = 0 

575 for w, g in zip(wants, gots): 

576 fraction = w.group("fraction") 

577 exponent = w.group("exponent1") 

578 if exponent is None: 

579 exponent = w.group("exponent2") 

580 if fraction is None: 

581 precision = 0 

582 else: 

583 precision = len(fraction) 

584 if exponent is not None: 

585 precision -= int(exponent) 

586 if float(w.group()) == approx(float(g.group()), abs=10 ** -precision): 

587 # They're close enough. Replace the text we actually 

588 # got with the text we want, so that it will match when we 

589 # check the string literally. 

590 got = ( 

591 got[: g.start() + offset] + w.group() + got[g.end() + offset :] 

592 ) 

593 offset += w.end() - w.start() - (g.end() - g.start()) 

594 return got 

595 

596 return LiteralsOutputChecker 

597 

598 

599def _get_checker() -> "doctest.OutputChecker": 

600 """ 

601 Returns a doctest.OutputChecker subclass that supports some 

602 additional options: 

603 

604 * ALLOW_UNICODE and ALLOW_BYTES options to ignore u'' and b'' 

605 prefixes (respectively) in string literals. Useful when the same 

606 doctest should run in Python 2 and Python 3. 

607 

608 * NUMBER to ignore floating-point differences smaller than the 

609 precision of the literal number in the doctest. 

610 

611 An inner class is used to avoid importing "doctest" at the module 

612 level. 

613 """ 

614 global CHECKER_CLASS 

615 if CHECKER_CLASS is None: 

616 CHECKER_CLASS = _init_checker_class() 

617 return CHECKER_CLASS() 

618 

619 

620def _get_allow_unicode_flag() -> int: 

621 """ 

622 Registers and returns the ALLOW_UNICODE flag. 

623 """ 

624 import doctest 

625 

626 return doctest.register_optionflag("ALLOW_UNICODE") 

627 

628 

629def _get_allow_bytes_flag() -> int: 

630 """ 

631 Registers and returns the ALLOW_BYTES flag. 

632 """ 

633 import doctest 

634 

635 return doctest.register_optionflag("ALLOW_BYTES") 

636 

637 

638def _get_number_flag() -> int: 

639 """ 

640 Registers and returns the NUMBER flag. 

641 """ 

642 import doctest 

643 

644 return doctest.register_optionflag("NUMBER") 

645 

646 

647def _get_report_choice(key: str) -> int: 

648 """ 

649 This function returns the actual `doctest` module flag value, we want to do it as late as possible to avoid 

650 importing `doctest` and all its dependencies when parsing options, as it adds overhead and breaks tests. 

651 """ 

652 import doctest 

653 

654 return { 

655 DOCTEST_REPORT_CHOICE_UDIFF: doctest.REPORT_UDIFF, 

656 DOCTEST_REPORT_CHOICE_CDIFF: doctest.REPORT_CDIFF, 

657 DOCTEST_REPORT_CHOICE_NDIFF: doctest.REPORT_NDIFF, 

658 DOCTEST_REPORT_CHOICE_ONLY_FIRST_FAILURE: doctest.REPORT_ONLY_FIRST_FAILURE, 

659 DOCTEST_REPORT_CHOICE_NONE: 0, 

660 }[key] 

661 

662 

663@pytest.fixture(scope="session") 

664def doctest_namespace(): 

665 """ 

666 Fixture that returns a :py:class:`dict` that will be injected into the namespace of doctests. 

667 """ 

668 return dict()