Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of Hypothesis, which may be found at 

2# https://github.com/HypothesisWorks/hypothesis/ 

3# 

4# Most of this work is copyright (C) 2013-2020 David R. MacIver 

5# (david@drmaciver.com), but it contains contributions by others. See 

6# CONTRIBUTING.rst for a full list of people who may hold copyright, and 

7# consult the git log if you need to determine who owns an individual 

8# contribution. 

9# 

10# This Source Code Form is subject to the terms of the Mozilla Public License, 

11# v. 2.0. If a copy of the MPL was not distributed with this file, You can 

12# obtain one at https://mozilla.org/MPL/2.0/. 

13# 

14# END HEADER 

15 

16"""A module controlling settings for Hypothesis to use in falsification. 

17 

18Either an explicit settings object can be used or the default object on 

19this module can be modified. 

20""" 

21 

22import contextlib 

23import datetime 

24import inspect 

25import threading 

26import warnings 

27from enum import Enum, IntEnum, unique 

28from typing import Any, Dict, List 

29 

30import attr 

31 

32from hypothesis.errors import ( 

33 HypothesisDeprecationWarning, 

34 InvalidArgument, 

35 InvalidState, 

36) 

37from hypothesis.internal.reflection import get_pretty_function_description 

38from hypothesis.internal.validation import check_type, try_convert 

39from hypothesis.utils.conventions import not_set 

40from hypothesis.utils.dynamicvariables import DynamicVariable 

41 

42__all__ = ["settings"] 

43 

44all_settings = {} # type: Dict[str, Setting] 

45 

46 

47class settingsProperty: 

48 def __init__(self, name, show_default): 

49 self.name = name 

50 self.show_default = show_default 

51 

52 def __get__(self, obj, type=None): 

53 if obj is None: 

54 return self 

55 else: 

56 try: 

57 result = obj.__dict__[self.name] 

58 # This is a gross hack, but it preserves the old behaviour that 

59 # you can change the storage directory and it will be reflected 

60 # in the default database. 

61 if self.name == "database" and result is not_set: 

62 from hypothesis.database import ExampleDatabase 

63 

64 result = ExampleDatabase(not_set) 

65 return result 

66 except KeyError: 

67 raise AttributeError(self.name) 

68 

69 def __set__(self, obj, value): 

70 obj.__dict__[self.name] = value 

71 

72 def __delete__(self, obj): 

73 raise AttributeError("Cannot delete attribute %s" % (self.name,)) 

74 

75 @property 

76 def __doc__(self): 

77 description = all_settings[self.name].description 

78 default = ( 

79 repr(getattr(settings.default, self.name)) 

80 if self.show_default 

81 else "(dynamically calculated)" 

82 ) 

83 return "%s\n\ndefault value: ``%s``" % (description, default) 

84 

85 

86default_variable = DynamicVariable(None) 

87 

88 

89class settingsMeta(type): 

90 def __init__(self, *args, **kwargs): 

91 super().__init__(*args, **kwargs) 

92 

93 @property 

94 def default(self): 

95 v = default_variable.value 

96 if v is not None: 

97 return v 

98 if hasattr(settings, "_current_profile"): 

99 settings.load_profile(settings._current_profile) 

100 assert default_variable.value is not None 

101 return default_variable.value 

102 

103 def _assign_default_internal(self, value): 

104 default_variable.value = value 

105 

106 def __setattr__(self, name, value): 

107 if name == "default": 

108 raise AttributeError( 

109 "Cannot assign to the property settings.default - " 

110 "consider using settings.load_profile instead." 

111 ) 

112 elif not (isinstance(value, settingsProperty) or name.startswith("_")): 

113 raise AttributeError( 

114 "Cannot assign hypothesis.settings.%s=%r - the settings " 

115 "class is immutable. You can change the global default " 

116 "settings with settings.load_profile, or use @settings(...) " 

117 "to decorate your test instead." % (name, value) 

118 ) 

119 return type.__setattr__(self, name, value) 

120 

121 

122class settings(metaclass=settingsMeta): 

123 """A settings object controls a variety of parameters that are used in 

124 falsification. These may control both the falsification strategy and the 

125 details of the data that is generated. 

126 

127 Default values are picked up from the settings.default object and 

128 changes made there will be picked up in newly created settings. 

129 """ 

130 

131 _WHITELISTED_REAL_PROPERTIES = ["_construction_complete", "storage"] 

132 __definitions_are_locked = False 

133 _profiles = {} # type: dict 

134 __module__ = "hypothesis" 

135 

136 def __getattr__(self, name): 

137 if name in all_settings: 

138 return all_settings[name].default 

139 else: 

140 raise AttributeError("settings has no attribute %s" % (name,)) 

141 

142 def __init__(self, parent: "settings" = None, **kwargs: Any) -> None: 

143 if parent is not None and not isinstance(parent, settings): 

144 raise InvalidArgument( 

145 "Invalid argument: parent=%r is not a settings instance" % (parent,) 

146 ) 

147 if kwargs.get("derandomize"): 

148 if kwargs.get("database") is not None: 

149 raise InvalidArgument( 

150 "derandomize=True implies database=None, so passing " 

151 "database=%r too is invalid." % (kwargs["database"],) 

152 ) 

153 kwargs["database"] = None 

154 self._construction_complete = False 

155 defaults = parent or settings.default 

156 if defaults is not None: 

157 for setting in all_settings.values(): 

158 if kwargs.get(setting.name, not_set) is not_set: 

159 kwargs[setting.name] = getattr(defaults, setting.name) 

160 else: 

161 if setting.validator: 

162 kwargs[setting.name] = setting.validator(kwargs[setting.name]) 

163 for name, value in kwargs.items(): 

164 if name not in all_settings: 

165 raise InvalidArgument( 

166 "Invalid argument: %r is not a valid setting" % (name,) 

167 ) 

168 setattr(self, name, value) 

169 self.storage = threading.local() 

170 self._construction_complete = True 

171 

172 def __call__(self, test): 

173 """Make the settings object (self) an attribute of the test. 

174 

175 The settings are later discovered by looking them up on the test itself. 

176 """ 

177 if not callable(test): 

178 raise InvalidArgument( 

179 "settings objects can be called as a decorator with @given, " 

180 "but decorated test=%r is not callable." % (test,) 

181 ) 

182 if inspect.isclass(test): 

183 from hypothesis.stateful import RuleBasedStateMachine 

184 

185 if issubclass(test, RuleBasedStateMachine): 

186 attr_name = "_hypothesis_internal_settings_applied" 

187 if getattr(test, attr_name, False): 

188 raise InvalidArgument( 

189 "Applying the @settings decorator twice would " 

190 "overwrite the first version; merge their arguments " 

191 "instead." 

192 ) 

193 setattr(test, attr_name, True) 

194 test.TestCase.settings = self 

195 return test 

196 else: 

197 raise InvalidArgument( 

198 "@settings(...) can only be used as a decorator on " 

199 "functions, or on subclasses of RuleBasedStateMachine." 

200 ) 

201 if hasattr(test, "_hypothesis_internal_settings_applied"): 

202 # Can't use _hypothesis_internal_use_settings as an indicator that 

203 # @settings was applied, because @given also assigns that attribute. 

204 raise InvalidArgument( 

205 "%s has already been decorated with a settings object." 

206 "\n Previous: %r\n This: %r" 

207 % ( 

208 get_pretty_function_description(test), 

209 test._hypothesis_internal_use_settings, 

210 self, 

211 ) 

212 ) 

213 

214 test._hypothesis_internal_use_settings = self 

215 test._hypothesis_internal_settings_applied = True 

216 return test 

217 

218 @classmethod 

219 def _define_setting( 

220 cls, 

221 name, 

222 description, 

223 default, 

224 options=None, 

225 validator=None, 

226 show_default=True, 

227 ): 

228 """Add a new setting. 

229 

230 - name is the name of the property that will be used to access the 

231 setting. This must be a valid python identifier. 

232 - description will appear in the property's docstring 

233 - default is the default value. This may be a zero argument 

234 function in which case it is evaluated and its result is stored 

235 the first time it is accessed on any given settings object. 

236 """ 

237 if settings.__definitions_are_locked: 

238 raise InvalidState( 

239 "settings have been locked and may no longer be defined." 

240 ) 

241 if options is not None: 

242 options = tuple(options) 

243 assert default in options 

244 else: 

245 assert validator is not None 

246 

247 all_settings[name] = Setting( 

248 name=name, 

249 description=description.strip(), 

250 default=default, 

251 options=options, 

252 validator=validator, 

253 ) 

254 setattr(settings, name, settingsProperty(name, show_default)) 

255 

256 @classmethod 

257 def lock_further_definitions(cls): 

258 settings.__definitions_are_locked = True 

259 

260 def __setattr__(self, name, value): 

261 if name in settings._WHITELISTED_REAL_PROPERTIES: 

262 return object.__setattr__(self, name, value) 

263 elif name in all_settings: 

264 if self._construction_complete: 

265 raise AttributeError( 

266 "settings objects are immutable and may not be assigned to" 

267 " after construction." 

268 ) 

269 else: 

270 setting = all_settings[name] 

271 if setting.options is not None and value not in setting.options: 

272 raise InvalidArgument( 

273 "Invalid %s, %r. Valid options: %r" 

274 % (name, value, setting.options) 

275 ) 

276 return object.__setattr__(self, name, value) 

277 else: 

278 raise AttributeError("No such setting %s" % (name,)) 

279 

280 def __repr__(self): 

281 bits = ("%s=%r" % (name, getattr(self, name)) for name in all_settings) 

282 return "settings(%s)" % ", ".join(sorted(bits)) 

283 

284 def show_changed(self): 

285 bits = [] 

286 for name, setting in all_settings.items(): 

287 value = getattr(self, name) 

288 if value != setting.default: 

289 bits.append("%s=%r" % (name, value)) 

290 return ", ".join(sorted(bits, key=len)) 

291 

292 @staticmethod 

293 def register_profile(name: str, parent: "settings" = None, **kwargs: Any) -> None: 

294 """Registers a collection of values to be used as a settings profile. 

295 

296 Settings profiles can be loaded by name - for example, you might 

297 create a 'fast' profile which runs fewer examples, keep the 'default' 

298 profile, and create a 'ci' profile that increases the number of 

299 examples and uses a different database to store failures. 

300 

301 The arguments to this method are exactly as for 

302 :class:`~hypothesis.settings`: optional ``parent`` settings, and 

303 keyword arguments for each setting that will be set differently to 

304 parent (or settings.default, if parent is None). 

305 """ 

306 check_type(str, name, "name") 

307 settings._profiles[name] = settings(parent=parent, **kwargs) 

308 

309 @staticmethod 

310 def get_profile(name: str) -> "settings": 

311 """Return the profile with the given name.""" 

312 check_type(str, name, "name") 

313 try: 

314 return settings._profiles[name] 

315 except KeyError: 

316 raise InvalidArgument("Profile %r is not registered" % (name,)) 

317 

318 @staticmethod 

319 def load_profile(name: str) -> None: 

320 """Loads in the settings defined in the profile provided. 

321 

322 If the profile does not exist, InvalidArgument will be raised. 

323 Any setting not defined in the profile will be the library 

324 defined default for that setting. 

325 """ 

326 check_type(str, name, "name") 

327 settings._current_profile = name 

328 settings._assign_default_internal(settings.get_profile(name)) 

329 

330 

331@contextlib.contextmanager 

332def local_settings(s): 

333 default_context_manager = default_variable.with_value(s) 

334 with default_context_manager: 

335 yield s 

336 

337 

338@attr.s() 

339class Setting: 

340 name = attr.ib() 

341 description = attr.ib() 

342 default = attr.ib() 

343 options = attr.ib() 

344 validator = attr.ib() 

345 

346 

347def _max_examples_validator(x): 

348 check_type(int, x, name="max_examples") 

349 if x < 1: 

350 raise InvalidArgument( 

351 "max_examples=%r should be at least one. You can disable example " 

352 "generation with the `phases` setting instead." % (x,) 

353 ) 

354 return x 

355 

356 

357settings._define_setting( 

358 "max_examples", 

359 default=100, 

360 validator=_max_examples_validator, 

361 description=""" 

362Once this many satisfying examples have been considered without finding any 

363counter-example, falsification will terminate. 

364 

365The default value is chosen to suit a workflow where the test will be part of 

366a suite that is regularly executed locally or on a CI server, balancing total 

367running time against the chance of missing a bug. 

368 

369If you are writing one-off tests, running tens of thousands of examples is 

370quite reasonable as Hypothesis may miss uncommon bugs with default settings. 

371For very complex code, we have observed Hypothesis finding novel bugs after 

372*several million* examples while testing :pypi:`SymPy`. 

373""", 

374) 

375 

376 

377settings._define_setting( 

378 "derandomize", 

379 default=False, 

380 options=(True, False), 

381 description=""" 

382If this is True then hypothesis will run in deterministic mode 

383where each falsification uses a random number generator that is seeded 

384based on the hypothesis to falsify, which will be consistent across 

385multiple runs. This has the advantage that it will eliminate any 

386randomness from your tests, which may be preferable for some situations. 

387It does have the disadvantage of making your tests less likely to 

388find novel breakages. 

389""", 

390) 

391 

392 

393def _validate_database(db): 

394 from hypothesis.database import ExampleDatabase 

395 

396 if db is None or isinstance(db, ExampleDatabase): 

397 return db 

398 raise InvalidArgument( 

399 "Arguments to the database setting must be None or an instance of " 

400 "ExampleDatabase. Try passing database=ExampleDatabase(%r), or " 

401 "construct and use one of the specific subclasses in " 

402 "hypothesis.database" % (db,) 

403 ) 

404 

405 

406settings._define_setting( 

407 "database", 

408 default=not_set, 

409 show_default=False, 

410 description=""" 

411An instance of hypothesis.database.ExampleDatabase that will be 

412used to save examples to and load previous examples from. May be ``None`` 

413in which case no storage will be used, ``":memory:"`` for an in-memory 

414database, or any path for a directory-based example database. 

415""", 

416 validator=_validate_database, 

417) 

418 

419 

420@unique 

421class Phase(IntEnum): 

422 explicit = 0 

423 reuse = 1 

424 generate = 2 

425 target = 3 

426 shrink = 4 

427 

428 def __repr__(self): 

429 return "Phase.%s" % (self.name,) 

430 

431 

432@unique 

433class HealthCheck(Enum): 

434 """Arguments for :attr:`~hypothesis.settings.suppress_health_check`. 

435 

436 Each member of this enum is a type of health check to suppress. 

437 """ 

438 

439 def __repr__(self): 

440 return "%s.%s" % (self.__class__.__name__, self.name) 

441 

442 @classmethod 

443 def all(cls) -> List["HealthCheck"]: 

444 return list(HealthCheck) 

445 

446 data_too_large = 1 

447 """Check for when the typical size of the examples you are generating 

448 exceeds the maximum allowed size too often.""" 

449 

450 filter_too_much = 2 

451 """Check for when the test is filtering out too many examples, either 

452 through use of :func:`~hypothesis.assume()` or :ref:`filter() <filtering>`, 

453 or occasionally for Hypothesis internal reasons.""" 

454 

455 too_slow = 3 

456 """Check for when your data generation is extremely slow and likely to hurt 

457 testing.""" 

458 

459 return_value = 5 

460 """Checks if your tests return a non-None value (which will be ignored and 

461 is unlikely to do what you want).""" 

462 

463 large_base_example = 7 

464 """Checks if the natural example to shrink towards is very large.""" 

465 

466 not_a_test_method = 8 

467 """Checks if :func:`@given <hypothesis.given>` has been applied to a 

468 method defined by :class:`python:unittest.TestCase` (i.e. not a test).""" 

469 

470 

471@unique 

472class Verbosity(IntEnum): 

473 quiet = 0 

474 normal = 1 

475 verbose = 2 

476 debug = 3 

477 

478 def __repr__(self): 

479 return "Verbosity.%s" % (self.name,) 

480 

481 

482settings._define_setting( 

483 "verbosity", 

484 options=tuple(Verbosity), 

485 default=Verbosity.normal, 

486 description="Control the verbosity level of Hypothesis messages", 

487) 

488 

489 

490def _validate_phases(phases): 

491 phases = tuple(phases) 

492 for a in phases: 

493 if not isinstance(a, Phase): 

494 raise InvalidArgument("%r is not a valid phase" % (a,)) 

495 return tuple(p for p in list(Phase) if p in phases) 

496 

497 

498settings._define_setting( 

499 "phases", 

500 default=tuple(Phase), 

501 description=( 

502 "Control which phases should be run. " 

503 "See :ref:`the full documentation for more details <phases>`" 

504 ), 

505 validator=_validate_phases, 

506) 

507 

508 

509def _validate_stateful_step_count(x): 

510 check_type(int, x, name="stateful_step_count") 

511 if x < 1: 

512 raise InvalidArgument("stateful_step_count=%r must be at least one." % (x,)) 

513 return x 

514 

515 

516settings._define_setting( 

517 name="stateful_step_count", 

518 default=50, 

519 validator=_validate_stateful_step_count, 

520 description=""" 

521Number of steps to run a stateful program for before giving up on it breaking. 

522""", 

523) 

524 

525settings._define_setting( 

526 name="report_multiple_bugs", 

527 default=True, 

528 options=(True, False), 

529 description=""" 

530Because Hypothesis runs the test many times, it can sometimes find multiple 

531bugs in a single run. Reporting all of them at once is usually very useful, 

532but replacing the exceptions can occasionally clash with debuggers. 

533If disabled, only the exception with the smallest minimal example is raised. 

534""", 

535) 

536 

537 

538def validate_health_check_suppressions(suppressions): 

539 suppressions = try_convert(list, suppressions, "suppress_health_check") 

540 for s in suppressions: 

541 if not isinstance(s, HealthCheck): 

542 raise InvalidArgument( 

543 "Non-HealthCheck value %r of type %s is invalid in suppress_health_check." 

544 % (s, type(s).__name__) 

545 ) 

546 return suppressions 

547 

548 

549settings._define_setting( 

550 "suppress_health_check", 

551 default=(), 

552 description="""A list of :class:`~hypothesis.HealthCheck` items to disable.""", 

553 validator=validate_health_check_suppressions, 

554) 

555 

556 

557class duration(datetime.timedelta): 

558 """A timedelta specifically measured in milliseconds.""" 

559 

560 def __repr__(self): 

561 ms = self.total_seconds() * 1000 

562 return "timedelta(milliseconds=%r)" % (int(ms) if ms == int(ms) else ms,) 

563 

564 

565def _validate_deadline(x): 

566 if x is None: 

567 return x 

568 invalid_deadline_error = InvalidArgument( 

569 "deadline=%r (type %s) must be a timedelta object, an integer or float " 

570 "number of milliseconds, or None to disable the per-test-case deadline." 

571 % (x, type(x).__name__) 

572 ) 

573 if isinstance(x, (int, float)): 

574 if isinstance(x, bool): 

575 raise invalid_deadline_error 

576 try: 

577 x = duration(milliseconds=x) 

578 except OverflowError: 

579 raise InvalidArgument( 

580 "deadline=%r is invalid, because it is too large to represent " 

581 "as a timedelta. Use deadline=None to disable deadlines." % (x,) 

582 ) from None 

583 if isinstance(x, datetime.timedelta): 

584 if x <= datetime.timedelta(0): 

585 raise InvalidArgument( 

586 "deadline=%r is invalid, because it is impossible to meet a " 

587 "deadline <= 0. Use deadline=None to disable deadlines." % (x,) 

588 ) 

589 return duration(seconds=x.total_seconds()) 

590 raise invalid_deadline_error 

591 

592 

593settings._define_setting( 

594 "deadline", 

595 default=duration(milliseconds=200), 

596 validator=_validate_deadline, 

597 description=""" 

598If set, a duration (as timedelta, or integer or float number of milliseconds) 

599that each individual example (i.e. each time your test 

600function is called, not the whole decorated test) within a test is not 

601allowed to exceed. Tests which take longer than that may be converted into 

602errors (but will not necessarily be if close to the deadline, to allow some 

603variability in test run time). 

604 

605Set this to None to disable this behaviour entirely. 

606""", 

607) 

608 

609 

610settings._define_setting( 

611 "print_blob", 

612 default=False, 

613 options=(True, False), 

614 description=""" 

615If set to ``True``, Hypothesis will print code for failing examples that can be used with 

616:func:`@reproduce_failure <hypothesis.reproduce_failure>` to reproduce the failing example. 

617""", 

618) 

619 

620settings.lock_further_definitions() 

621 

622 

623def note_deprecation(message: str, *, since: str) -> None: 

624 if since != "RELEASEDAY": 

625 date = datetime.datetime.strptime(since, "%Y-%m-%d").date() 

626 assert datetime.date(2016, 1, 1) <= date 

627 warnings.warn(HypothesisDeprecationWarning(message), stacklevel=2) 

628 

629 

630settings.register_profile("default", settings()) 

631settings.load_profile("default") 

632assert settings.default is not None