Mercurial > public > mercurial-scm > hg

comparison tests/test-verify-repo-operations.py @ 28255:f75f7d39cca3

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
testing: generate tests operations using Hypothesis The idea of this patch is to expand the use of Hypothesis within Mercurial to use its concept of "stateful testing". The result is a test which runs a sequence of operations against a Mercurial repository. Each operation is given a set of allowed ways it can fail. Any other non-zero exit code is a test failure. At the end, the whole sequence is then reverified by generating a .t test and testing it again in pure mode (this is also useful for catching non-determinism bugs). This has proven reasonably effective at finding bugs, and has identified two problems in the shelve extension already (issue5113 and issue5112).
author David R. MacIver <david@drmaciver.com>
date Wed, 24 Feb 2016 13:05:45 +0000
parents
children 55325bdf6c13
comparison
equal deleted inserted replaced
28251:4591cd6b6794 28255:f75f7d39cca3
1 from __future__ import print_function, absolute_import
2
3 """Fuzz testing for operations against a Mercurial repository
4
5 This uses Hypothesis's stateful testing to generate random repository
6 operations and test Mercurial using them, both to see if there are any
7 unexpected errors and to compare different versions of it."""
8
9 import os
10 import sys
11
12 # These tests require Hypothesis and pytz to be installed.
13 # Running 'pip install hypothesis pytz' will achieve that.
14 # Note: This won't work if you're running Python < 2.7.
15 try:
16 from hypothesis.extra.datetime import datetimes
17 except ImportError:
18 sys.stderr.write("skipped: hypothesis or pytz not installed" + os.linesep)
19 sys.exit(80)
20
21 # If you are running an old version of pip you may find that the enum34
22 # backport is not installed automatically. If so 'pip install enum34' will
23 # fix this problem.
24 try:
25 import enum
26 assert enum # Silence pyflakes
27 except ImportError:
28 sys.stderr.write("skipped: enum34 not installed" + os.linesep)
29 sys.exit(80)
30
31 import binascii
32 from contextlib import contextmanager
33 import errno
34 import pipes
35 import shutil
36 import silenttestrunner
37 import subprocess
38
39 from hypothesis.errors import HypothesisException
40 from hypothesis.stateful import rule, RuleBasedStateMachine, Bundle
41 from hypothesis import settings, note, strategies as st
42 from hypothesis.configuration import set_hypothesis_home_dir
43
44 testdir = os.path.abspath(os.environ["TESTDIR"])
45
46 # We store Hypothesis examples here rather in the temporary test directory
47 # so that when rerunning a failing test this always results in refinding the
48 # previous failure. This directory is in .hgignore and should not be checked in
49 # but is useful to have for development.
50 set_hypothesis_home_dir(os.path.join(testdir, ".hypothesis"))
51
52 runtests = os.path.join(os.environ["RUNTESTDIR"], "run-tests.py")
53 testtmp = os.environ["TESTTMP"]
54 assert os.path.isdir(testtmp)
55
56 generatedtests = os.path.join(testdir, "hypothesis-generated")
57
58 try:
59 os.makedirs(generatedtests)
60 except OSError:
61 pass
62
63 # We write out generated .t files to a file in order to ease debugging and to
64 # give a starting point for turning failures Hypothesis finds into normal
65 # tests. In order to ensure that multiple copies of this test can be run in
66 # parallel we use atomic file create to ensure that we always get a unique
67 # name.
68 file_index = 0
69 while True:
70 file_index += 1
71 savefile = os.path.join(generatedtests, "test-generated-%d.t" % (
72 file_index,
73 ))
74 try:
75 os.close(os.open(savefile, os.O_CREAT | os.O_EXCL | os.O_WRONLY))
76 break
77 except OSError as e:
78 if e.errno != errno.EEXIST:
79 raise
80 assert os.path.exists(savefile)
81
82 hgrc = os.path.join(".hg", "hgrc")
83
84 filecharacters = (
85 "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
86 "[]^_`;=@{}~ !#$%&'()+,-"
87 )
88
89 files = st.text(filecharacters, min_size=1).map(lambda x: x.strip()).filter(
90 bool).map(lambda s: s.encode('ascii'))
91
92 safetext = st.text(st.characters(
93 min_codepoint=1, max_codepoint=127,
94 blacklist_categories=('Cc', 'Cs')), min_size=1).map(
95 lambda s: s.encode('utf-8')
96 )
97
98 @contextmanager
99 def acceptableerrors(*args):
100 """Sometimes we know an operation we're about to perform might fail, and
101 we're OK with some of the failures. In those cases this may be used as a
102 context manager and will swallow expected failures, as identified by
103 substrings of the error message Mercurial emits."""
104 try:
105 yield
106 except subprocess.CalledProcessError as e:
107 if not any(a in e.output for a in args):
108 note(e.output)
109 raise
110
111 class verifyingstatemachine(RuleBasedStateMachine):
112 """This defines the set of acceptable operations on a Mercurial repository
113 using Hypothesis's RuleBasedStateMachine.
114
115 The general concept is that we manage multiple repositories inside a
116 repos/ directory in our temporary test location. Some of these are freshly
117 inited, some are clones of the others. Our current working directory is
118 always inside one of these repositories while the tests are running.
119
120 Hypothesis then performs a series of operations against these repositories,
121 including hg commands, generating contents and editing the .hgrc file.
122 If these operations fail in unexpected ways or behave differently in
123 different configurations of Mercurial, the test will fail and a minimized
124 .t test file will be written to the hypothesis-generated directory to
125 exhibit that failure.
126
127 Operations are defined as methods with @rule() decorators. See the
128 Hypothesis documentation at
129 http://hypothesis.readthedocs.org/en/release/stateful.html for more
130 details."""
131
132 # A bundle is a reusable collection of previously generated data which may
133 # be provided as arguments to future operations.
134 paths = Bundle('paths')
135 contents = Bundle('contents')
136 committimes = Bundle('committimes')
137
138 def __init__(self):
139 super(verifyingstatemachine, self).__init__()
140 self.repodir = os.path.join(testtmp, "repo")
141 if os.path.exists(self.repodir):
142 shutil.rmtree(self.repodir)
143 os.chdir(testtmp)
144 self.log = []
145 self.failed = False
146
147 self.mkdirp("repo")
148 self.cd("repo")
149 self.hg("init")
150
151 def teardown(self):
152 """On teardown we clean up after ourselves as usual, but we also
153 do some additional testing: We generate a .t file based on our test
154 run using run-test.py -i to get the correct output.
155
156 We then test it in a number of other configurations, verifying that
157 each passes the same test."""
158 super(verifyingstatemachine, self).teardown()
159 try:
160 shutil.rmtree(self.repodir)
161 except OSError:
162 pass
163 ttest = os.linesep.join(" " + l for l in self.log)
164 os.chdir(testtmp)
165 path = os.path.join(testtmp, "test-generated.t")
166 with open(path, 'w') as o:
167 o.write(ttest + os.linesep)
168 with open(os.devnull, "w") as devnull:
169 rewriter = subprocess.Popen(
170 [runtests, "--local", "-i", path], stdin=subprocess.PIPE,
171 stdout=devnull, stderr=devnull,
172 )
173 rewriter.communicate("yes")
174 with open(path, 'r') as i:
175 ttest = i.read()
176
177 e = None
178 if not self.failed:
179 try:
180 output = subprocess.check_output([
181 runtests, path, "--local", "--pure"
182 ], stderr=subprocess.STDOUT)
183 assert "Ran 1 test" in output, output
184 except subprocess.CalledProcessError as e:
185 note(e.output)
186 finally:
187 os.unlink(path)
188 try:
189 os.unlink(path + ".err")
190 except OSError:
191 pass
192 if self.failed or e is not None:
193 with open(savefile, "wb") as o:
194 o.write(ttest)
195 if e is not None:
196 raise e
197
198 def execute_step(self, step):
199 try:
200 return super(verifyingstatemachine, self).execute_step(step)
201 except (HypothesisException, KeyboardInterrupt):
202 raise
203 except Exception:
204 self.failed = True
205 raise
206
207 # Section: Basic commands.
208 def mkdirp(self, path):
209 if os.path.exists(path):
210 return
211 self.log.append(
212 "$ mkdir -p -- %s" % (pipes.quote(os.path.relpath(path)),))
213 os.makedirs(path)
214
215 def cd(self, path):
216 path = os.path.relpath(path)
217 if path == ".":
218 return
219 os.chdir(path)
220 self.log.append("$ cd -- %s" % (pipes.quote(path),))
221
222 def hg(self, *args):
223 self.command("hg", *args)
224
225 def command(self, *args):
226 self.log.append("$ " + ' '.join(map(pipes.quote, args)))
227 subprocess.check_output(args, stderr=subprocess.STDOUT)
228
229 # Section: Set up basic data
230 # This section has no side effects but generates data that we will want
231 # to use later.
232 @rule(
233 target=paths,
234 source=st.lists(files, min_size=1).map(lambda l: os.path.join(*l)))
235 def genpath(self, source):
236 return source
237
238 @rule(
239 target=committimes,
240 when=datetimes(min_year=1970, max_year=2038) | st.none())
241 def gentime(self, when):
242 return when
243
244 @rule(
245 target=contents,
246 content=st.one_of(
247 st.binary(),
248 st.text().map(lambda x: x.encode('utf-8'))
249 ))
250 def gencontent(self, content):
251 return content
252
253 @rule(target=paths, source=paths)
254 def lowerpath(self, source):
255 return source.lower()
256
257 @rule(target=paths, source=paths)
258 def upperpath(self, source):
259 return source.upper()
260
261 # Section: Basic path operations
262 @rule(path=paths, content=contents)
263 def writecontent(self, path, content):
264 self.unadded_changes = True
265 if os.path.isdir(path):
266 return
267 parent = os.path.dirname(path)
268 if parent:
269 try:
270 self.mkdirp(parent)
271 except OSError:
272 # It may be the case that there is a regular file that has
273 # previously been created that has the same name as an ancestor
274 # of the current path. This will cause mkdirp to fail with this
275 # error. We just turn this into a no-op in that case.
276 return
277 with open(path, 'wb') as o:
278 o.write(content)
279 self.log.append((
280 "$ python -c 'import binascii; "
281 "print(binascii.unhexlify(\"%s\"))' > %s") % (
282 binascii.hexlify(content),
283 pipes.quote(path),
284 ))
285
286 @rule(path=paths)
287 def addpath(self, path):
288 if os.path.exists(path):
289 self.hg("add", "--", path)
290
291 @rule(path=paths)
292 def forgetpath(self, path):
293 if os.path.exists(path):
294 with acceptableerrors(
295 "file is already untracked",
296 ):
297 self.hg("forget", "--", path)
298
299 @rule(s=st.none() | st.integers(0, 100))
300 def addremove(self, s):
301 args = ["addremove"]
302 if s is not None:
303 args.extend(["-s", str(s)])
304 self.hg(*args)
305
306 @rule(path=paths)
307 def removepath(self, path):
308 if os.path.exists(path):
309 with acceptableerrors(
310 'file is untracked',
311 'file has been marked for add',
312 'file is modified',
313 ):
314 self.hg("remove", "--", path)
315
316 @rule(
317 message=safetext,
318 amend=st.booleans(),
319 when=committimes,
320 addremove=st.booleans(),
321 secret=st.booleans(),
322 close_branch=st.booleans(),
323 )
324 def maybecommit(
325 self, message, amend, when, addremove, secret, close_branch
326 ):
327 command = ["commit"]
328 errors = ["nothing changed"]
329 if amend:
330 errors.append("cannot amend public changesets")
331 command.append("--amend")
332 command.append("-m" + pipes.quote(message))
333 if secret:
334 command.append("--secret")
335 if close_branch:
336 command.append("--close-branch")
337 errors.append("can only close branch heads")
338 if addremove:
339 command.append("--addremove")
340 if when is not None:
341 if when.year == 1970:
342 errors.append('negative date value')
343 if when.year == 2038:
344 errors.append('exceeds 32 bits')
345 command.append("--date=%s" % (
346 when.strftime('%Y-%m-%d %H:%M:%S %z'),))
347
348 with acceptableerrors(*errors):
349 self.hg(*command)
350
351 # Section: Simple side effect free "check" operations
352 @rule()
353 def log(self):
354 self.hg("log")
355
356 @rule()
357 def verify(self):
358 self.hg("verify")
359
360 @rule()
361 def diff(self):
362 self.hg("diff", "--nodates")
363
364 @rule()
365 def status(self):
366 self.hg("status")
367
368 @rule()
369 def export(self):
370 self.hg("export")
371
372 settings.register_profile(
373 'default', settings(
374 timeout=300,
375 stateful_step_count=50,
376 max_examples=10,
377 )
378 )
379
380 settings.register_profile(
381 'fast', settings(
382 timeout=10,
383 stateful_step_count=20,
384 max_examples=5,
385 min_satisfying_examples=1,
386 max_shrinks=0,
387 )
388 )
389
390 settings.load_profile(os.getenv('HYPOTHESIS_PROFILE', 'default'))
391
392 verifyingtest = verifyingstatemachine.TestCase
393
394 verifyingtest.settings = settings.default
395
396 if __name__ == '__main__':
397 try:
398 silenttestrunner.main(__name__)
399 finally:
400 # So as to prevent proliferation of useless test files, if we never
401 # actually wrote a failing test we clean up after ourselves and delete
402 # the file for doing so that we owned.
403 if os.path.exists(savefile) and os.path.getsize(savefile) == 0:
404 os.unlink(savefile)