Mercurial > public > mercurial-scm > hg-stable

comparison contrib/python-zstandard/tests/test_compressor_fuzzing.py @ 42070:675775c33ab6

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
zstandard: vendor python-zstandard 0.11 The upstream source distribution from PyPI was extracted. Unwanted files were removed. The clang-format ignore list was updated to reflect the new source of files. The project contains a vendored copy of zstandard 1.3.8. The old version was 1.3.6. This should result in some minor performance wins. test-check-py3-compat.t was updated to reflect now-passing tests on Python 3.8. Some HTTP tests were updated to reflect new zstd compression output. # no-check-commit because 3rd party code has different style guidelines Differential Revision: https://phab.mercurial-scm.org/D6199
author Gregory Szorc <gregory.szorc@gmail.com>
date Thu, 04 Apr 2019 17:34:43 -0700
parents 73fef626dae3
children de7838053207
comparison
equal deleted inserted replaced
42069:668eff08387f 42070:675775c33ab6
10 10
11 import zstandard as zstd 11 import zstandard as zstd
12 12
13 from . common import ( 13 from . common import (
14 make_cffi, 14 make_cffi,
15 NonClosingBytesIO,
15 random_input_data, 16 random_input_data,
16 ) 17 )
17 18
18 19
19 @unittest.skipUnless('ZSTD_SLOW_TESTS' in os.environ, 'ZSTD_SLOW_TESTS not set') 20 @unittest.skipUnless('ZSTD_SLOW_TESTS' in os.environ, 'ZSTD_SLOW_TESTS not set')
20 @make_cffi 21 @make_cffi
21 class TestCompressor_stream_reader_fuzzing(unittest.TestCase): 22 class TestCompressor_stream_reader_fuzzing(unittest.TestCase):
23 @hypothesis.settings(
24 suppress_health_check=[hypothesis.HealthCheck.large_base_example])
25 @hypothesis.given(original=strategies.sampled_from(random_input_data()),
26 level=strategies.integers(min_value=1, max_value=5),
27 source_read_size=strategies.integers(1, 16384),
28 read_size=strategies.integers(-1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE))
29 def test_stream_source_read(self, original, level, source_read_size,
30 read_size):
31 if read_size == 0:
32 read_size = -1
33
34 refctx = zstd.ZstdCompressor(level=level)
35 ref_frame = refctx.compress(original)
36
37 cctx = zstd.ZstdCompressor(level=level)
38 with cctx.stream_reader(io.BytesIO(original), size=len(original),
39 read_size=source_read_size) as reader:
40 chunks = []
41 while True:
42 chunk = reader.read(read_size)
43 if not chunk:
44 break
45
46 chunks.append(chunk)
47
48 self.assertEqual(b''.join(chunks), ref_frame)
49
50 @hypothesis.settings(
51 suppress_health_check=[hypothesis.HealthCheck.large_base_example])
52 @hypothesis.given(original=strategies.sampled_from(random_input_data()),
53 level=strategies.integers(min_value=1, max_value=5),
54 source_read_size=strategies.integers(1, 16384),
55 read_size=strategies.integers(-1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE))
56 def test_buffer_source_read(self, original, level, source_read_size,
57 read_size):
58 if read_size == 0:
59 read_size = -1
60
61 refctx = zstd.ZstdCompressor(level=level)
62 ref_frame = refctx.compress(original)
63
64 cctx = zstd.ZstdCompressor(level=level)
65 with cctx.stream_reader(original, size=len(original),
66 read_size=source_read_size) as reader:
67 chunks = []
68 while True:
69 chunk = reader.read(read_size)
70 if not chunk:
71 break
72
73 chunks.append(chunk)
74
75 self.assertEqual(b''.join(chunks), ref_frame)
76
77 @hypothesis.settings(
78 suppress_health_check=[hypothesis.HealthCheck.large_base_example])
22 @hypothesis.given(original=strategies.sampled_from(random_input_data()), 79 @hypothesis.given(original=strategies.sampled_from(random_input_data()),
23 level=strategies.integers(min_value=1, max_value=5), 80 level=strategies.integers(min_value=1, max_value=5),
24 source_read_size=strategies.integers(1, 16384), 81 source_read_size=strategies.integers(1, 16384),
25 read_sizes=strategies.data()) 82 read_sizes=strategies.data())
26 def test_stream_source_read_variance(self, original, level, source_read_size, 83 def test_stream_source_read_variance(self, original, level, source_read_size,
31 cctx = zstd.ZstdCompressor(level=level) 88 cctx = zstd.ZstdCompressor(level=level)
32 with cctx.stream_reader(io.BytesIO(original), size=len(original), 89 with cctx.stream_reader(io.BytesIO(original), size=len(original),
33 read_size=source_read_size) as reader: 90 read_size=source_read_size) as reader:
34 chunks = [] 91 chunks = []
35 while True: 92 while True:
36 read_size = read_sizes.draw(strategies.integers(1, 16384)) 93 read_size = read_sizes.draw(strategies.integers(-1, 16384))
37 chunk = reader.read(read_size) 94 chunk = reader.read(read_size)
38 95 if not chunk and read_size:
39 if not chunk: 96 break
40 break 97
41 chunks.append(chunk) 98 chunks.append(chunk)
42 99
43 self.assertEqual(b''.join(chunks), ref_frame) 100 self.assertEqual(b''.join(chunks), ref_frame)
44 101
102 @hypothesis.settings(
103 suppress_health_check=[hypothesis.HealthCheck.large_base_example])
45 @hypothesis.given(original=strategies.sampled_from(random_input_data()), 104 @hypothesis.given(original=strategies.sampled_from(random_input_data()),
46 level=strategies.integers(min_value=1, max_value=5), 105 level=strategies.integers(min_value=1, max_value=5),
47 source_read_size=strategies.integers(1, 16384), 106 source_read_size=strategies.integers(1, 16384),
48 read_sizes=strategies.data()) 107 read_sizes=strategies.data())
49 def test_buffer_source_read_variance(self, original, level, source_read_size, 108 def test_buffer_source_read_variance(self, original, level, source_read_size,
55 cctx = zstd.ZstdCompressor(level=level) 114 cctx = zstd.ZstdCompressor(level=level)
56 with cctx.stream_reader(original, size=len(original), 115 with cctx.stream_reader(original, size=len(original),
57 read_size=source_read_size) as reader: 116 read_size=source_read_size) as reader:
58 chunks = [] 117 chunks = []
59 while True: 118 while True:
119 read_size = read_sizes.draw(strategies.integers(-1, 16384))
120 chunk = reader.read(read_size)
121 if not chunk and read_size:
122 break
123
124 chunks.append(chunk)
125
126 self.assertEqual(b''.join(chunks), ref_frame)
127
128 @hypothesis.settings(
129 suppress_health_check=[hypothesis.HealthCheck.large_base_example])
130 @hypothesis.given(original=strategies.sampled_from(random_input_data()),
131 level=strategies.integers(min_value=1, max_value=5),
132 source_read_size=strategies.integers(1, 16384),
133 read_size=strategies.integers(1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE))
134 def test_stream_source_readinto(self, original, level,
135 source_read_size, read_size):
136 refctx = zstd.ZstdCompressor(level=level)
137 ref_frame = refctx.compress(original)
138
139 cctx = zstd.ZstdCompressor(level=level)
140 with cctx.stream_reader(io.BytesIO(original), size=len(original),
141 read_size=source_read_size) as reader:
142 chunks = []
143 while True:
144 b = bytearray(read_size)
145 count = reader.readinto(b)
146
147 if not count:
148 break
149
150 chunks.append(bytes(b[0:count]))
151
152 self.assertEqual(b''.join(chunks), ref_frame)
153
154 @hypothesis.settings(
155 suppress_health_check=[hypothesis.HealthCheck.large_base_example])
156 @hypothesis.given(original=strategies.sampled_from(random_input_data()),
157 level=strategies.integers(min_value=1, max_value=5),
158 source_read_size=strategies.integers(1, 16384),
159 read_size=strategies.integers(1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE))
160 def test_buffer_source_readinto(self, original, level,
161 source_read_size, read_size):
162
163 refctx = zstd.ZstdCompressor(level=level)
164 ref_frame = refctx.compress(original)
165
166 cctx = zstd.ZstdCompressor(level=level)
167 with cctx.stream_reader(original, size=len(original),
168 read_size=source_read_size) as reader:
169 chunks = []
170 while True:
171 b = bytearray(read_size)
172 count = reader.readinto(b)
173
174 if not count:
175 break
176
177 chunks.append(bytes(b[0:count]))
178
179 self.assertEqual(b''.join(chunks), ref_frame)
180
181 @hypothesis.settings(
182 suppress_health_check=[hypothesis.HealthCheck.large_base_example])
183 @hypothesis.given(original=strategies.sampled_from(random_input_data()),
184 level=strategies.integers(min_value=1, max_value=5),
185 source_read_size=strategies.integers(1, 16384),
186 read_sizes=strategies.data())
187 def test_stream_source_readinto_variance(self, original, level,
188 source_read_size, read_sizes):
189 refctx = zstd.ZstdCompressor(level=level)
190 ref_frame = refctx.compress(original)
191
192 cctx = zstd.ZstdCompressor(level=level)
193 with cctx.stream_reader(io.BytesIO(original), size=len(original),
194 read_size=source_read_size) as reader:
195 chunks = []
196 while True:
60 read_size = read_sizes.draw(strategies.integers(1, 16384)) 197 read_size = read_sizes.draw(strategies.integers(1, 16384))
61 chunk = reader.read(read_size) 198 b = bytearray(read_size)
199 count = reader.readinto(b)
200
201 if not count:
202 break
203
204 chunks.append(bytes(b[0:count]))
205
206 self.assertEqual(b''.join(chunks), ref_frame)
207
208 @hypothesis.settings(
209 suppress_health_check=[hypothesis.HealthCheck.large_base_example])
210 @hypothesis.given(original=strategies.sampled_from(random_input_data()),
211 level=strategies.integers(min_value=1, max_value=5),
212 source_read_size=strategies.integers(1, 16384),
213 read_sizes=strategies.data())
214 def test_buffer_source_readinto_variance(self, original, level,
215 source_read_size, read_sizes):
216
217 refctx = zstd.ZstdCompressor(level=level)
218 ref_frame = refctx.compress(original)
219
220 cctx = zstd.ZstdCompressor(level=level)
221 with cctx.stream_reader(original, size=len(original),
222 read_size=source_read_size) as reader:
223 chunks = []
224 while True:
225 read_size = read_sizes.draw(strategies.integers(1, 16384))
226 b = bytearray(read_size)
227 count = reader.readinto(b)
228
229 if not count:
230 break
231
232 chunks.append(bytes(b[0:count]))
233
234 self.assertEqual(b''.join(chunks), ref_frame)
235
236 @hypothesis.settings(
237 suppress_health_check=[hypothesis.HealthCheck.large_base_example])
238 @hypothesis.given(original=strategies.sampled_from(random_input_data()),
239 level=strategies.integers(min_value=1, max_value=5),
240 source_read_size=strategies.integers(1, 16384),
241 read_size=strategies.integers(-1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE))
242 def test_stream_source_read1(self, original, level, source_read_size,
243 read_size):
244 if read_size == 0:
245 read_size = -1
246
247 refctx = zstd.ZstdCompressor(level=level)
248 ref_frame = refctx.compress(original)
249
250 cctx = zstd.ZstdCompressor(level=level)
251 with cctx.stream_reader(io.BytesIO(original), size=len(original),
252 read_size=source_read_size) as reader:
253 chunks = []
254 while True:
255 chunk = reader.read1(read_size)
62 if not chunk: 256 if not chunk:
63 break 257 break
258
64 chunks.append(chunk) 259 chunks.append(chunk)
65 260
66 self.assertEqual(b''.join(chunks), ref_frame) 261 self.assertEqual(b''.join(chunks), ref_frame)
262
263 @hypothesis.settings(
264 suppress_health_check=[hypothesis.HealthCheck.large_base_example])
265 @hypothesis.given(original=strategies.sampled_from(random_input_data()),
266 level=strategies.integers(min_value=1, max_value=5),
267 source_read_size=strategies.integers(1, 16384),
268 read_size=strategies.integers(-1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE))
269 def test_buffer_source_read1(self, original, level, source_read_size,
270 read_size):
271 if read_size == 0:
272 read_size = -1
273
274 refctx = zstd.ZstdCompressor(level=level)
275 ref_frame = refctx.compress(original)
276
277 cctx = zstd.ZstdCompressor(level=level)
278 with cctx.stream_reader(original, size=len(original),
279 read_size=source_read_size) as reader:
280 chunks = []
281 while True:
282 chunk = reader.read1(read_size)
283 if not chunk:
284 break
285
286 chunks.append(chunk)
287
288 self.assertEqual(b''.join(chunks), ref_frame)
289
290 @hypothesis.settings(
291 suppress_health_check=[hypothesis.HealthCheck.large_base_example])
292 @hypothesis.given(original=strategies.sampled_from(random_input_data()),
293 level=strategies.integers(min_value=1, max_value=5),
294 source_read_size=strategies.integers(1, 16384),
295 read_sizes=strategies.data())
296 def test_stream_source_read1_variance(self, original, level, source_read_size,
297 read_sizes):
298 refctx = zstd.ZstdCompressor(level=level)
299 ref_frame = refctx.compress(original)
300
301 cctx = zstd.ZstdCompressor(level=level)
302 with cctx.stream_reader(io.BytesIO(original), size=len(original),
303 read_size=source_read_size) as reader:
304 chunks = []
305 while True:
306 read_size = read_sizes.draw(strategies.integers(-1, 16384))
307 chunk = reader.read1(read_size)
308 if not chunk and read_size:
309 break
310
311 chunks.append(chunk)
312
313 self.assertEqual(b''.join(chunks), ref_frame)
314
315 @hypothesis.settings(
316 suppress_health_check=[hypothesis.HealthCheck.large_base_example])
317 @hypothesis.given(original=strategies.sampled_from(random_input_data()),
318 level=strategies.integers(min_value=1, max_value=5),
319 source_read_size=strategies.integers(1, 16384),
320 read_sizes=strategies.data())
321 def test_buffer_source_read1_variance(self, original, level, source_read_size,
322 read_sizes):
323
324 refctx = zstd.ZstdCompressor(level=level)
325 ref_frame = refctx.compress(original)
326
327 cctx = zstd.ZstdCompressor(level=level)
328 with cctx.stream_reader(original, size=len(original),
329 read_size=source_read_size) as reader:
330 chunks = []
331 while True:
332 read_size = read_sizes.draw(strategies.integers(-1, 16384))
333 chunk = reader.read1(read_size)
334 if not chunk and read_size:
335 break
336
337 chunks.append(chunk)
338
339 self.assertEqual(b''.join(chunks), ref_frame)
340
341
342 @hypothesis.settings(
343 suppress_health_check=[hypothesis.HealthCheck.large_base_example])
344 @hypothesis.given(original=strategies.sampled_from(random_input_data()),
345 level=strategies.integers(min_value=1, max_value=5),
346 source_read_size=strategies.integers(1, 16384),
347 read_size=strategies.integers(1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE))
348 def test_stream_source_readinto1(self, original, level, source_read_size,
349 read_size):
350 if read_size == 0:
351 read_size = -1
352
353 refctx = zstd.ZstdCompressor(level=level)
354 ref_frame = refctx.compress(original)
355
356 cctx = zstd.ZstdCompressor(level=level)
357 with cctx.stream_reader(io.BytesIO(original), size=len(original),
358 read_size=source_read_size) as reader:
359 chunks = []
360 while True:
361 b = bytearray(read_size)
362 count = reader.readinto1(b)
363
364 if not count:
365 break
366
367 chunks.append(bytes(b[0:count]))
368
369 self.assertEqual(b''.join(chunks), ref_frame)
370
371 @hypothesis.settings(
372 suppress_health_check=[hypothesis.HealthCheck.large_base_example])
373 @hypothesis.given(original=strategies.sampled_from(random_input_data()),
374 level=strategies.integers(min_value=1, max_value=5),
375 source_read_size=strategies.integers(1, 16384),
376 read_size=strategies.integers(1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE))
377 def test_buffer_source_readinto1(self, original, level, source_read_size,
378 read_size):
379 if read_size == 0:
380 read_size = -1
381
382 refctx = zstd.ZstdCompressor(level=level)
383 ref_frame = refctx.compress(original)
384
385 cctx = zstd.ZstdCompressor(level=level)
386 with cctx.stream_reader(original, size=len(original),
387 read_size=source_read_size) as reader:
388 chunks = []
389 while True:
390 b = bytearray(read_size)
391 count = reader.readinto1(b)
392
393 if not count:
394 break
395
396 chunks.append(bytes(b[0:count]))
397
398 self.assertEqual(b''.join(chunks), ref_frame)
399
400 @hypothesis.settings(
401 suppress_health_check=[hypothesis.HealthCheck.large_base_example])
402 @hypothesis.given(original=strategies.sampled_from(random_input_data()),
403 level=strategies.integers(min_value=1, max_value=5),
404 source_read_size=strategies.integers(1, 16384),
405 read_sizes=strategies.data())
406 def test_stream_source_readinto1_variance(self, original, level, source_read_size,
407 read_sizes):
408 refctx = zstd.ZstdCompressor(level=level)
409 ref_frame = refctx.compress(original)
410
411 cctx = zstd.ZstdCompressor(level=level)
412 with cctx.stream_reader(io.BytesIO(original), size=len(original),
413 read_size=source_read_size) as reader:
414 chunks = []
415 while True:
416 read_size = read_sizes.draw(strategies.integers(1, 16384))
417 b = bytearray(read_size)
418 count = reader.readinto1(b)
419
420 if not count:
421 break
422
423 chunks.append(bytes(b[0:count]))
424
425 self.assertEqual(b''.join(chunks), ref_frame)
426
427 @hypothesis.settings(
428 suppress_health_check=[hypothesis.HealthCheck.large_base_example])
429 @hypothesis.given(original=strategies.sampled_from(random_input_data()),
430 level=strategies.integers(min_value=1, max_value=5),
431 source_read_size=strategies.integers(1, 16384),
432 read_sizes=strategies.data())
433 def test_buffer_source_readinto1_variance(self, original, level, source_read_size,
434 read_sizes):
435
436 refctx = zstd.ZstdCompressor(level=level)
437 ref_frame = refctx.compress(original)
438
439 cctx = zstd.ZstdCompressor(level=level)
440 with cctx.stream_reader(original, size=len(original),
441 read_size=source_read_size) as reader:
442 chunks = []
443 while True:
444 read_size = read_sizes.draw(strategies.integers(1, 16384))
445 b = bytearray(read_size)
446 count = reader.readinto1(b)
447
448 if not count:
449 break
450
451 chunks.append(bytes(b[0:count]))
452
453 self.assertEqual(b''.join(chunks), ref_frame)
454
67 455
68 456
69 @unittest.skipUnless('ZSTD_SLOW_TESTS' in os.environ, 'ZSTD_SLOW_TESTS not set') 457 @unittest.skipUnless('ZSTD_SLOW_TESTS' in os.environ, 'ZSTD_SLOW_TESTS not set')
70 @make_cffi 458 @make_cffi
71 class TestCompressor_stream_writer_fuzzing(unittest.TestCase): 459 class TestCompressor_stream_writer_fuzzing(unittest.TestCase):
75 def test_write_size_variance(self, original, level, write_size): 463 def test_write_size_variance(self, original, level, write_size):
76 refctx = zstd.ZstdCompressor(level=level) 464 refctx = zstd.ZstdCompressor(level=level)
77 ref_frame = refctx.compress(original) 465 ref_frame = refctx.compress(original)
78 466
79 cctx = zstd.ZstdCompressor(level=level) 467 cctx = zstd.ZstdCompressor(level=level)
80 b = io.BytesIO() 468 b = NonClosingBytesIO()
81 with cctx.stream_writer(b, size=len(original), write_size=write_size) as compressor: 469 with cctx.stream_writer(b, size=len(original), write_size=write_size) as compressor:
82 compressor.write(original) 470 compressor.write(original)
83 471
84 self.assertEqual(b.getvalue(), ref_frame) 472 self.assertEqual(b.getvalue(), ref_frame)
85 473
217 605
218 cctx = zstd.ZstdCompressor(level=1, 606 cctx = zstd.ZstdCompressor(level=1,
219 write_checksum=True, 607 write_checksum=True,
220 **kwargs) 608 **kwargs)
221 609
610 if not hasattr(cctx, 'multi_compress_to_buffer'):
611 self.skipTest('multi_compress_to_buffer not available')
612
222 result = cctx.multi_compress_to_buffer(original, threads=-1) 613 result = cctx.multi_compress_to_buffer(original, threads=-1)
223 614
224 self.assertEqual(len(result), len(original)) 615 self.assertEqual(len(result), len(original))
225 616
226 # The frame produced via the batch APIs may not be bit identical to that 617 # The frame produced via the batch APIs may not be bit identical to that