mercurial-scm/hg: contrib/python-zstandard/c-ext/compressor.c annotate

annotate contrib/python-zstandard/c-ext/compressor.c @ 31796:e0dc40530c5a

zstd: vendor python-zstandard 0.8.0 Commit 81e1f5bbf1fc54808649562d3ed829730765c540 from https://github.com/indygreg/python-zstandard is imported without modifications (other than removing unwanted files). Updates relevant to Mercurial include: * Support for multi-threaded compression (we can use this for bundle and wire protocol compression). * APIs for batch compression and decompression operations using multiple threads and optimal memory allocation mechanism. (Can be useful for revlog perf improvements.) * A ``BufferWithSegments`` type that models a single memory buffer containing N discrete items of known lengths. This type can be used for very efficient 0-copy data operations. # no-check-commit

author	Gregory Szorc <gregory.szorc@gmail.com>
date	Sat, 01 Apr 2017 15:24:03 -0700
parents	c32454d69b85
children	b1fb341d8a61

rev	line source
30435 b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1 /**
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	2 * Copyright (c) 2016-present, Gregory Szorc
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	3 * All rights reserved.
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	4 *
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	5 * This software may be modified and distributed under the terms
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	6 * of the BSD license. See the LICENSE file for details.
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	7 */
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	8
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	9 #include "python-zstandard.h"
31796 e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	10 #include "pool.h"
30435 b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	11
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	12 extern PyObject* ZstdError;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	13
31796 e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	14 int populate_cdict(ZstdCompressor* compressor, ZSTD_parameters* zparams) {
30822 b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	15 ZSTD_customMem zmem;
31796 e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	16
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	17 if (compressor->cdict \|\| !compressor->dict \|\| !compressor->dict->dictData) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	18 return 0;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	19 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	20
30822 b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	21 Py_BEGIN_ALLOW_THREADS
b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	22 memset(&zmem, 0, sizeof(zmem));
b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	23 compressor->cdict = ZSTD_createCDict_advanced(compressor->dict->dictData,
30895 c32454d69b85 zstd: vendor python-zstandard 0.7.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30830 diff changeset	24 compressor->dict->dictSize, 1, *zparams, zmem);
30822 b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	25 Py_END_ALLOW_THREADS
b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	26
b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	27 if (!compressor->cdict) {
b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	28 PyErr_SetString(ZstdError, "could not create compression dictionary");
b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	29 return 1;
b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	30 }
b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	31
b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	32 return 0;
b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	33 }
b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	34
30435 b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	35 /**
31796 e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	36 * Ensure the ZSTD_CStream on a ZstdCompressor instance is initialized.
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	37 *
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	38 * Returns 0 on success. Other value on failure. Will set a Python exception
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	39 * on failure.
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	40 */
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	41 int init_cstream(ZstdCompressor* compressor, unsigned long long sourceSize) {
30435 b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	42 ZSTD_parameters zparams;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	43 void* dictData = NULL;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	44 size_t dictSize = 0;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	45 size_t zresult;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	46
31796 e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	47 if (compressor->cstream) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	48 zresult = ZSTD_resetCStream(compressor->cstream, sourceSize);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	49 if (ZSTD_isError(zresult)) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	50 PyErr_Format(ZstdError, "could not reset CStream: %s",
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	51 ZSTD_getErrorName(zresult));
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	52 return -1;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	53 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	54
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	55 return 0;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	56 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	57
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	58 compressor->cstream = ZSTD_createCStream();
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	59 if (!compressor->cstream) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	60 PyErr_SetString(ZstdError, "could not create CStream");
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	61 return -1;
30435 b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	62 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	63
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	64 if (compressor->dict) {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	65 dictData = compressor->dict->dictData;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	66 dictSize = compressor->dict->dictSize;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	67 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	68
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	69 memset(&zparams, 0, sizeof(zparams));
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	70 if (compressor->cparams) {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	71 ztopy_compression_parameters(compressor->cparams, &zparams.cParams);
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	72 /* Do NOT call ZSTD_adjustCParams() here because the compression params
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	73 come from the user. */
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	74 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	75 else {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	76 zparams.cParams = ZSTD_getCParams(compressor->compressionLevel, sourceSize, dictSize);
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	77 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	78
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	79 zparams.fParams = compressor->fparams;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	80
31796 e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	81 zresult = ZSTD_initCStream_advanced(compressor->cstream, dictData, dictSize,
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	82 zparams, sourceSize);
30435 b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	83
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	84 if (ZSTD_isError(zresult)) {
31796 e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	85 ZSTD_freeCStream(compressor->cstream);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	86 compressor->cstream = NULL;
30435 b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	87 PyErr_Format(ZstdError, "cannot init CStream: %s", ZSTD_getErrorName(zresult));
31796 e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	88 return -1;
30435 b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	89 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	90
31796 e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	91 return 0;;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	92 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	93
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	94 int init_mtcstream(ZstdCompressor* compressor, Py_ssize_t sourceSize) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	95 size_t zresult;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	96 void* dictData = NULL;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	97 size_t dictSize = 0;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	98 ZSTD_parameters zparams;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	99
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	100 assert(compressor->mtcctx);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	101
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	102 if (compressor->dict) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	103 dictData = compressor->dict->dictData;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	104 dictSize = compressor->dict->dictSize;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	105 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	106
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	107 memset(&zparams, 0, sizeof(zparams));
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	108 if (compressor->cparams) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	109 ztopy_compression_parameters(compressor->cparams, &zparams.cParams);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	110 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	111 else {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	112 zparams.cParams = ZSTD_getCParams(compressor->compressionLevel, sourceSize, dictSize);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	113 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	114
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	115 zparams.fParams = compressor->fparams;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	116
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	117 zresult = ZSTDMT_initCStream_advanced(compressor->mtcctx, dictData, dictSize,
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	118 zparams, sourceSize);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	119
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	120 if (ZSTD_isError(zresult)) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	121 PyErr_Format(ZstdError, "cannot init CStream: %s", ZSTD_getErrorName(zresult));
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	122 return -1;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	123 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	124
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	125 return 0;
30435 b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	126 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	127
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	128 PyDoc_STRVAR(ZstdCompressor__doc__,
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	129 "ZstdCompressor(level=None, dict_data=None, compression_params=None)\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	130 "\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	131 "Create an object used to perform Zstandard compression.\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	132 "\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	133 "An instance can compress data various ways. Instances can be used multiple\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	134 "times. Each compression operation will use the compression parameters\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	135 "defined at construction time.\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	136 "\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	137 "Compression can be configured via the following names arguments:\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	138 "\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	139 "level\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	140 " Integer compression level.\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	141 "dict_data\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	142 " A ``ZstdCompressionDict`` to be used to compress with dictionary data.\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	143 "compression_params\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	144 " A ``CompressionParameters`` instance defining low-level compression"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	145 " parameters. If defined, this will overwrite the ``level`` argument.\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	146 "write_checksum\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	147 " If True, a 4 byte content checksum will be written with the compressed\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	148 " data, allowing the decompressor to perform content verification.\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	149 "write_content_size\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	150 " If True, the decompressed content size will be included in the header of\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	151 " the compressed data. This data will only be written if the compressor\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	152 " knows the size of the input data.\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	153 "write_dict_id\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	154 " Determines whether the dictionary ID will be written into the compressed\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	155 " data. Defaults to True. Only adds content to the compressed data if\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	156 " a dictionary is being used.\n"
31796 e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	157 "threads\n"
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	158 " Number of threads to use to compress data concurrently. When set,\n"
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	159 " compression operations are performed on multiple threads. The default\n"
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	160 " value (0) disables multi-threaded compression. A value of ``-1`` means to\n"
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	161 " set the number of threads to the number of detected logical CPUs.\n"
30435 b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	162 );
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	163
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	164 static int ZstdCompressor_init(ZstdCompressor* self, PyObject* args, PyObject* kwargs) {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	165 static char* kwlist[] = {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	166 "level",
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	167 "dict_data",
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	168 "compression_params",
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	169 "write_checksum",
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	170 "write_content_size",
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	171 "write_dict_id",
31796 e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	172 "threads",
30435 b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	173 NULL
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	174 };
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	175
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	176 int level = 3;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	177 ZstdCompressionDict* dict = NULL;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	178 CompressionParametersObject* params = NULL;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	179 PyObject* writeChecksum = NULL;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	180 PyObject* writeContentSize = NULL;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	181 PyObject* writeDictID = NULL;
31796 e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	182 int threads = 0;
30435 b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	183
31796 e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	184 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "\|iO!O!OOOi:ZstdCompressor",
30895 c32454d69b85 zstd: vendor python-zstandard 0.7.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30830 diff changeset	185 kwlist, &level, &ZstdCompressionDictType, &dict,
30435 b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	186 &CompressionParametersType, &params,
31796 e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	187 &writeChecksum, &writeContentSize, &writeDictID, &threads)) {
30435 b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	188 return -1;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	189 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	190
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	191 if (level < 1) {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	192 PyErr_SetString(PyExc_ValueError, "level must be greater than 0");
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	193 return -1;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	194 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	195
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	196 if (level > ZSTD_maxCLevel()) {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	197 PyErr_Format(PyExc_ValueError, "level must be less than %d",
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	198 ZSTD_maxCLevel() + 1);
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	199 return -1;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	200 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	201
31796 e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	202 if (threads < 0) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	203 threads = cpu_count();
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	204 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	205
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	206 self->threads = threads;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	207
30822 b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	208 /* We create a ZSTD_CCtx for reuse among multiple operations to reduce the
b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	209 overhead of each compression operation. */
31796 e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	210 if (threads) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	211 self->mtcctx = ZSTDMT_createCCtx(threads);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	212 if (!self->mtcctx) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	213 PyErr_NoMemory();
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	214 return -1;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	215 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	216 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	217 else {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	218 self->cctx = ZSTD_createCCtx();
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	219 if (!self->cctx) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	220 PyErr_NoMemory();
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	221 return -1;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	222 }
30822 b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	223 }
b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	224
30435 b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	225 self->compressionLevel = level;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	226
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	227 if (dict) {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	228 self->dict = dict;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	229 Py_INCREF(dict);
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	230 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	231
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	232 if (params) {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	233 self->cparams = params;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	234 Py_INCREF(params);
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	235 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	236
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	237 memset(&self->fparams, 0, sizeof(self->fparams));
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	238
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	239 if (writeChecksum && PyObject_IsTrue(writeChecksum)) {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	240 self->fparams.checksumFlag = 1;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	241 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	242 if (writeContentSize && PyObject_IsTrue(writeContentSize)) {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	243 self->fparams.contentSizeFlag = 1;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	244 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	245 if (writeDictID && PyObject_Not(writeDictID)) {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	246 self->fparams.noDictIDFlag = 1;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	247 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	248
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	249 return 0;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	250 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	251
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	252 static void ZstdCompressor_dealloc(ZstdCompressor* self) {
31796 e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	253 if (self->cstream) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	254 ZSTD_freeCStream(self->cstream);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	255 self->cstream = NULL;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	256 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	257
30435 b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	258 Py_XDECREF(self->cparams);
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	259 Py_XDECREF(self->dict);
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	260
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	261 if (self->cdict) {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	262 ZSTD_freeCDict(self->cdict);
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	263 self->cdict = NULL;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	264 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	265
30822 b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	266 if (self->cctx) {
b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	267 ZSTD_freeCCtx(self->cctx);
b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	268 self->cctx = NULL;
b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	269 }
b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	270
31796 e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	271 if (self->mtcctx) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	272 ZSTDMT_freeCCtx(self->mtcctx);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	273 self->mtcctx = NULL;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	274 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	275
30435 b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	276 PyObject_Del(self);
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	277 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	278
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	279 PyDoc_STRVAR(ZstdCompressor_copy_stream__doc__,
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	280 "copy_stream(ifh, ofh[, size=0, read_size=default, write_size=default])\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	281 "compress data between streams\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	282 "\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	283 "Data will be read from ``ifh``, compressed, and written to ``ofh``.\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	284 "``ifh`` must have a ``read(size)`` method. ``ofh`` must have a ``write(data)``\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	285 "method.\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	286 "\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	287 "An optional ``size`` argument specifies the size of the source stream.\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	288 "If defined, compression parameters will be tuned based on the size.\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	289 "\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	290 "Optional arguments ``read_size`` and ``write_size`` define the chunk sizes\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	291 "of ``read()`` and ``write()`` operations, respectively. By default, they use\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	292 "the default compression stream input and output sizes, respectively.\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	293 );
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	294
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	295 static PyObject* ZstdCompressor_copy_stream(ZstdCompressor* self, PyObject* args, PyObject* kwargs) {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	296 static char* kwlist[] = {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	297 "ifh",
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	298 "ofh",
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	299 "size",
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	300 "read_size",
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	301 "write_size",
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	302 NULL
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	303 };
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	304
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	305 PyObject* source;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	306 PyObject* dest;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	307 Py_ssize_t sourceSize = 0;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	308 size_t inSize = ZSTD_CStreamInSize();
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	309 size_t outSize = ZSTD_CStreamOutSize();
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	310 ZSTD_inBuffer input;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	311 ZSTD_outBuffer output;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	312 Py_ssize_t totalRead = 0;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	313 Py_ssize_t totalWrite = 0;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	314 char* readBuffer;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	315 Py_ssize_t readSize;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	316 PyObject* readResult;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	317 PyObject* res = NULL;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	318 size_t zresult;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	319 PyObject* writeResult;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	320 PyObject* totalReadPy;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	321 PyObject* totalWritePy;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	322
30895 c32454d69b85 zstd: vendor python-zstandard 0.7.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30830 diff changeset	323 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OO\|nkk:copy_stream", kwlist,
c32454d69b85 zstd: vendor python-zstandard 0.7.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30830 diff changeset	324 &source, &dest, &sourceSize, &inSize, &outSize)) {
30435 b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	325 return NULL;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	326 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	327
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	328 if (!PyObject_HasAttrString(source, "read")) {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	329 PyErr_SetString(PyExc_ValueError, "first argument must have a read() method");
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	330 return NULL;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	331 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	332
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	333 if (!PyObject_HasAttrString(dest, "write")) {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	334 PyErr_SetString(PyExc_ValueError, "second argument must have a write() method");
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	335 return NULL;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	336 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	337
30830 08fa3a76a080 zstd: prevent potential free() of uninitialized memory Gregory Szorc <gregory.szorc@gmail.com> parents: 30822 diff changeset	338 /* Prevent free on uninitialized memory in finally. */
08fa3a76a080 zstd: prevent potential free() of uninitialized memory Gregory Szorc <gregory.szorc@gmail.com> parents: 30822 diff changeset	339 output.dst = NULL;
08fa3a76a080 zstd: prevent potential free() of uninitialized memory Gregory Szorc <gregory.szorc@gmail.com> parents: 30822 diff changeset	340
31796 e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	341 if (self->mtcctx) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	342 if (init_mtcstream(self, sourceSize)) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	343 res = NULL;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	344 goto finally;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	345 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	346 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	347 else {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	348 if (0 != init_cstream(self, sourceSize)) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	349 res = NULL;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	350 goto finally;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	351 }
30435 b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	352 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	353
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	354 output.dst = PyMem_Malloc(outSize);
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	355 if (!output.dst) {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	356 PyErr_NoMemory();
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	357 res = NULL;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	358 goto finally;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	359 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	360 output.size = outSize;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	361 output.pos = 0;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	362
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	363 while (1) {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	364 /* Try to read from source stream. */
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	365 readResult = PyObject_CallMethod(source, "read", "n", inSize);
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	366 if (!readResult) {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	367 PyErr_SetString(ZstdError, "could not read() from source");
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	368 goto finally;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	369 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	370
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	371 PyBytes_AsStringAndSize(readResult, &readBuffer, &readSize);
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	372
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	373 /* If no data was read, we're at EOF. */
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	374 if (0 == readSize) {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	375 break;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	376 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	377
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	378 totalRead += readSize;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	379
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	380 /* Send data to compressor */
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	381 input.src = readBuffer;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	382 input.size = readSize;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	383 input.pos = 0;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	384
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	385 while (input.pos < input.size) {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	386 Py_BEGIN_ALLOW_THREADS
31796 e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	387 if (self->mtcctx) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	388 zresult = ZSTDMT_compressStream(self->mtcctx, &output, &input);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	389 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	390 else {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	391 zresult = ZSTD_compressStream(self->cstream, &output, &input);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	392 }
30435 b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	393 Py_END_ALLOW_THREADS
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	394
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	395 if (ZSTD_isError(zresult)) {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	396 res = NULL;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	397 PyErr_Format(ZstdError, "zstd compress error: %s", ZSTD_getErrorName(zresult));
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	398 goto finally;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	399 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	400
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	401 if (output.pos) {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	402 #if PY_MAJOR_VERSION >= 3
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	403 writeResult = PyObject_CallMethod(dest, "write", "y#",
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	404 #else
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	405 writeResult = PyObject_CallMethod(dest, "write", "s#",
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	406 #endif
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	407 output.dst, output.pos);
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	408 Py_XDECREF(writeResult);
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	409 totalWrite += output.pos;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	410 output.pos = 0;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	411 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	412 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	413 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	414
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	415 /* We've finished reading. Now flush the compressor stream. */
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	416 while (1) {
31796 e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	417 if (self->mtcctx) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	418 zresult = ZSTDMT_endStream(self->mtcctx, &output);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	419 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	420 else {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	421 zresult = ZSTD_endStream(self->cstream, &output);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	422 }
30435 b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	423 if (ZSTD_isError(zresult)) {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	424 PyErr_Format(ZstdError, "error ending compression stream: %s",
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	425 ZSTD_getErrorName(zresult));
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	426 res = NULL;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	427 goto finally;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	428 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	429
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	430 if (output.pos) {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	431 #if PY_MAJOR_VERSION >= 3
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	432 writeResult = PyObject_CallMethod(dest, "write", "y#",
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	433 #else
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	434 writeResult = PyObject_CallMethod(dest, "write", "s#",
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	435 #endif
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	436 output.dst, output.pos);
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	437 totalWrite += output.pos;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	438 Py_XDECREF(writeResult);
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	439 output.pos = 0;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	440 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	441
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	442 if (!zresult) {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	443 break;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	444 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	445 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	446
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	447 totalReadPy = PyLong_FromSsize_t(totalRead);
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	448 totalWritePy = PyLong_FromSsize_t(totalWrite);
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	449 res = PyTuple_Pack(2, totalReadPy, totalWritePy);
31796 e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	450 Py_DECREF(totalReadPy);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	451 Py_DECREF(totalWritePy);
30435 b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	452
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	453 finally:
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	454 if (output.dst) {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	455 PyMem_Free(output.dst);
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	456 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	457
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	458 return res;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	459 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	460
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	461 PyDoc_STRVAR(ZstdCompressor_compress__doc__,
30822 b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	462 "compress(data, allow_empty=False)\n"
30435 b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	463 "\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	464 "Compress data in a single operation.\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	465 "\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	466 "This is the simplest mechanism to perform compression: simply pass in a\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	467 "value and get a compressed value back. It is almost the most prone to abuse.\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	468 "The input and output values must fit in memory, so passing in very large\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	469 "values can result in excessive memory usage. For this reason, one of the\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	470 "streaming based APIs is preferred for larger values.\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	471 );
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	472
30822 b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	473 static PyObject* ZstdCompressor_compress(ZstdCompressor* self, PyObject* args, PyObject* kwargs) {
b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	474 static char* kwlist[] = {
b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	475 "data",
b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	476 "allow_empty",
b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	477 NULL
b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	478 };
b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	479
30435 b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	480 const char* source;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	481 Py_ssize_t sourceSize;
30822 b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	482 PyObject* allowEmpty = NULL;
30435 b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	483 size_t destSize;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	484 PyObject* output;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	485 char* dest;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	486 void* dictData = NULL;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	487 size_t dictSize = 0;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	488 size_t zresult;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	489 ZSTD_parameters zparams;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	490
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	491 #if PY_MAJOR_VERSION >= 3
30895 c32454d69b85 zstd: vendor python-zstandard 0.7.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30830 diff changeset	492 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y#\|O:compress",
30435 b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	493 #else
30895 c32454d69b85 zstd: vendor python-zstandard 0.7.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30830 diff changeset	494 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s#\|O:compress",
30435 b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	495 #endif
30822 b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	496 kwlist, &source, &sourceSize, &allowEmpty)) {
b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	497 return NULL;
b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	498 }
b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	499
31796 e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	500 if (self->threads && self->dict) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	501 PyErr_SetString(ZstdError,
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	502 "compress() cannot be used with both dictionaries and multi-threaded compression");
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	503 return NULL;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	504 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	505
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	506 if (self->threads && self->cparams) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	507 PyErr_SetString(ZstdError,
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	508 "compress() cannot be used with both compression parameters and multi-threaded compression");
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	509 return NULL;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	510 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	511
30822 b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	512 /* Limitation in zstd C API doesn't let decompression side distinguish
b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	513 between content size of 0 and unknown content size. This can make round
b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	514 tripping via Python difficult. Until this is fixed, require a flag
b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	515 to fire the footgun.
b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	516 https://github.com/indygreg/python-zstandard/issues/11 */
b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	517 if (0 == sourceSize && self->fparams.contentSizeFlag
b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	518 && (!allowEmpty \|\| PyObject_Not(allowEmpty))) {
b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	519 PyErr_SetString(PyExc_ValueError, "cannot write empty inputs when writing content sizes");
30435 b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	520 return NULL;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	521 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	522
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	523 destSize = ZSTD_compressBound(sourceSize);
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	524 output = PyBytes_FromStringAndSize(NULL, destSize);
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	525 if (!output) {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	526 return NULL;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	527 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	528
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	529 dest = PyBytes_AsString(output);
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	530
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	531 if (self->dict) {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	532 dictData = self->dict->dictData;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	533 dictSize = self->dict->dictSize;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	534 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	535
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	536 memset(&zparams, 0, sizeof(zparams));
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	537 if (!self->cparams) {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	538 zparams.cParams = ZSTD_getCParams(self->compressionLevel, sourceSize, dictSize);
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	539 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	540 else {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	541 ztopy_compression_parameters(self->cparams, &zparams.cParams);
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	542 /* Do NOT call ZSTD_adjustCParams() here because the compression params
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	543 come from the user. */
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	544 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	545
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	546 zparams.fParams = self->fparams;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	547
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	548 /* The raw dict data has to be processed before it can be used. Since this
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	549 adds overhead - especially if multiple dictionary compression operations
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	550 are performed on the same ZstdCompressor instance - we create a
30822 b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	551 ZSTD_CDict once and reuse it for all operations.
30435 b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	552
30822 b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	553 Note: the compression parameters used for the first invocation (possibly
b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	554 derived from the source size) will be reused on all subsequent invocations.
b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	555 https://github.com/facebook/zstd/issues/358 contains more info. We could
b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	556 potentially add an argument somewhere to control this behavior.
b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	557 */
31796 e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	558 if (0 != populate_cdict(self, &zparams)) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	559 Py_DECREF(output);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	560 return NULL;
30435 b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	561 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	562
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	563 Py_BEGIN_ALLOW_THREADS
31796 e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	564 if (self->mtcctx) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	565 zresult = ZSTDMT_compressCCtx(self->mtcctx, dest, destSize,
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	566 source, sourceSize, self->compressionLevel);
30435 b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	567 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	568 else {
31796 e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	569 /* By avoiding ZSTD_compress(), we don't necessarily write out content
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	570 size. This means the argument to ZstdCompressor to control frame
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	571 parameters is honored. */
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	572 if (self->cdict) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	573 zresult = ZSTD_compress_usingCDict(self->cctx, dest, destSize,
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	574 source, sourceSize, self->cdict);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	575 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	576 else {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	577 zresult = ZSTD_compress_advanced(self->cctx, dest, destSize,
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	578 source, sourceSize, dictData, dictSize, zparams);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	579 }
30435 b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	580 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	581 Py_END_ALLOW_THREADS
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	582
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	583 if (ZSTD_isError(zresult)) {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	584 PyErr_Format(ZstdError, "cannot compress: %s", ZSTD_getErrorName(zresult));
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	585 Py_CLEAR(output);
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	586 return NULL;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	587 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	588 else {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	589 Py_SIZE(output) = zresult;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	590 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	591
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	592 return output;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	593 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	594
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	595 PyDoc_STRVAR(ZstdCompressionObj__doc__,
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	596 "compressobj()\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	597 "\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	598 "Return an object exposing ``compress(data)`` and ``flush()`` methods.\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	599 "\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	600 "The returned object exposes an API similar to ``zlib.compressobj`` and\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	601 "``bz2.BZ2Compressor`` so that callers can swap in the zstd compressor\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	602 "without changing how compression is performed.\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	603 );
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	604
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	605 static ZstdCompressionObj* ZstdCompressor_compressobj(ZstdCompressor* self, PyObject* args, PyObject* kwargs) {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	606 static char* kwlist[] = {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	607 "size",
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	608 NULL
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	609 };
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	610
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	611 Py_ssize_t inSize = 0;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	612 size_t outSize = ZSTD_CStreamOutSize();
31796 e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	613 ZstdCompressionObj* result = NULL;
30435 b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	614
30895 c32454d69b85 zstd: vendor python-zstandard 0.7.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30830 diff changeset	615 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "\|n:compressobj", kwlist, &inSize)) {
30435 b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	616 return NULL;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	617 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	618
31796 e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	619 result = (ZstdCompressionObj)PyObject_CallObject((PyObject)&ZstdCompressionObjType, NULL);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	620 if (!result) {
30435 b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	621 return NULL;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	622 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	623
31796 e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	624 if (self->mtcctx) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	625 if (init_mtcstream(self, inSize)) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	626 Py_DECREF(result);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	627 return NULL;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	628 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	629 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	630 else {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	631 if (0 != init_cstream(self, inSize)) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	632 Py_DECREF(result);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	633 return NULL;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	634 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	635 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	636
30435 b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	637 result->output.dst = PyMem_Malloc(outSize);
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	638 if (!result->output.dst) {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	639 PyErr_NoMemory();
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	640 Py_DECREF(result);
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	641 return NULL;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	642 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	643 result->output.size = outSize;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	644 result->compressor = self;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	645 Py_INCREF(result->compressor);
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	646
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	647 return result;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	648 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	649
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	650 PyDoc_STRVAR(ZstdCompressor_read_from__doc__,
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	651 "read_from(reader, [size=0, read_size=default, write_size=default])\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	652 "Read uncompress data from a reader and return an iterator\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	653 "\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	654 "Returns an iterator of compressed data produced from reading from ``reader``.\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	655 "\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	656 "Uncompressed data will be obtained from ``reader`` by calling the\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	657 "``read(size)`` method of it. The source data will be streamed into a\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	658 "compressor. As compressed data is available, it will be exposed to the\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	659 "iterator.\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	660 "\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	661 "Data is read from the source in chunks of ``read_size``. Compressed chunks\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	662 "are at most ``write_size`` bytes. Both values default to the zstd input and\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	663 "and output defaults, respectively.\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	664 "\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	665 "The caller is partially in control of how fast data is fed into the\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	666 "compressor by how it consumes the returned iterator. The compressor will\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	667 "not consume from the reader unless the caller consumes from the iterator.\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	668 );
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	669
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	670 static ZstdCompressorIterator* ZstdCompressor_read_from(ZstdCompressor* self, PyObject* args, PyObject* kwargs) {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	671 static char* kwlist[] = {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	672 "reader",
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	673 "size",
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	674 "read_size",
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	675 "write_size",
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	676 NULL
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	677 };
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	678
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	679 PyObject* reader;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	680 Py_ssize_t sourceSize = 0;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	681 size_t inSize = ZSTD_CStreamInSize();
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	682 size_t outSize = ZSTD_CStreamOutSize();
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	683 ZstdCompressorIterator* result;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	684
30895 c32454d69b85 zstd: vendor python-zstandard 0.7.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30830 diff changeset	685 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O\|nkk:read_from", kwlist,
c32454d69b85 zstd: vendor python-zstandard 0.7.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30830 diff changeset	686 &reader, &sourceSize, &inSize, &outSize)) {
30435 b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	687 return NULL;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	688 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	689
31796 e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	690 result = (ZstdCompressorIterator)PyObject_CallObject((PyObject)&ZstdCompressorIteratorType, NULL);
30435 b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	691 if (!result) {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	692 return NULL;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	693 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	694 if (PyObject_HasAttrString(reader, "read")) {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	695 result->reader = reader;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	696 Py_INCREF(result->reader);
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	697 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	698 else if (1 == PyObject_CheckBuffer(reader)) {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	699 result->buffer = PyMem_Malloc(sizeof(Py_buffer));
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	700 if (!result->buffer) {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	701 goto except;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	702 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	703
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	704 memset(result->buffer, 0, sizeof(Py_buffer));
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	705
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	706 if (0 != PyObject_GetBuffer(reader, result->buffer, PyBUF_CONTIG_RO)) {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	707 goto except;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	708 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	709
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	710 sourceSize = result->buffer->len;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	711 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	712 else {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	713 PyErr_SetString(PyExc_ValueError,
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	714 "must pass an object with a read() method or conforms to buffer protocol");
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	715 goto except;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	716 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	717
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	718 result->compressor = self;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	719 Py_INCREF(result->compressor);
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	720
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	721 result->sourceSize = sourceSize;
31796 e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	722
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	723 if (self->mtcctx) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	724 if (init_mtcstream(self, sourceSize)) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	725 goto except;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	726 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	727 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	728 else {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	729 if (0 != init_cstream(self, sourceSize)) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	730 goto except;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	731 }
30435 b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	732 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	733
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	734 result->inSize = inSize;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	735 result->outSize = outSize;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	736
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	737 result->output.dst = PyMem_Malloc(outSize);
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	738 if (!result->output.dst) {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	739 PyErr_NoMemory();
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	740 goto except;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	741 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	742 result->output.size = outSize;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	743
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	744 goto finally;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	745
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	746 except:
31796 e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	747 Py_XDECREF(result->compressor);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	748 Py_XDECREF(result->reader);
30435 b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	749 Py_DECREF(result);
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	750 result = NULL;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	751
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	752 finally:
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	753 return result;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	754 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	755
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	756 PyDoc_STRVAR(ZstdCompressor_write_to___doc__,
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	757 "Create a context manager to write compressed data to an object.\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	758 "\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	759 "The passed object must have a ``write()`` method.\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	760 "\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	761 "The caller feeds input data to the object by calling ``compress(data)``.\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	762 "Compressed data is written to the argument given to this function.\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	763 "\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	764 "The function takes an optional ``size`` argument indicating the total size\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	765 "of the eventual input. If specified, the size will influence compression\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	766 "parameter tuning and could result in the size being written into the\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	767 "header of the compressed data.\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	768 "\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	769 "An optional ``write_size`` argument is also accepted. It defines the maximum\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	770 "byte size of chunks fed to ``write()``. By default, it uses the zstd default\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	771 "for a compressor output stream.\n"
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	772 );
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	773
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	774 static ZstdCompressionWriter* ZstdCompressor_write_to(ZstdCompressor* self, PyObject* args, PyObject* kwargs) {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	775 static char* kwlist[] = {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	776 "writer",
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	777 "size",
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	778 "write_size",
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	779 NULL
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	780 };
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	781
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	782 PyObject* writer;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	783 ZstdCompressionWriter* result;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	784 Py_ssize_t sourceSize = 0;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	785 size_t outSize = ZSTD_CStreamOutSize();
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	786
30895 c32454d69b85 zstd: vendor python-zstandard 0.7.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30830 diff changeset	787 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O\|nk:write_to", kwlist,
c32454d69b85 zstd: vendor python-zstandard 0.7.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30830 diff changeset	788 &writer, &sourceSize, &outSize)) {
30435 b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	789 return NULL;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	790 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	791
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	792 if (!PyObject_HasAttrString(writer, "write")) {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	793 PyErr_SetString(PyExc_ValueError, "must pass an object with a write() method");
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	794 return NULL;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	795 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	796
31796 e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	797 result = (ZstdCompressionWriter)PyObject_CallObject((PyObject)&ZstdCompressionWriterType, NULL);
30435 b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	798 if (!result) {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	799 return NULL;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	800 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	801
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	802 result->compressor = self;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	803 Py_INCREF(result->compressor);
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	804
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	805 result->writer = writer;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	806 Py_INCREF(result->writer);
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	807
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	808 result->sourceSize = sourceSize;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	809 result->outSize = outSize;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	810
31796 e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	811 return result;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	812 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	813
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	814 typedef struct {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	815 void* sourceData;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	816 size_t sourceSize;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	817 } DataSource;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	818
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	819 typedef struct {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	820 DataSource* sources;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	821 Py_ssize_t sourcesSize;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	822 unsigned long long totalSourceSize;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	823 } DataSources;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	824
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	825 typedef struct {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	826 void* dest;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	827 Py_ssize_t destSize;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	828 BufferSegment* segments;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	829 Py_ssize_t segmentsSize;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	830 } DestBuffer;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	831
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	832 typedef enum {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	833 WorkerError_none = 0,
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	834 WorkerError_zstd = 1,
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	835 WorkerError_no_memory = 2,
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	836 } WorkerError;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	837
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	838 /**
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	839 * Holds state for an individual worker performing multi_compress_to_buffer work.
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	840 */
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	841 typedef struct {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	842 /* Used for compression. */
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	843 ZSTD_CCtx* cctx;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	844 ZSTD_CDict* cdict;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	845 int cLevel;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	846 CompressionParametersObject* cParams;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	847 ZSTD_frameParameters fParams;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	848
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	849 /* What to compress. */
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	850 DataSource* sources;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	851 Py_ssize_t sourcesSize;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	852 Py_ssize_t startOffset;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	853 Py_ssize_t endOffset;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	854 unsigned long long totalSourceSize;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	855
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	856 /* Result storage. */
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	857 DestBuffer* destBuffers;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	858 Py_ssize_t destCount;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	859
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	860 /* Error tracking. */
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	861 WorkerError error;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	862 size_t zresult;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	863 Py_ssize_t errorOffset;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	864 } WorkerState;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	865
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	866 static void compress_worker(WorkerState* state) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	867 Py_ssize_t inputOffset = state->startOffset;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	868 Py_ssize_t remainingItems = state->endOffset - state->startOffset + 1;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	869 Py_ssize_t currentBufferStartOffset = state->startOffset;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	870 size_t zresult;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	871 ZSTD_parameters zparams;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	872 void* newDest;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	873 size_t allocationSize;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	874 size_t boundSize;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	875 Py_ssize_t destOffset = 0;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	876 DataSource* sources = state->sources;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	877 DestBuffer* destBuffer;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	878
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	879 assert(!state->destBuffers);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	880 assert(0 == state->destCount);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	881
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	882 if (state->cParams) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	883 ztopy_compression_parameters(state->cParams, &zparams.cParams);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	884 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	885
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	886 zparams.fParams = state->fParams;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	887
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	888 /*
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	889 * The total size of the compressed data is unknown until we actually
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	890 * compress data. That means we can't pre-allocate the exact size we need.
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	891 *
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	892 * There is a cost to every allocation and reallocation. So, it is in our
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	893 * interest to minimize the number of allocations.
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	894 *
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	895 * There is also a cost to too few allocations. If allocations are too
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	896 * large they may fail. If buffers are shared and all inputs become
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	897 * irrelevant at different lifetimes, then a reference to one segment
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	898 * in the buffer will keep the entire buffer alive. This leads to excessive
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	899 * memory usage.
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	900 *
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	901 * Our current strategy is to assume a compression ratio of 16:1 and
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	902 * allocate buffers of that size, rounded up to the nearest power of 2
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	903 * (because computers like round numbers). That ratio is greater than what
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	904 * most inputs achieve. This is by design: we don't want to over-allocate.
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	905 * But we don't want to under-allocate and lead to too many buffers either.
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	906 */
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	907
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	908 state->destCount = 1;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	909
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	910 state->destBuffers = calloc(1, sizeof(DestBuffer));
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	911 if (NULL == state->destBuffers) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	912 state->error = WorkerError_no_memory;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	913 return;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	914 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	915
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	916 destBuffer = &state->destBuffers[state->destCount - 1];
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	917
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	918 /*
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	919 * Rather than track bounds and grow the segments buffer, allocate space
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	920 * to hold remaining items then truncate when we're done with it.
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	921 */
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	922 destBuffer->segments = calloc(remainingItems, sizeof(BufferSegment));
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	923 if (NULL == destBuffer->segments) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	924 state->error = WorkerError_no_memory;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	925 return;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	926 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	927
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	928 destBuffer->segmentsSize = remainingItems;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	929
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	930 allocationSize = roundpow2(state->totalSourceSize >> 4);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	931
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	932 /* If the maximum size of the output is larger than that, round up. */
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	933 boundSize = ZSTD_compressBound(sources[inputOffset].sourceSize);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	934
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	935 if (boundSize > allocationSize) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	936 allocationSize = roundpow2(boundSize);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	937 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	938
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	939 destBuffer->dest = malloc(allocationSize);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	940 if (NULL == destBuffer->dest) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	941 state->error = WorkerError_no_memory;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	942 return;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	943 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	944
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	945 destBuffer->destSize = allocationSize;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	946
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	947 for (inputOffset = state->startOffset; inputOffset <= state->endOffset; inputOffset++) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	948 void* source = sources[inputOffset].sourceData;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	949 size_t sourceSize = sources[inputOffset].sourceSize;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	950 size_t destAvailable;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	951 void* dest;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	952
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	953 destAvailable = destBuffer->destSize - destOffset;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	954 boundSize = ZSTD_compressBound(sourceSize);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	955
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	956 /*
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	957 * Not enough space in current buffer to hold largest compressed output.
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	958 * So allocate and switch to a new output buffer.
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	959 */
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	960 if (boundSize > destAvailable) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	961 /*
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	962 * The downsizing of the existing buffer is optional. It should be cheap
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	963 * (unlike growing). So we just do it.
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	964 */
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	965 if (destAvailable) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	966 newDest = realloc(destBuffer->dest, destOffset);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	967 if (NULL == newDest) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	968 state->error = WorkerError_no_memory;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	969 return;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	970 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	971
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	972 destBuffer->dest = newDest;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	973 destBuffer->destSize = destOffset;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	974 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	975
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	976 /* Truncate segments buffer. */
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	977 newDest = realloc(destBuffer->segments,
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	978 (inputOffset - currentBufferStartOffset + 1) * sizeof(BufferSegment));
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	979 if (NULL == newDest) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	980 state->error = WorkerError_no_memory;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	981 return;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	982 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	983
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	984 destBuffer->segments = newDest;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	985 destBuffer->segmentsSize = inputOffset - currentBufferStartOffset;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	986
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	987 /* Grow space for new struct. */
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	988 /* TODO consider over-allocating so we don't do this every time. */
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	989 newDest = realloc(state->destBuffers, (state->destCount + 1) * sizeof(DestBuffer));
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	990 if (NULL == newDest) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	991 state->error = WorkerError_no_memory;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	992 return;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	993 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	994
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	995 state->destBuffers = newDest;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	996 state->destCount++;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	997
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	998 destBuffer = &state->destBuffers[state->destCount - 1];
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	999
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1000 /* Don't take any chances with non-NULL pointers. */
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1001 memset(destBuffer, 0, sizeof(DestBuffer));
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1002
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1003 /**
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1004 * We could dynamically update allocation size based on work done so far.
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1005 * For now, keep is simple.
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1006 */
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1007 allocationSize = roundpow2(state->totalSourceSize >> 4);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1008
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1009 if (boundSize > allocationSize) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1010 allocationSize = roundpow2(boundSize);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1011 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1012
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1013 destBuffer->dest = malloc(allocationSize);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1014 if (NULL == destBuffer->dest) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1015 state->error = WorkerError_no_memory;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1016 return;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1017 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1018
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1019 destBuffer->destSize = allocationSize;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1020 destAvailable = allocationSize;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1021 destOffset = 0;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1022
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1023 destBuffer->segments = calloc(remainingItems, sizeof(BufferSegment));
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1024 if (NULL == destBuffer->segments) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1025 state->error = WorkerError_no_memory;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1026 return;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1027 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1028
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1029 destBuffer->segmentsSize = remainingItems;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1030 currentBufferStartOffset = inputOffset;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1031 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1032
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1033 dest = (char*)destBuffer->dest + destOffset;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1034
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1035 if (state->cdict) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1036 zresult = ZSTD_compress_usingCDict(state->cctx, dest, destAvailable,
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1037 source, sourceSize, state->cdict);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1038 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1039 else {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1040 if (!state->cParams) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1041 zparams.cParams = ZSTD_getCParams(state->cLevel, sourceSize, 0);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1042 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1043
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1044 zresult = ZSTD_compress_advanced(state->cctx, dest, destAvailable,
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1045 source, sourceSize, NULL, 0, zparams);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1046 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1047
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1048 if (ZSTD_isError(zresult)) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1049 state->error = WorkerError_zstd;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1050 state->zresult = zresult;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1051 state->errorOffset = inputOffset;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1052 break;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1053 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1054
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1055 destBuffer->segments[inputOffset - currentBufferStartOffset].offset = destOffset;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1056 destBuffer->segments[inputOffset - currentBufferStartOffset].length = zresult;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1057
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1058 destOffset += zresult;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1059 remainingItems--;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1060 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1061
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1062 if (destBuffer->destSize > destOffset) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1063 newDest = realloc(destBuffer->dest, destOffset);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1064 if (NULL == newDest) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1065 state->error = WorkerError_no_memory;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1066 return;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1067 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1068
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1069 destBuffer->dest = newDest;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1070 destBuffer->destSize = destOffset;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1071 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1072 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1073
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1074 ZstdBufferWithSegmentsCollection* compress_from_datasources(ZstdCompressor* compressor,
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1075 DataSources* sources, unsigned int threadCount) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1076 ZSTD_parameters zparams;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1077 unsigned long long bytesPerWorker;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1078 POOL_ctx* pool = NULL;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1079 WorkerState* workerStates = NULL;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1080 Py_ssize_t i;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1081 unsigned long long workerBytes = 0;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1082 Py_ssize_t workerStartOffset = 0;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1083 size_t currentThread = 0;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1084 int errored = 0;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1085 Py_ssize_t segmentsCount = 0;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1086 Py_ssize_t segmentIndex;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1087 PyObject* segmentsArg = NULL;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1088 ZstdBufferWithSegments* buffer;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1089 ZstdBufferWithSegmentsCollection* result = NULL;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1090
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1091 assert(sources->sourcesSize > 0);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1092 assert(sources->totalSourceSize > 0);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1093 assert(threadCount >= 1);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1094
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1095 /* More threads than inputs makes no sense. */
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1096 threadCount = sources->sourcesSize < threadCount ? (unsigned int)sources->sourcesSize
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1097 : threadCount;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1098
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1099 /* TODO lower thread count when input size is too small and threads would add
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1100 overhead. */
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1101
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1102 /*
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1103 * When dictionaries are used, parameters are derived from the size of the
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1104 * first element.
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1105 *
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1106 * TODO come up with a better mechanism.
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1107 */
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1108 memset(&zparams, 0, sizeof(zparams));
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1109 if (compressor->cparams) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1110 ztopy_compression_parameters(compressor->cparams, &zparams.cParams);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1111 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1112 else {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1113 zparams.cParams = ZSTD_getCParams(compressor->compressionLevel,
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1114 sources->sources[0].sourceSize,
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1115 compressor->dict ? compressor->dict->dictSize : 0);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1116 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1117
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1118 zparams.fParams = compressor->fparams;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1119
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1120 if (0 != populate_cdict(compressor, &zparams)) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1121 return NULL;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1122 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1123
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1124 workerStates = PyMem_Malloc(threadCount * sizeof(WorkerState));
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1125 if (NULL == workerStates) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1126 PyErr_NoMemory();
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1127 goto finally;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1128 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1129
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1130 memset(workerStates, 0, threadCount * sizeof(WorkerState));
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1131
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1132 if (threadCount > 1) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1133 pool = POOL_create(threadCount, 1);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1134 if (NULL == pool) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1135 PyErr_SetString(ZstdError, "could not initialize zstd thread pool");
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1136 goto finally;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1137 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1138 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1139
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1140 bytesPerWorker = sources->totalSourceSize / threadCount;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1141
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1142 for (i = 0; i < threadCount; i++) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1143 workerStates[i].cctx = ZSTD_createCCtx();
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1144 if (!workerStates[i].cctx) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1145 PyErr_NoMemory();
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1146 goto finally;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1147 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1148
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1149 workerStates[i].cdict = compressor->cdict;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1150 workerStates[i].cLevel = compressor->compressionLevel;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1151 workerStates[i].cParams = compressor->cparams;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1152 workerStates[i].fParams = compressor->fparams;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1153
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1154 workerStates[i].sources = sources->sources;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1155 workerStates[i].sourcesSize = sources->sourcesSize;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1156 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1157
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1158 Py_BEGIN_ALLOW_THREADS
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1159 for (i = 0; i < sources->sourcesSize; i++) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1160 workerBytes += sources->sources[i].sourceSize;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1161
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1162 /*
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1163 * The last worker/thread needs to handle all remaining work. Don't
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1164 * trigger it prematurely. Defer to the block outside of the loop
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1165 * to run the last worker/thread. But do still process this loop
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1166 * so workerBytes is correct.
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1167 */
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1168 if (currentThread == threadCount - 1) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1169 continue;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1170 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1171
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1172 if (workerBytes >= bytesPerWorker) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1173 assert(currentThread < threadCount);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1174 workerStates[currentThread].totalSourceSize = workerBytes;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1175 workerStates[currentThread].startOffset = workerStartOffset;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1176 workerStates[currentThread].endOffset = i;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1177
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1178 if (threadCount > 1) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1179 POOL_add(pool, (POOL_function)compress_worker, &workerStates[currentThread]);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1180 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1181 else {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1182 compress_worker(&workerStates[currentThread]);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1183 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1184
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1185 currentThread++;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1186 workerStartOffset = i + 1;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1187 workerBytes = 0;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1188 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1189 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1190
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1191 if (workerBytes) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1192 assert(currentThread < threadCount);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1193 workerStates[currentThread].totalSourceSize = workerBytes;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1194 workerStates[currentThread].startOffset = workerStartOffset;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1195 workerStates[currentThread].endOffset = sources->sourcesSize - 1;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1196
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1197 if (threadCount > 1) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1198 POOL_add(pool, (POOL_function)compress_worker, &workerStates[currentThread]);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1199 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1200 else {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1201 compress_worker(&workerStates[currentThread]);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1202 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1203 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1204
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1205 if (threadCount > 1) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1206 POOL_free(pool);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1207 pool = NULL;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1208 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1209
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1210 Py_END_ALLOW_THREADS
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1211
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1212 for (i = 0; i < threadCount; i++) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1213 switch (workerStates[i].error) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1214 case WorkerError_no_memory:
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1215 PyErr_NoMemory();
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1216 errored = 1;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1217 break;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1218
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1219 case WorkerError_zstd:
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1220 PyErr_Format(ZstdError, "error compressing item %zd: %s",
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1221 workerStates[i].errorOffset, ZSTD_getErrorName(workerStates[i].zresult));
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1222 errored = 1;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1223 break;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1224 default:
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1225 ;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1226 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1227
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1228 if (errored) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1229 break;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1230 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1231
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1232 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1233
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1234 if (errored) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1235 goto finally;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1236 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1237
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1238 segmentsCount = 0;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1239 for (i = 0; i < threadCount; i++) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1240 WorkerState* state = &workerStates[i];
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1241 segmentsCount += state->destCount;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1242 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1243
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1244 segmentsArg = PyTuple_New(segmentsCount);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1245 if (NULL == segmentsArg) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1246 goto finally;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1247 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1248
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1249 segmentIndex = 0;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1250
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1251 for (i = 0; i < threadCount; i++) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1252 Py_ssize_t j;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1253 WorkerState* state = &workerStates[i];
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1254
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1255 for (j = 0; j < state->destCount; j++) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1256 DestBuffer* destBuffer = &state->destBuffers[j];
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1257 buffer = BufferWithSegments_FromMemory(destBuffer->dest, destBuffer->destSize,
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1258 destBuffer->segments, destBuffer->segmentsSize);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1259
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1260 if (NULL == buffer) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1261 goto finally;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1262 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1263
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1264 /* Tell instance to use free() instsead of PyMem_Free(). */
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1265 buffer->useFree = 1;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1266
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1267 /*
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1268 * BufferWithSegments_FromMemory takes ownership of the backing memory.
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1269 * Unset it here so it doesn't get freed below.
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1270 */
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1271 destBuffer->dest = NULL;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1272 destBuffer->segments = NULL;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1273
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1274 PyTuple_SET_ITEM(segmentsArg, segmentIndex++, (PyObject*)buffer);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1275 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1276 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1277
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1278 result = (ZstdBufferWithSegmentsCollection*)PyObject_CallObject(
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1279 (PyObject*)&ZstdBufferWithSegmentsCollectionType, segmentsArg);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1280
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1281 finally:
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1282 Py_CLEAR(segmentsArg);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1283
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1284 if (pool) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1285 POOL_free(pool);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1286 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1287
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1288 if (workerStates) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1289 Py_ssize_t j;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1290
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1291 for (i = 0; i < threadCount; i++) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1292 WorkerState state = workerStates[i];
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1293
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1294 if (state.cctx) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1295 ZSTD_freeCCtx(state.cctx);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1296 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1297
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1298 /* malloc() is used in worker thread. */
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1299
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1300 for (j = 0; j < state.destCount; j++) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1301 if (state.destBuffers) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1302 free(state.destBuffers[j].dest);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1303 free(state.destBuffers[j].segments);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1304 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1305 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1306
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1307
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1308 free(state.destBuffers);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1309 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1310
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1311 PyMem_Free(workerStates);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1312 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1313
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1314 return result;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1315 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1316
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1317 PyDoc_STRVAR(ZstdCompressor_multi_compress_to_buffer__doc__,
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1318 "Compress multiple pieces of data as a single operation\n"
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1319 "\n"
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1320 "Receives a ``BufferWithSegmentsCollection``, a ``BufferWithSegments``, or\n"
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1321 "a list of bytes like objects holding data to compress.\n"
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1322 "\n"
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1323 "Returns a ``BufferWithSegmentsCollection`` holding compressed data.\n"
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1324 "\n"
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1325 "This function is optimized to perform multiple compression operations as\n"
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1326 "as possible with as little overhead as possbile.\n"
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1327 );
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1328
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1329 static ZstdBufferWithSegmentsCollection* ZstdCompressor_multi_compress_to_buffer(ZstdCompressor* self, PyObject* args, PyObject* kwargs) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1330 static char* kwlist[] = {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1331 "data",
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1332 "threads",
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1333 NULL
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1334 };
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1335
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1336 PyObject* data;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1337 int threads = 0;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1338 Py_buffer* dataBuffers = NULL;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1339 DataSources sources;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1340 Py_ssize_t i;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1341 Py_ssize_t sourceCount = 0;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1342 ZstdBufferWithSegmentsCollection* result = NULL;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1343
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1344 if (self->mtcctx) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1345 PyErr_SetString(ZstdError,
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1346 "function cannot be called on ZstdCompressor configured for multi-threaded compression");
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1347 return NULL;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1348 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1349
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1350 memset(&sources, 0, sizeof(sources));
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1351
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1352 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O\|i:multi_compress_to_buffer", kwlist,
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1353 &data, &threads)) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1354 return NULL;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1355 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1356
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1357 if (threads < 0) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1358 threads = cpu_count();
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1359 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1360
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1361 if (threads < 2) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1362 threads = 1;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1363 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1364
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1365 if (PyObject_TypeCheck(data, &ZstdBufferWithSegmentsType)) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1366 ZstdBufferWithSegments* buffer = (ZstdBufferWithSegments*)data;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1367
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1368 sources.sources = PyMem_Malloc(buffer->segmentCount * sizeof(DataSource));
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1369 if (NULL == sources.sources) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1370 PyErr_NoMemory();
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1371 goto finally;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1372 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1373
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1374 for (i = 0; i < buffer->segmentCount; i++) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1375 sources.sources[i].sourceData = (char*)buffer->data + buffer->segments[i].offset;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1376 sources.sources[i].sourceSize = buffer->segments[i].length;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1377 sources.totalSourceSize += buffer->segments[i].length;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1378 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1379
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1380 sources.sourcesSize = buffer->segmentCount;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1381 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1382 else if (PyObject_TypeCheck(data, &ZstdBufferWithSegmentsCollectionType)) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1383 Py_ssize_t j;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1384 Py_ssize_t offset = 0;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1385 ZstdBufferWithSegments* buffer;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1386 ZstdBufferWithSegmentsCollection* collection = (ZstdBufferWithSegmentsCollection*)data;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1387
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1388 sourceCount = BufferWithSegmentsCollection_length(collection);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1389
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1390 sources.sources = PyMem_Malloc(sourceCount * sizeof(DataSource));
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1391 if (NULL == sources.sources) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1392 PyErr_NoMemory();
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1393 goto finally;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1394 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1395
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1396 for (i = 0; i < collection->bufferCount; i++) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1397 buffer = collection->buffers[i];
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1398
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1399 for (j = 0; j < buffer->segmentCount; j++) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1400 sources.sources[offset].sourceData = (char*)buffer->data + buffer->segments[j].offset;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1401 sources.sources[offset].sourceSize = buffer->segments[j].length;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1402 sources.totalSourceSize += buffer->segments[j].length;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1403
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1404 offset++;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1405 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1406 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1407
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1408 sources.sourcesSize = sourceCount;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1409 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1410 else if (PyList_Check(data)) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1411 sourceCount = PyList_GET_SIZE(data);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1412
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1413 sources.sources = PyMem_Malloc(sourceCount * sizeof(DataSource));
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1414 if (NULL == sources.sources) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1415 PyErr_NoMemory();
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1416 goto finally;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1417 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1418
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1419 /*
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1420 * It isn't clear whether the address referred to by Py_buffer.buf
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1421 * is still valid after PyBuffer_Release. We we hold a reference to all
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1422 * Py_buffer instances for the duration of the operation.
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1423 */
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1424 dataBuffers = PyMem_Malloc(sourceCount * sizeof(Py_buffer));
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1425 if (NULL == dataBuffers) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1426 PyErr_NoMemory();
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1427 goto finally;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1428 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1429
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1430 memset(dataBuffers, 0, sourceCount * sizeof(Py_buffer));
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1431
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1432 for (i = 0; i < sourceCount; i++) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1433 if (0 != PyObject_GetBuffer(PyList_GET_ITEM(data, i),
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1434 &dataBuffers[i], PyBUF_CONTIG_RO)) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1435 PyErr_Clear();
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1436 PyErr_Format(PyExc_TypeError, "item %zd not a bytes like object", i);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1437 goto finally;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1438 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1439
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1440 sources.sources[i].sourceData = dataBuffers[i].buf;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1441 sources.sources[i].sourceSize = dataBuffers[i].len;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1442 sources.totalSourceSize += dataBuffers[i].len;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1443 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1444
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1445 sources.sourcesSize = sourceCount;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1446 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1447 else {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1448 PyErr_SetString(PyExc_TypeError, "argument must be list of BufferWithSegments");
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1449 goto finally;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1450 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1451
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1452 if (0 == sources.sourcesSize) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1453 PyErr_SetString(PyExc_ValueError, "no source elements found");
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1454 goto finally;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1455 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1456
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1457 if (0 == sources.totalSourceSize) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1458 PyErr_SetString(PyExc_ValueError, "source elements are empty");
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1459 goto finally;
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1460 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1461
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1462 result = compress_from_datasources(self, &sources, threads);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1463
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1464 finally:
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1465 PyMem_Free(sources.sources);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1466
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1467 if (dataBuffers) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1468 for (i = 0; i < sourceCount; i++) {
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1469 PyBuffer_Release(&dataBuffers[i]);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1470 }
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1471
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1472 PyMem_Free(dataBuffers);
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1473 }
30435 b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1474
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1475 return result;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1476 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1477
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1478 static PyMethodDef ZstdCompressor_methods[] = {
30822 b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	1479 { "compress", (PyCFunction)ZstdCompressor_compress,
b54a2984cdd4 zstd: vendor python-zstandard 0.6.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30435 diff changeset	1480 METH_VARARGS \| METH_KEYWORDS, ZstdCompressor_compress__doc__ },
30435 b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1481 { "compressobj", (PyCFunction)ZstdCompressor_compressobj,
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1482 METH_VARARGS \| METH_KEYWORDS, ZstdCompressionObj__doc__ },
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1483 { "copy_stream", (PyCFunction)ZstdCompressor_copy_stream,
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1484 METH_VARARGS \| METH_KEYWORDS, ZstdCompressor_copy_stream__doc__ },
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1485 { "read_from", (PyCFunction)ZstdCompressor_read_from,
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1486 METH_VARARGS \| METH_KEYWORDS, ZstdCompressor_read_from__doc__ },
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1487 { "write_to", (PyCFunction)ZstdCompressor_write_to,
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1488 METH_VARARGS \| METH_KEYWORDS, ZstdCompressor_write_to___doc__ },
31796 e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1489 { "multi_compress_to_buffer", (PyCFunction)ZstdCompressor_multi_compress_to_buffer,
e0dc40530c5a zstd: vendor python-zstandard 0.8.0 Gregory Szorc <gregory.szorc@gmail.com> parents: 30895 diff changeset	1490 METH_VARARGS \| METH_KEYWORDS, ZstdCompressor_multi_compress_to_buffer__doc__ },
30435 b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1491 { NULL, NULL }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1492 };
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1493
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1494 PyTypeObject ZstdCompressorType = {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1495 PyVarObject_HEAD_INIT(NULL, 0)
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1496 "zstd.ZstdCompressor", /* tp_name */
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1497 sizeof(ZstdCompressor), /* tp_basicsize */
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1498 0, /* tp_itemsize */
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1499 (destructor)ZstdCompressor_dealloc, /* tp_dealloc */
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1500 0, /* tp_print */
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1501 0, /* tp_getattr */
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1502 0, /* tp_setattr */
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1503 0, /* tp_compare */
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1504 0, /* tp_repr */
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1505 0, /* tp_as_number */
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1506 0, /* tp_as_sequence */
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1507 0, /* tp_as_mapping */
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1508 0, /* tp_hash */
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1509 0, /* tp_call */
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1510 0, /* tp_str */
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1511 0, /* tp_getattro */
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1512 0, /* tp_setattro */
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1513 0, /* tp_as_buffer */
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1514 Py_TPFLAGS_DEFAULT \| Py_TPFLAGS_BASETYPE, /* tp_flags */
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1515 ZstdCompressor__doc__, /* tp_doc */
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1516 0, /* tp_traverse */
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1517 0, /* tp_clear */
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1518 0, /* tp_richcompare */
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1519 0, /* tp_weaklistoffset */
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1520 0, /* tp_iter */
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1521 0, /* tp_iternext */
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1522 ZstdCompressor_methods, /* tp_methods */
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1523 0, /* tp_members */
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1524 0, /* tp_getset */
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1525 0, /* tp_base */
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1526 0, /* tp_dict */
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1527 0, /* tp_descr_get */
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1528 0, /* tp_descr_set */
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1529 0, /* tp_dictoffset */
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1530 (initproc)ZstdCompressor_init, /* tp_init */
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1531 0, /* tp_alloc */
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1532 PyType_GenericNew, /* tp_new */
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1533 };
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1534
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1535 void compressor_module_init(PyObject* mod) {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1536 Py_TYPE(&ZstdCompressorType) = &PyType_Type;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1537 if (PyType_Ready(&ZstdCompressorType) < 0) {
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1538 return;
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1539 }
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1540
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1541 Py_INCREF((PyObject*)&ZstdCompressorType);
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1542 PyModule_AddObject(mod, "ZstdCompressor",
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1543 (PyObject*)&ZstdCompressorType);
b86a448a2965 zstd: vendor python-zstandard 0.5.0 Gregory Szorc <gregory.szorc@gmail.com> parents: diff changeset	1544 }

Mercurial > public > mercurial-scm > hg

annotate contrib/python-zstandard/c-ext/compressor.c @ 31796:e0dc40530c5a