diff -r 2b130e26c3a4 -r e0dc40530c5a contrib/python-zstandard/c-ext/python-zstandard.h --- a/contrib/python-zstandard/c-ext/python-zstandard.h Sat Apr 01 13:43:52 2017 -0700 +++ b/contrib/python-zstandard/c-ext/python-zstandard.h Sat Apr 01 15:24:03 2017 -0700 @@ -15,14 +15,20 @@ #include "mem.h" #include "zstd.h" #include "zdict.h" +#include "zstdmt_compress.h" -#define PYTHON_ZSTANDARD_VERSION "0.7.0" +#define PYTHON_ZSTANDARD_VERSION "0.8.0" typedef enum { compressorobj_flush_finish, compressorobj_flush_block, } CompressorObj_Flush; +/* + Represents a CompressionParameters type. + + This type is basically a wrapper around ZSTD_compressionParameters. +*/ typedef struct { PyObject_HEAD unsigned windowLog; @@ -36,6 +42,11 @@ extern PyTypeObject CompressionParametersType; +/* + Represents a FrameParameters type. + + This type is basically a wrapper around ZSTD_frameParams. +*/ typedef struct { PyObject_HEAD unsigned long long frameContentSize; @@ -46,34 +57,55 @@ extern PyTypeObject FrameParametersType; -typedef struct { - PyObject_HEAD - unsigned selectivityLevel; - int compressionLevel; - unsigned notificationLevel; - unsigned dictID; -} DictParametersObject; +/* + Represents a ZstdCompressionDict type. -extern PyTypeObject DictParametersType; - + Instances hold data used for a zstd compression dictionary. +*/ typedef struct { PyObject_HEAD + /* Pointer to dictionary data. Owned by self. */ void* dictData; + /* Size of dictionary data. */ size_t dictSize; + /* k parameter for cover dictionaries. Only populated by train_cover_dict(). */ + unsigned k; + /* d parameter for cover dictionaries. Only populated by train_cover_dict(). */ + unsigned d; } ZstdCompressionDict; extern PyTypeObject ZstdCompressionDictType; +/* + Represents a ZstdCompressor type. +*/ typedef struct { PyObject_HEAD + /* Configured compression level. Should be always set. */ int compressionLevel; + /* Number of threads to use for operations. */ + unsigned int threads; + /* Pointer to compression dictionary to use. NULL if not using dictionary + compression. */ ZstdCompressionDict* dict; + /* Compression context to use. Populated during object construction. NULL + if using multi-threaded compression. */ ZSTD_CCtx* cctx; + /* Multi-threaded compression context to use. Populated during object + construction. NULL if not using multi-threaded compression. */ + ZSTDMT_CCtx* mtcctx; + /* Digest compression dictionary. NULL initially. Populated on first use. */ ZSTD_CDict* cdict; + /* Low-level compression parameter control. NULL unless passed to + constructor. Takes precedence over `compressionLevel` if defined. */ CompressionParametersObject* cparams; + /* Controls zstd frame options. */ ZSTD_frameParameters fparams; + /* Holds state for streaming compression. Shared across all invocation. + Populated on first use. */ + ZSTD_CStream* cstream; } ZstdCompressor; extern PyTypeObject ZstdCompressorType; @@ -82,7 +114,6 @@ PyObject_HEAD ZstdCompressor* compressor; - ZSTD_CStream* cstream; ZSTD_outBuffer output; int finished; } ZstdCompressionObj; @@ -96,7 +127,6 @@ PyObject* writer; Py_ssize_t sourceSize; size_t outSize; - ZSTD_CStream* cstream; int entered; } ZstdCompressionWriter; @@ -113,7 +143,6 @@ size_t inSize; size_t outSize; - ZSTD_CStream* cstream; ZSTD_inBuffer input; ZSTD_outBuffer output; int finishedOutput; @@ -130,6 +159,7 @@ ZstdCompressionDict* dict; ZSTD_DDict* ddict; + ZSTD_DStream* dstream; } ZstdDecompressor; extern PyTypeObject ZstdDecompressorType; @@ -138,7 +168,6 @@ PyObject_HEAD ZstdDecompressor* decompressor; - ZSTD_DStream* dstream; int finished; } ZstdDecompressionObj; @@ -150,7 +179,6 @@ ZstdDecompressor* decompressor; PyObject* writer; size_t outSize; - ZSTD_DStream* dstream; int entered; } ZstdDecompressionWriter; @@ -166,7 +194,6 @@ size_t inSize; size_t outSize; size_t skipBytes; - ZSTD_DStream* dstream; ZSTD_inBuffer input; ZSTD_outBuffer output; Py_ssize_t readCount; @@ -181,10 +208,78 @@ PyObject* chunk; } DecompressorIteratorResult; +typedef struct { + unsigned long long offset; + unsigned long long length; +} BufferSegment; + +typedef struct { + PyObject_HEAD + + PyObject* parent; + BufferSegment* segments; + Py_ssize_t segmentCount; +} ZstdBufferSegments; + +extern PyTypeObject ZstdBufferSegmentsType; + +typedef struct { + PyObject_HEAD + + PyObject* parent; + void* data; + Py_ssize_t dataSize; + unsigned long long offset; +} ZstdBufferSegment; + +extern PyTypeObject ZstdBufferSegmentType; + +typedef struct { + PyObject_HEAD + + Py_buffer parent; + void* data; + unsigned long long dataSize; + BufferSegment* segments; + Py_ssize_t segmentCount; + int useFree; +} ZstdBufferWithSegments; + +extern PyTypeObject ZstdBufferWithSegmentsType; + +/** + * An ordered collection of BufferWithSegments exposed as a squashed collection. + * + * This type provides a virtual view spanning multiple BufferWithSegments + * instances. It allows multiple instances to be "chained" together and + * exposed as a single collection. e.g. if there are 2 buffers holding + * 10 segments each, then o[14] will access the 5th segment in the 2nd buffer. + */ +typedef struct { + PyObject_HEAD + + /* An array of buffers that should be exposed through this instance. */ + ZstdBufferWithSegments** buffers; + /* Number of elements in buffers array. */ + Py_ssize_t bufferCount; + /* Array of first offset in each buffer instance. 0th entry corresponds + to number of elements in the 0th buffer. 1st entry corresponds to the + sum of elements in 0th and 1st buffers. */ + Py_ssize_t* firstElements; +} ZstdBufferWithSegmentsCollection; + +extern PyTypeObject ZstdBufferWithSegmentsCollectionType; + void ztopy_compression_parameters(CompressionParametersObject* params, ZSTD_compressionParameters* zparams); CompressionParametersObject* get_compression_parameters(PyObject* self, PyObject* args); FrameParametersObject* get_frame_parameters(PyObject* self, PyObject* args); PyObject* estimate_compression_context_size(PyObject* self, PyObject* args); -ZSTD_CStream* CStream_from_ZstdCompressor(ZstdCompressor* compressor, Py_ssize_t sourceSize); -ZSTD_DStream* DStream_from_ZstdDecompressor(ZstdDecompressor* decompressor); +int init_cstream(ZstdCompressor* compressor, unsigned long long sourceSize); +int init_mtcstream(ZstdCompressor* compressor, Py_ssize_t sourceSize); +int init_dstream(ZstdDecompressor* decompressor); ZstdCompressionDict* train_dictionary(PyObject* self, PyObject* args, PyObject* kwargs); +ZstdCompressionDict* train_cover_dictionary(PyObject* self, PyObject* args, PyObject* kwargs); +ZstdBufferWithSegments* BufferWithSegments_FromMemory(void* data, unsigned long long dataSize, BufferSegment* segments, Py_ssize_t segmentsSize); +Py_ssize_t BufferWithSegmentsCollection_length(ZstdBufferWithSegmentsCollection*); +int cpu_count(void); +size_t roundpow2(size_t);