|
1 /* |
|
2 * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. |
|
3 * All rights reserved. |
|
4 * |
|
5 * This source code is licensed under the BSD-style license found in the |
|
6 * LICENSE file in the root directory of this source tree. An additional grant |
|
7 * of patent rights can be found in the PATENTS file in the same directory. |
|
8 */ |
|
9 |
|
10 #ifndef ZSTD_H_235446 |
|
11 #define ZSTD_H_235446 |
|
12 |
|
13 #if defined (__cplusplus) |
|
14 extern "C" { |
|
15 #endif |
|
16 |
|
17 /* ====== Dependency ======*/ |
|
18 #include <stddef.h> /* size_t */ |
|
19 |
|
20 |
|
21 /* ====== Export for Windows ======*/ |
|
22 /* |
|
23 * ZSTD_DLL_EXPORT : |
|
24 * Enable exporting of functions when building a Windows DLL |
|
25 */ |
|
26 #if defined(_WIN32) && defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) |
|
27 # define ZSTDLIB_API __declspec(dllexport) |
|
28 #else |
|
29 # define ZSTDLIB_API |
|
30 #endif |
|
31 |
|
32 |
|
33 /******************************************************************************************************* |
|
34 Introduction |
|
35 |
|
36 zstd, short for Zstandard, is a fast lossless compression algorithm, targeting real-time compression scenarios |
|
37 at zlib-level and better compression ratios. The zstd compression library provides in-memory compression and |
|
38 decompression functions. The library supports compression levels from 1 up to ZSTD_maxCLevel() which is 22. |
|
39 Levels >= 20, labelled `--ultra`, should be used with caution, as they require more memory. |
|
40 Compression can be done in: |
|
41 - a single step (described as Simple API) |
|
42 - a single step, reusing a context (described as Explicit memory management) |
|
43 - unbounded multiple steps (described as Streaming compression) |
|
44 The compression ratio achievable on small data can be highly improved using compression with a dictionary in: |
|
45 - a single step (described as Simple dictionary API) |
|
46 - a single step, reusing a dictionary (described as Fast dictionary API) |
|
47 |
|
48 Advanced experimental functions can be accessed using #define ZSTD_STATIC_LINKING_ONLY before including zstd.h. |
|
49 These APIs shall never be used with a dynamic library. |
|
50 They are not "stable", their definition may change in the future. Only static linking is allowed. |
|
51 *********************************************************************************************************/ |
|
52 |
|
53 /*------ Version ------*/ |
|
54 ZSTDLIB_API unsigned ZSTD_versionNumber (void); /**< returns version number of ZSTD */ |
|
55 |
|
56 #define ZSTD_VERSION_MAJOR 1 |
|
57 #define ZSTD_VERSION_MINOR 1 |
|
58 #define ZSTD_VERSION_RELEASE 1 |
|
59 |
|
60 #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE |
|
61 #define ZSTD_QUOTE(str) #str |
|
62 #define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str) |
|
63 #define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION) |
|
64 |
|
65 #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) |
|
66 |
|
67 |
|
68 /*************************************** |
|
69 * Simple API |
|
70 ***************************************/ |
|
71 /*! ZSTD_compress() : |
|
72 Compresses `src` content as a single zstd compressed frame into already allocated `dst`. |
|
73 Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`. |
|
74 @return : compressed size written into `dst` (<= `dstCapacity), |
|
75 or an error code if it fails (which can be tested using ZSTD_isError()) */ |
|
76 ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity, |
|
77 const void* src, size_t srcSize, |
|
78 int compressionLevel); |
|
79 |
|
80 /*! ZSTD_decompress() : |
|
81 `compressedSize` : must be the _exact_ size of a single compressed frame. |
|
82 `dstCapacity` is an upper bound of originalSize. |
|
83 If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data. |
|
84 @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), |
|
85 or an errorCode if it fails (which can be tested using ZSTD_isError()) */ |
|
86 ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity, |
|
87 const void* src, size_t compressedSize); |
|
88 |
|
89 /*! ZSTD_getDecompressedSize() : |
|
90 * 'src' is the start of a zstd compressed frame. |
|
91 * @return : content size to be decompressed, as a 64-bits value _if known_, 0 otherwise. |
|
92 * note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode. |
|
93 * When `return==0`, data to decompress could be any size. |
|
94 * In which case, it's necessary to use streaming mode to decompress data. |
|
95 * Optionally, application can still use ZSTD_decompress() while relying on implied limits. |
|
96 * (For example, data may be necessarily cut into blocks <= 16 KB). |
|
97 * note 2 : decompressed size is always present when compression is done with ZSTD_compress() |
|
98 * note 3 : decompressed size can be very large (64-bits value), |
|
99 * potentially larger than what local system can handle as a single memory segment. |
|
100 * In which case, it's necessary to use streaming mode to decompress data. |
|
101 * note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified. |
|
102 * Always ensure result fits within application's authorized limits. |
|
103 * Each application can set its own limits. |
|
104 * note 5 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameParams() to know more. */ |
|
105 ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); |
|
106 |
|
107 |
|
108 /*====== Helper functions ======*/ |
|
109 ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */ |
|
110 ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case scenario */ |
|
111 ZSTDLIB_API unsigned ZSTD_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ |
|
112 ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */ |
|
113 |
|
114 |
|
115 /*************************************** |
|
116 * Explicit memory management |
|
117 ***************************************/ |
|
118 /*= Compression context |
|
119 * When compressing many messages / blocks, |
|
120 * it is recommended to allocate a context just once, and re-use it for each successive compression operation. |
|
121 * This will make the situation much easier for the system's memory. |
|
122 * Use one context per thread for parallel execution in multi-threaded environments. */ |
|
123 typedef struct ZSTD_CCtx_s ZSTD_CCtx; |
|
124 ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void); |
|
125 ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); |
|
126 |
|
127 /*! ZSTD_compressCCtx() : |
|
128 Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()) */ |
|
129 ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel); |
|
130 |
|
131 /*= Decompression context */ |
|
132 typedef struct ZSTD_DCtx_s ZSTD_DCtx; |
|
133 ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void); |
|
134 ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); |
|
135 |
|
136 /*! ZSTD_decompressDCtx() : |
|
137 * Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()) */ |
|
138 ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); |
|
139 |
|
140 |
|
141 /************************** |
|
142 * Simple dictionary API |
|
143 ***************************/ |
|
144 /*! ZSTD_compress_usingDict() : |
|
145 * Compression using a predefined Dictionary (see dictBuilder/zdict.h). |
|
146 * Note : This function load the dictionary, resulting in significant startup delay. */ |
|
147 ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, |
|
148 void* dst, size_t dstCapacity, |
|
149 const void* src, size_t srcSize, |
|
150 const void* dict,size_t dictSize, |
|
151 int compressionLevel); |
|
152 |
|
153 /*! ZSTD_decompress_usingDict() : |
|
154 * Decompression using a predefined Dictionary (see dictBuilder/zdict.h). |
|
155 * Dictionary must be identical to the one used during compression. |
|
156 * Note : This function load the dictionary, resulting in significant startup delay */ |
|
157 ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, |
|
158 void* dst, size_t dstCapacity, |
|
159 const void* src, size_t srcSize, |
|
160 const void* dict,size_t dictSize); |
|
161 |
|
162 |
|
163 /**************************** |
|
164 * Fast dictionary API |
|
165 ****************************/ |
|
166 typedef struct ZSTD_CDict_s ZSTD_CDict; |
|
167 |
|
168 /*! ZSTD_createCDict() : |
|
169 * When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once. |
|
170 * ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay. |
|
171 * ZSTD_CDict can be created once and used by multiple threads concurrently, as its usage is read-only. |
|
172 * `dict` can be released after ZSTD_CDict creation */ |
|
173 ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel); |
|
174 |
|
175 /*! ZSTD_freeCDict() : |
|
176 * Function frees memory allocated by ZSTD_createCDict() */ |
|
177 ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict); |
|
178 |
|
179 /*! ZSTD_compress_usingCDict() : |
|
180 * Compression using a digested Dictionary. |
|
181 * Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times. |
|
182 * Note that compression level is decided during dictionary creation */ |
|
183 ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, |
|
184 void* dst, size_t dstCapacity, |
|
185 const void* src, size_t srcSize, |
|
186 const ZSTD_CDict* cdict); |
|
187 |
|
188 |
|
189 typedef struct ZSTD_DDict_s ZSTD_DDict; |
|
190 |
|
191 /*! ZSTD_createDDict() : |
|
192 * Create a digested dictionary, ready to start decompression operation without startup delay. |
|
193 * `dict` can be released after creation */ |
|
194 ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize); |
|
195 |
|
196 /*! ZSTD_freeDDict() : |
|
197 * Function frees memory allocated with ZSTD_createDDict() */ |
|
198 ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict); |
|
199 |
|
200 /*! ZSTD_decompress_usingDDict() : |
|
201 * Decompression using a digested Dictionary |
|
202 * Faster startup than ZSTD_decompress_usingDict(), recommended when same dictionary is used multiple times. */ |
|
203 ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, |
|
204 void* dst, size_t dstCapacity, |
|
205 const void* src, size_t srcSize, |
|
206 const ZSTD_DDict* ddict); |
|
207 |
|
208 |
|
209 /**************************** |
|
210 * Streaming |
|
211 ****************************/ |
|
212 |
|
213 typedef struct ZSTD_inBuffer_s { |
|
214 const void* src; /**< start of input buffer */ |
|
215 size_t size; /**< size of input buffer */ |
|
216 size_t pos; /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */ |
|
217 } ZSTD_inBuffer; |
|
218 |
|
219 typedef struct ZSTD_outBuffer_s { |
|
220 void* dst; /**< start of output buffer */ |
|
221 size_t size; /**< size of output buffer */ |
|
222 size_t pos; /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */ |
|
223 } ZSTD_outBuffer; |
|
224 |
|
225 |
|
226 |
|
227 /*-*********************************************************************** |
|
228 * Streaming compression - HowTo |
|
229 * |
|
230 * A ZSTD_CStream object is required to track streaming operation. |
|
231 * Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources. |
|
232 * ZSTD_CStream objects can be reused multiple times on consecutive compression operations. |
|
233 * It is recommended to re-use ZSTD_CStream in situations where many streaming operations will be achieved consecutively, |
|
234 * since it will play nicer with system's memory, by re-using already allocated memory. |
|
235 * Use one separate ZSTD_CStream per thread for parallel execution. |
|
236 * |
|
237 * Start a new compression by initializing ZSTD_CStream. |
|
238 * Use ZSTD_initCStream() to start a new compression operation. |
|
239 * Use ZSTD_initCStream_usingDict() for a compression which requires a dictionary. |
|
240 * |
|
241 * Use ZSTD_compressStream() repetitively to consume input stream. |
|
242 * The function will automatically update both `pos` fields. |
|
243 * Note that it may not consume the entire input, in which case `pos < size`, |
|
244 * and it's up to the caller to present again remaining data. |
|
245 * @return : a size hint, preferred nb of bytes to use as input for next function call |
|
246 * (it's just a hint, to help latency a little, any other value will work fine) |
|
247 * (note : the size hint is guaranteed to be <= ZSTD_CStreamInSize() ) |
|
248 * or an error code, which can be tested using ZSTD_isError(). |
|
249 * |
|
250 * At any moment, it's possible to flush whatever data remains within buffer, using ZSTD_flushStream(). |
|
251 * `output->pos` will be updated. |
|
252 * Note some content might still be left within internal buffer if `output->size` is too small. |
|
253 * @return : nb of bytes still present within internal buffer (0 if it's empty) |
|
254 * or an error code, which can be tested using ZSTD_isError(). |
|
255 * |
|
256 * ZSTD_endStream() instructs to finish a frame. |
|
257 * It will perform a flush and write frame epilogue. |
|
258 * The epilogue is required for decoders to consider a frame completed. |
|
259 * Similar to ZSTD_flushStream(), it may not be able to flush the full content if `output->size` is too small. |
|
260 * In which case, call again ZSTD_endStream() to complete the flush. |
|
261 * @return : nb of bytes still present within internal buffer (0 if it's empty) |
|
262 * or an error code, which can be tested using ZSTD_isError(). |
|
263 * |
|
264 * *******************************************************************/ |
|
265 |
|
266 /*===== Streaming compression functions ======*/ |
|
267 typedef struct ZSTD_CStream_s ZSTD_CStream; |
|
268 ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void); |
|
269 ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs); |
|
270 ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); |
|
271 ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input); |
|
272 ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); |
|
273 ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); |
|
274 |
|
275 ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */ |
|
276 ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */ |
|
277 |
|
278 |
|
279 |
|
280 /*-*************************************************************************** |
|
281 * Streaming decompression - HowTo |
|
282 * |
|
283 * A ZSTD_DStream object is required to track streaming operations. |
|
284 * Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources. |
|
285 * ZSTD_DStream objects can be re-used multiple times. |
|
286 * |
|
287 * Use ZSTD_initDStream() to start a new decompression operation, |
|
288 * or ZSTD_initDStream_usingDict() if decompression requires a dictionary. |
|
289 * @return : recommended first input size |
|
290 * |
|
291 * Use ZSTD_decompressStream() repetitively to consume your input. |
|
292 * The function will update both `pos` fields. |
|
293 * If `input.pos < input.size`, some input has not been consumed. |
|
294 * It's up to the caller to present again remaining data. |
|
295 * If `output.pos < output.size`, decoder has flushed everything it could. |
|
296 * @return : 0 when a frame is completely decoded and fully flushed, |
|
297 * an error code, which can be tested using ZSTD_isError(), |
|
298 * any other value > 0, which means there is still some work to do to complete the frame. |
|
299 * The return value is a suggested next input size (just an hint, to help latency). |
|
300 * *******************************************************************************/ |
|
301 |
|
302 /*===== Streaming decompression functions =====*/ |
|
303 typedef struct ZSTD_DStream_s ZSTD_DStream; |
|
304 ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void); |
|
305 ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds); |
|
306 ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds); |
|
307 ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input); |
|
308 |
|
309 ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */ |
|
310 ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */ |
|
311 |
|
312 |
|
313 |
|
314 #ifdef ZSTD_STATIC_LINKING_ONLY |
|
315 |
|
316 /**************************************************************************************** |
|
317 * START OF ADVANCED AND EXPERIMENTAL FUNCTIONS |
|
318 * The definitions in this section are considered experimental. |
|
319 * They should never be used with a dynamic library, as they may change in the future. |
|
320 * They are provided for advanced usages. |
|
321 * Use them only in association with static linking. |
|
322 * ***************************************************************************************/ |
|
323 |
|
324 /* --- Constants ---*/ |
|
325 #define ZSTD_MAGICNUMBER 0xFD2FB528 /* v0.8 */ |
|
326 #define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U |
|
327 |
|
328 #define ZSTD_WINDOWLOG_MAX_32 25 |
|
329 #define ZSTD_WINDOWLOG_MAX_64 27 |
|
330 #define ZSTD_WINDOWLOG_MAX ((U32)(MEM_32bits() ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64)) |
|
331 #define ZSTD_WINDOWLOG_MIN 10 |
|
332 #define ZSTD_HASHLOG_MAX ZSTD_WINDOWLOG_MAX |
|
333 #define ZSTD_HASHLOG_MIN 6 |
|
334 #define ZSTD_CHAINLOG_MAX (ZSTD_WINDOWLOG_MAX+1) |
|
335 #define ZSTD_CHAINLOG_MIN ZSTD_HASHLOG_MIN |
|
336 #define ZSTD_HASHLOG3_MAX 17 |
|
337 #define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1) |
|
338 #define ZSTD_SEARCHLOG_MIN 1 |
|
339 #define ZSTD_SEARCHLENGTH_MAX 7 /* only for ZSTD_fast, other strategies are limited to 6 */ |
|
340 #define ZSTD_SEARCHLENGTH_MIN 3 /* only for ZSTD_btopt, other strategies are limited to 4 */ |
|
341 #define ZSTD_TARGETLENGTH_MIN 4 |
|
342 #define ZSTD_TARGETLENGTH_MAX 999 |
|
343 |
|
344 #define ZSTD_FRAMEHEADERSIZE_MAX 18 /* for static allocation */ |
|
345 static const size_t ZSTD_frameHeaderSize_prefix = 5; |
|
346 static const size_t ZSTD_frameHeaderSize_min = 6; |
|
347 static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX; |
|
348 static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable frame length */ |
|
349 |
|
350 |
|
351 /*--- Advanced types ---*/ |
|
352 typedef enum { ZSTD_fast, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt, ZSTD_btopt2 } ZSTD_strategy; /* from faster to stronger */ |
|
353 |
|
354 typedef struct { |
|
355 unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */ |
|
356 unsigned chainLog; /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */ |
|
357 unsigned hashLog; /**< dispatch table : larger == faster, more memory */ |
|
358 unsigned searchLog; /**< nb of searches : larger == more compression, slower */ |
|
359 unsigned searchLength; /**< match length searched : larger == faster decompression, sometimes less compression */ |
|
360 unsigned targetLength; /**< acceptable match size for optimal parser (only) : larger == more compression, slower */ |
|
361 ZSTD_strategy strategy; |
|
362 } ZSTD_compressionParameters; |
|
363 |
|
364 typedef struct { |
|
365 unsigned contentSizeFlag; /**< 1: content size will be in frame header (if known). */ |
|
366 unsigned checksumFlag; /**< 1: will generate a 22-bits checksum at end of frame, to be used for error detection by decompressor */ |
|
367 unsigned noDictIDFlag; /**< 1: no dict ID will be saved into frame header (if dictionary compression) */ |
|
368 } ZSTD_frameParameters; |
|
369 |
|
370 typedef struct { |
|
371 ZSTD_compressionParameters cParams; |
|
372 ZSTD_frameParameters fParams; |
|
373 } ZSTD_parameters; |
|
374 |
|
375 /*= Custom memory allocation functions */ |
|
376 typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size); |
|
377 typedef void (*ZSTD_freeFunction) (void* opaque, void* address); |
|
378 typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem; |
|
379 |
|
380 |
|
381 /*************************************** |
|
382 * Advanced compression functions |
|
383 ***************************************/ |
|
384 /*! ZSTD_estimateCCtxSize() : |
|
385 * Gives the amount of memory allocated for a ZSTD_CCtx given a set of compression parameters. |
|
386 * `frameContentSize` is an optional parameter, provide `0` if unknown */ |
|
387 ZSTDLIB_API size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams); |
|
388 |
|
389 /*! ZSTD_createCCtx_advanced() : |
|
390 * Create a ZSTD compression context using external alloc and free functions */ |
|
391 ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); |
|
392 |
|
393 /*! ZSTD_sizeofCCtx() : |
|
394 * Gives the amount of memory used by a given ZSTD_CCtx */ |
|
395 ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx); |
|
396 |
|
397 /*! ZSTD_createCDict_advanced() : |
|
398 * Create a ZSTD_CDict using external alloc and free, and customized compression parameters */ |
|
399 ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, |
|
400 ZSTD_parameters params, ZSTD_customMem customMem); |
|
401 |
|
402 /*! ZSTD_sizeof_CDict() : |
|
403 * Gives the amount of memory used by a given ZSTD_sizeof_CDict */ |
|
404 ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict); |
|
405 |
|
406 /*! ZSTD_getParams() : |
|
407 * same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of a `ZSTD_compressionParameters`. |
|
408 * All fields of `ZSTD_frameParameters` are set to default (0) */ |
|
409 ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSize, size_t dictSize); |
|
410 |
|
411 /*! ZSTD_getCParams() : |
|
412 * @return ZSTD_compressionParameters structure for a selected compression level and srcSize. |
|
413 * `srcSize` value is optional, select 0 if not known */ |
|
414 ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long srcSize, size_t dictSize); |
|
415 |
|
416 /*! ZSTD_checkCParams() : |
|
417 * Ensure param values remain within authorized range */ |
|
418 ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params); |
|
419 |
|
420 /*! ZSTD_adjustCParams() : |
|
421 * optimize params for a given `srcSize` and `dictSize`. |
|
422 * both values are optional, select `0` if unknown. */ |
|
423 ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize); |
|
424 |
|
425 /*! ZSTD_compress_advanced() : |
|
426 * Same as ZSTD_compress_usingDict(), with fine-tune control of each compression parameter */ |
|
427 ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, |
|
428 void* dst, size_t dstCapacity, |
|
429 const void* src, size_t srcSize, |
|
430 const void* dict,size_t dictSize, |
|
431 ZSTD_parameters params); |
|
432 |
|
433 |
|
434 /*--- Advanced decompression functions ---*/ |
|
435 |
|
436 /*! ZSTD_estimateDCtxSize() : |
|
437 * Gives the potential amount of memory allocated to create a ZSTD_DCtx */ |
|
438 ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void); |
|
439 |
|
440 /*! ZSTD_createDCtx_advanced() : |
|
441 * Create a ZSTD decompression context using external alloc and free functions */ |
|
442 ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); |
|
443 |
|
444 /*! ZSTD_sizeof_DCtx() : |
|
445 * Gives the amount of memory used by a given ZSTD_DCtx */ |
|
446 ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); |
|
447 |
|
448 /*! ZSTD_sizeof_DDict() : |
|
449 * Gives the amount of memory used by a given ZSTD_DDict */ |
|
450 ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); |
|
451 |
|
452 |
|
453 /******************************************************************** |
|
454 * Advanced streaming functions |
|
455 ********************************************************************/ |
|
456 |
|
457 /*===== Advanced Streaming compression functions =====*/ |
|
458 ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); |
|
459 ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); |
|
460 ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, |
|
461 ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be zero == unknown */ |
|
462 ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /**< note : cdict will just be referenced, and must outlive compression session */ |
|
463 ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); /**< re-use compression parameters from previous init; skip dictionary loading stage; zcs must be init at least once before */ |
|
464 ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); |
|
465 |
|
466 |
|
467 /*===== Advanced Streaming decompression functions =====*/ |
|
468 typedef enum { ZSTDdsp_maxWindowSize } ZSTD_DStreamParameter_e; |
|
469 ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); |
|
470 ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); |
|
471 ZSTDLIB_API size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue); |
|
472 ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); /**< note : ddict will just be referenced, and must outlive decompression session */ |
|
473 ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); /**< re-use decompression parameters from previous init; saves dictionary loading */ |
|
474 ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); |
|
475 |
|
476 |
|
477 /********************************************************************* |
|
478 * Buffer-less and synchronous inner streaming functions |
|
479 * |
|
480 * This is an advanced API, giving full control over buffer management, for users which need direct control over memory. |
|
481 * But it's also a complex one, with many restrictions (documented below). |
|
482 * Prefer using normal streaming API for an easier experience |
|
483 ********************************************************************* */ |
|
484 |
|
485 /** |
|
486 Buffer-less streaming compression (synchronous mode) |
|
487 |
|
488 A ZSTD_CCtx object is required to track streaming operations. |
|
489 Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource. |
|
490 ZSTD_CCtx object can be re-used multiple times within successive compression operations. |
|
491 |
|
492 Start by initializing a context. |
|
493 Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression, |
|
494 or ZSTD_compressBegin_advanced(), for finer parameter control. |
|
495 It's also possible to duplicate a reference context which has already been initialized, using ZSTD_copyCCtx() |
|
496 |
|
497 Then, consume your input using ZSTD_compressContinue(). |
|
498 There are some important considerations to keep in mind when using this advanced function : |
|
499 - ZSTD_compressContinue() has no internal buffer. It uses externally provided buffer only. |
|
500 - Interface is synchronous : input is consumed entirely and produce 1+ (or more) compressed blocks. |
|
501 - Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario. |
|
502 Worst case evaluation is provided by ZSTD_compressBound(). |
|
503 ZSTD_compressContinue() doesn't guarantee recover after a failed compression. |
|
504 - ZSTD_compressContinue() presumes prior input ***is still accessible and unmodified*** (up to maximum distance size, see WindowLog). |
|
505 It remembers all previous contiguous blocks, plus one separated memory segment (which can itself consists of multiple contiguous blocks) |
|
506 - ZSTD_compressContinue() detects that prior input has been overwritten when `src` buffer overlaps. |
|
507 In which case, it will "discard" the relevant memory section from its history. |
|
508 |
|
509 Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum. |
|
510 It's possible to use a NULL,0 src content, in which case, it will write a final empty block to end the frame, |
|
511 Without last block mark, frames will be considered unfinished (broken) by decoders. |
|
512 |
|
513 You can then reuse `ZSTD_CCtx` (ZSTD_compressBegin()) to compress some new frame. |
|
514 */ |
|
515 |
|
516 /*===== Buffer-less streaming compression functions =====*/ |
|
517 ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); |
|
518 ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); |
|
519 ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); |
|
520 ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); |
|
521 ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); |
|
522 ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); |
|
523 |
|
524 |
|
525 |
|
526 /*- |
|
527 Buffer-less streaming decompression (synchronous mode) |
|
528 |
|
529 A ZSTD_DCtx object is required to track streaming operations. |
|
530 Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it. |
|
531 A ZSTD_DCtx object can be re-used multiple times. |
|
532 |
|
533 First typical operation is to retrieve frame parameters, using ZSTD_getFrameParams(). |
|
534 It fills a ZSTD_frameParams structure which provide important information to correctly decode the frame, |
|
535 such as the minimum rolling buffer size to allocate to decompress data (`windowSize`), |
|
536 and the dictionary ID used. |
|
537 (Note : content size is optional, it may not be present. 0 means : content size unknown). |
|
538 Note that these values could be wrong, either because of data malformation, or because an attacker is spoofing deliberate false information. |
|
539 As a consequence, check that values remain within valid application range, especially `windowSize`, before allocation. |
|
540 Each application can set its own limit, depending on local restrictions. For extended interoperability, it is recommended to support at least 8 MB. |
|
541 Frame parameters are extracted from the beginning of the compressed frame. |
|
542 Data fragment must be large enough to ensure successful decoding, typically `ZSTD_frameHeaderSize_max` bytes. |
|
543 @result : 0 : successful decoding, the `ZSTD_frameParams` structure is correctly filled. |
|
544 >0 : `srcSize` is too small, please provide at least @result bytes on next attempt. |
|
545 errorCode, which can be tested using ZSTD_isError(). |
|
546 |
|
547 Start decompression, with ZSTD_decompressBegin() or ZSTD_decompressBegin_usingDict(). |
|
548 Alternatively, you can copy a prepared context, using ZSTD_copyDCtx(). |
|
549 |
|
550 Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively. |
|
551 ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue(). |
|
552 ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail. |
|
553 |
|
554 @result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity). |
|
555 It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some metadata item. |
|
556 It can also be an error code, which can be tested with ZSTD_isError(). |
|
557 |
|
558 ZSTD_decompressContinue() needs previous data blocks during decompression, up to `windowSize`. |
|
559 They should preferably be located contiguously, prior to current block. |
|
560 Alternatively, a round buffer of sufficient size is also possible. Sufficient size is determined by frame parameters. |
|
561 ZSTD_decompressContinue() is very sensitive to contiguity, |
|
562 if 2 blocks don't follow each other, make sure that either the compressor breaks contiguity at the same place, |
|
563 or that previous contiguous segment is large enough to properly handle maximum back-reference. |
|
564 |
|
565 A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero. |
|
566 Context can then be reset to start a new decompression. |
|
567 |
|
568 Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType(). |
|
569 This information is not required to properly decode a frame. |
|
570 |
|
571 == Special case : skippable frames == |
|
572 |
|
573 Skippable frames allow integration of user-defined data into a flow of concatenated frames. |
|
574 Skippable frames will be ignored (skipped) by a decompressor. The format of skippable frames is as follows : |
|
575 a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F |
|
576 b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits |
|
577 c) Frame Content - any content (User Data) of length equal to Frame Size |
|
578 For skippable frames ZSTD_decompressContinue() always returns 0. |
|
579 For skippable frames ZSTD_getFrameParams() returns fparamsPtr->windowLog==0 what means that a frame is skippable. |
|
580 It also returns Frame Size as fparamsPtr->frameContentSize. |
|
581 */ |
|
582 |
|
583 typedef struct { |
|
584 unsigned long long frameContentSize; |
|
585 unsigned windowSize; |
|
586 unsigned dictID; |
|
587 unsigned checksumFlag; |
|
588 } ZSTD_frameParams; |
|
589 |
|
590 /*===== Buffer-less streaming decompression functions =====*/ |
|
591 ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input, see details below */ |
|
592 ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); |
|
593 ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); |
|
594 ZSTDLIB_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx); |
|
595 ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx); |
|
596 ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); |
|
597 typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e; |
|
598 ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); |
|
599 |
|
600 /** |
|
601 Block functions |
|
602 |
|
603 Block functions produce and decode raw zstd blocks, without frame metadata. |
|
604 Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes). |
|
605 User will have to take in charge required information to regenerate data, such as compressed and content sizes. |
|
606 |
|
607 A few rules to respect : |
|
608 - Compressing and decompressing require a context structure |
|
609 + Use ZSTD_createCCtx() and ZSTD_createDCtx() |
|
610 - It is necessary to init context before starting |
|
611 + compression : ZSTD_compressBegin() |
|
612 + decompression : ZSTD_decompressBegin() |
|
613 + variants _usingDict() are also allowed |
|
614 + copyCCtx() and copyDCtx() work too |
|
615 - Block size is limited, it must be <= ZSTD_getBlockSizeMax() |
|
616 + If you need to compress more, cut data into multiple blocks |
|
617 + Consider using the regular ZSTD_compress() instead, as frame metadata costs become negligible when source size is large. |
|
618 - When a block is considered not compressible enough, ZSTD_compressBlock() result will be zero. |
|
619 In which case, nothing is produced into `dst`. |
|
620 + User must test for such outcome and deal directly with uncompressed data |
|
621 + ZSTD_decompressBlock() doesn't accept uncompressed data as input !!! |
|
622 + In case of multiple successive blocks, decoder must be informed of uncompressed block existence to follow proper history. |
|
623 Use ZSTD_insertBlock() in such a case. |
|
624 */ |
|
625 |
|
626 #define ZSTD_BLOCKSIZE_ABSOLUTEMAX (128 * 1024) /* define, for static allocation */ |
|
627 /*===== Raw zstd block functions =====*/ |
|
628 ZSTDLIB_API size_t ZSTD_getBlockSizeMax(ZSTD_CCtx* cctx); |
|
629 ZSTDLIB_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); |
|
630 ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); |
|
631 ZSTDLIB_API size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert block into `dctx` history. Useful for uncompressed blocks */ |
|
632 |
|
633 |
|
634 #endif /* ZSTD_STATIC_LINKING_ONLY */ |
|
635 |
|
636 #if defined (__cplusplus) |
|
637 } |
|
638 #endif |
|
639 |
|
640 #endif /* ZSTD_H_235446 */ |