|
1 /** |
|
2 * Copyright (c) 2017-present, Gregory Szorc |
|
3 * All rights reserved. |
|
4 * |
|
5 * This software may be modified and distributed under the terms |
|
6 * of the BSD license. See the LICENSE file for details. |
|
7 */ |
|
8 |
|
9 #include "python-zstandard.h" |
|
10 |
|
11 extern PyObject* ZstdError; |
|
12 |
|
13 PyDoc_STRVAR(BufferWithSegments__doc__, |
|
14 "BufferWithSegments - A memory buffer holding known sub-segments.\n" |
|
15 "\n" |
|
16 "This type represents a contiguous chunk of memory containing N discrete\n" |
|
17 "items within sub-segments of that memory.\n" |
|
18 "\n" |
|
19 "Segments within the buffer are stored as an array of\n" |
|
20 "``(offset, length)`` pairs, where each element is an unsigned 64-bit\n" |
|
21 "integer using the host/native bit order representation.\n" |
|
22 "\n" |
|
23 "The type exists to facilitate operations against N>1 items without the\n" |
|
24 "overhead of Python object creation and management.\n" |
|
25 ); |
|
26 |
|
27 static void BufferWithSegments_dealloc(ZstdBufferWithSegments* self) { |
|
28 /* Backing memory is either canonically owned by a Py_buffer or by us. */ |
|
29 if (self->parent.buf) { |
|
30 PyBuffer_Release(&self->parent); |
|
31 } |
|
32 else if (self->useFree) { |
|
33 free(self->data); |
|
34 } |
|
35 else { |
|
36 PyMem_Free(self->data); |
|
37 } |
|
38 |
|
39 self->data = NULL; |
|
40 |
|
41 if (self->useFree) { |
|
42 free(self->segments); |
|
43 } |
|
44 else { |
|
45 PyMem_Free(self->segments); |
|
46 } |
|
47 |
|
48 self->segments = NULL; |
|
49 |
|
50 PyObject_Del(self); |
|
51 } |
|
52 |
|
53 static int BufferWithSegments_init(ZstdBufferWithSegments* self, PyObject* args, PyObject* kwargs) { |
|
54 static char* kwlist[] = { |
|
55 "data", |
|
56 "segments", |
|
57 NULL |
|
58 }; |
|
59 |
|
60 Py_buffer segments; |
|
61 Py_ssize_t segmentCount; |
|
62 Py_ssize_t i; |
|
63 |
|
64 memset(&self->parent, 0, sizeof(self->parent)); |
|
65 |
|
66 #if PY_MAJOR_VERSION >= 3 |
|
67 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y*y*:BufferWithSegments", |
|
68 #else |
|
69 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s*s*:BufferWithSegments", |
|
70 #endif |
|
71 kwlist, &self->parent, &segments)) { |
|
72 return -1; |
|
73 } |
|
74 |
|
75 if (!PyBuffer_IsContiguous(&self->parent, 'C') || self->parent.ndim > 1) { |
|
76 PyErr_SetString(PyExc_ValueError, "data buffer should be contiguous and have a single dimension"); |
|
77 goto except; |
|
78 } |
|
79 |
|
80 if (!PyBuffer_IsContiguous(&segments, 'C') || segments.ndim > 1) { |
|
81 PyErr_SetString(PyExc_ValueError, "segments buffer should be contiguous and have a single dimension"); |
|
82 goto except; |
|
83 } |
|
84 |
|
85 if (segments.len % sizeof(BufferSegment)) { |
|
86 PyErr_Format(PyExc_ValueError, "segments array size is not a multiple of %lu", |
|
87 sizeof(BufferSegment)); |
|
88 goto except; |
|
89 } |
|
90 |
|
91 segmentCount = segments.len / sizeof(BufferSegment); |
|
92 |
|
93 /* Validate segments data, as blindly trusting it could lead to arbitrary |
|
94 memory access. */ |
|
95 for (i = 0; i < segmentCount; i++) { |
|
96 BufferSegment* segment = &((BufferSegment*)(segments.buf))[i]; |
|
97 |
|
98 if (segment->offset + segment->length > (unsigned long long)self->parent.len) { |
|
99 PyErr_SetString(PyExc_ValueError, "offset within segments array references memory outside buffer"); |
|
100 goto except; |
|
101 return -1; |
|
102 } |
|
103 } |
|
104 |
|
105 /* Make a copy of the segments data. It is cheap to do so and is a guard |
|
106 against caller changing offsets, which has security implications. */ |
|
107 self->segments = PyMem_Malloc(segments.len); |
|
108 if (!self->segments) { |
|
109 PyErr_NoMemory(); |
|
110 goto except; |
|
111 } |
|
112 |
|
113 memcpy(self->segments, segments.buf, segments.len); |
|
114 PyBuffer_Release(&segments); |
|
115 |
|
116 self->data = self->parent.buf; |
|
117 self->dataSize = self->parent.len; |
|
118 self->segmentCount = segmentCount; |
|
119 |
|
120 return 0; |
|
121 |
|
122 except: |
|
123 PyBuffer_Release(&self->parent); |
|
124 PyBuffer_Release(&segments); |
|
125 return -1; |
|
126 }; |
|
127 |
|
128 /** |
|
129 * Construct a BufferWithSegments from existing memory and offsets. |
|
130 * |
|
131 * Ownership of the backing memory and BufferSegments will be transferred to |
|
132 * the created object and freed when the BufferWithSegments is destroyed. |
|
133 */ |
|
134 ZstdBufferWithSegments* BufferWithSegments_FromMemory(void* data, unsigned long long dataSize, |
|
135 BufferSegment* segments, Py_ssize_t segmentsSize) { |
|
136 ZstdBufferWithSegments* result = NULL; |
|
137 Py_ssize_t i; |
|
138 |
|
139 if (NULL == data) { |
|
140 PyErr_SetString(PyExc_ValueError, "data is NULL"); |
|
141 return NULL; |
|
142 } |
|
143 |
|
144 if (NULL == segments) { |
|
145 PyErr_SetString(PyExc_ValueError, "segments is NULL"); |
|
146 return NULL; |
|
147 } |
|
148 |
|
149 for (i = 0; i < segmentsSize; i++) { |
|
150 BufferSegment* segment = &segments[i]; |
|
151 |
|
152 if (segment->offset + segment->length > dataSize) { |
|
153 PyErr_SetString(PyExc_ValueError, "offset in segments overflows buffer size"); |
|
154 return NULL; |
|
155 } |
|
156 } |
|
157 |
|
158 result = PyObject_New(ZstdBufferWithSegments, &ZstdBufferWithSegmentsType); |
|
159 if (NULL == result) { |
|
160 return NULL; |
|
161 } |
|
162 |
|
163 result->useFree = 0; |
|
164 |
|
165 memset(&result->parent, 0, sizeof(result->parent)); |
|
166 result->data = data; |
|
167 result->dataSize = dataSize; |
|
168 result->segments = segments; |
|
169 result->segmentCount = segmentsSize; |
|
170 |
|
171 return result; |
|
172 } |
|
173 |
|
174 static Py_ssize_t BufferWithSegments_length(ZstdBufferWithSegments* self) { |
|
175 return self->segmentCount; |
|
176 } |
|
177 |
|
178 static ZstdBufferSegment* BufferWithSegments_item(ZstdBufferWithSegments* self, Py_ssize_t i) { |
|
179 ZstdBufferSegment* result = NULL; |
|
180 |
|
181 if (i < 0) { |
|
182 PyErr_SetString(PyExc_IndexError, "offset must be non-negative"); |
|
183 return NULL; |
|
184 } |
|
185 |
|
186 if (i >= self->segmentCount) { |
|
187 PyErr_Format(PyExc_IndexError, "offset must be less than %zd", self->segmentCount); |
|
188 return NULL; |
|
189 } |
|
190 |
|
191 result = (ZstdBufferSegment*)PyObject_CallObject((PyObject*)&ZstdBufferSegmentType, NULL); |
|
192 if (NULL == result) { |
|
193 return NULL; |
|
194 } |
|
195 |
|
196 result->parent = (PyObject*)self; |
|
197 Py_INCREF(self); |
|
198 |
|
199 result->data = (char*)self->data + self->segments[i].offset; |
|
200 result->dataSize = self->segments[i].length; |
|
201 result->offset = self->segments[i].offset; |
|
202 |
|
203 return result; |
|
204 } |
|
205 |
|
206 #if PY_MAJOR_VERSION >= 3 |
|
207 static int BufferWithSegments_getbuffer(ZstdBufferWithSegments* self, Py_buffer* view, int flags) { |
|
208 return PyBuffer_FillInfo(view, (PyObject*)self, self->data, self->dataSize, 1, flags); |
|
209 } |
|
210 #else |
|
211 static Py_ssize_t BufferWithSegments_getreadbuffer(ZstdBufferWithSegments* self, Py_ssize_t segment, void **ptrptr) { |
|
212 if (segment != 0) { |
|
213 PyErr_SetString(PyExc_ValueError, "segment number must be 0"); |
|
214 return -1; |
|
215 } |
|
216 |
|
217 *ptrptr = self->data; |
|
218 return self->dataSize; |
|
219 } |
|
220 |
|
221 static Py_ssize_t BufferWithSegments_getsegcount(ZstdBufferWithSegments* self, Py_ssize_t* len) { |
|
222 if (len) { |
|
223 *len = 1; |
|
224 } |
|
225 |
|
226 return 1; |
|
227 } |
|
228 #endif |
|
229 |
|
230 PyDoc_STRVAR(BufferWithSegments_tobytes__doc__, |
|
231 "Obtain a bytes instance for this buffer.\n" |
|
232 ); |
|
233 |
|
234 static PyObject* BufferWithSegments_tobytes(ZstdBufferWithSegments* self) { |
|
235 return PyBytes_FromStringAndSize(self->data, self->dataSize); |
|
236 } |
|
237 |
|
238 PyDoc_STRVAR(BufferWithSegments_segments__doc__, |
|
239 "Obtain a BufferSegments describing segments in this sintance.\n" |
|
240 ); |
|
241 |
|
242 static ZstdBufferSegments* BufferWithSegments_segments(ZstdBufferWithSegments* self) { |
|
243 ZstdBufferSegments* result = (ZstdBufferSegments*)PyObject_CallObject((PyObject*)&ZstdBufferSegmentsType, NULL); |
|
244 if (NULL == result) { |
|
245 return NULL; |
|
246 } |
|
247 |
|
248 result->parent = (PyObject*)self; |
|
249 Py_INCREF(self); |
|
250 result->segments = self->segments; |
|
251 result->segmentCount = self->segmentCount; |
|
252 |
|
253 return result; |
|
254 } |
|
255 |
|
256 static PySequenceMethods BufferWithSegments_sq = { |
|
257 (lenfunc)BufferWithSegments_length, /* sq_length */ |
|
258 0, /* sq_concat */ |
|
259 0, /* sq_repeat */ |
|
260 (ssizeargfunc)BufferWithSegments_item, /* sq_item */ |
|
261 0, /* sq_ass_item */ |
|
262 0, /* sq_contains */ |
|
263 0, /* sq_inplace_concat */ |
|
264 0 /* sq_inplace_repeat */ |
|
265 }; |
|
266 |
|
267 static PyBufferProcs BufferWithSegments_as_buffer = { |
|
268 #if PY_MAJOR_VERSION >= 3 |
|
269 (getbufferproc)BufferWithSegments_getbuffer, /* bf_getbuffer */ |
|
270 0 /* bf_releasebuffer */ |
|
271 #else |
|
272 (readbufferproc)BufferWithSegments_getreadbuffer, /* bf_getreadbuffer */ |
|
273 0, /* bf_getwritebuffer */ |
|
274 (segcountproc)BufferWithSegments_getsegcount, /* bf_getsegcount */ |
|
275 0 /* bf_getcharbuffer */ |
|
276 #endif |
|
277 }; |
|
278 |
|
279 static PyMethodDef BufferWithSegments_methods[] = { |
|
280 { "segments", (PyCFunction)BufferWithSegments_segments, |
|
281 METH_NOARGS, BufferWithSegments_segments__doc__ }, |
|
282 { "tobytes", (PyCFunction)BufferWithSegments_tobytes, |
|
283 METH_NOARGS, BufferWithSegments_tobytes__doc__ }, |
|
284 { NULL, NULL } |
|
285 }; |
|
286 |
|
287 static PyMemberDef BufferWithSegments_members[] = { |
|
288 { "size", T_ULONGLONG, offsetof(ZstdBufferWithSegments, dataSize), |
|
289 READONLY, "total size of the buffer in bytes" }, |
|
290 { NULL } |
|
291 }; |
|
292 |
|
293 PyTypeObject ZstdBufferWithSegmentsType = { |
|
294 PyVarObject_HEAD_INIT(NULL, 0) |
|
295 "zstd.BufferWithSegments", /* tp_name */ |
|
296 sizeof(ZstdBufferWithSegments),/* tp_basicsize */ |
|
297 0, /* tp_itemsize */ |
|
298 (destructor)BufferWithSegments_dealloc, /* tp_dealloc */ |
|
299 0, /* tp_print */ |
|
300 0, /* tp_getattr */ |
|
301 0, /* tp_setattr */ |
|
302 0, /* tp_compare */ |
|
303 0, /* tp_repr */ |
|
304 0, /* tp_as_number */ |
|
305 &BufferWithSegments_sq, /* tp_as_sequence */ |
|
306 0, /* tp_as_mapping */ |
|
307 0, /* tp_hash */ |
|
308 0, /* tp_call */ |
|
309 0, /* tp_str */ |
|
310 0, /* tp_getattro */ |
|
311 0, /* tp_setattro */ |
|
312 &BufferWithSegments_as_buffer, /* tp_as_buffer */ |
|
313 Py_TPFLAGS_DEFAULT, /* tp_flags */ |
|
314 BufferWithSegments__doc__, /* tp_doc */ |
|
315 0, /* tp_traverse */ |
|
316 0, /* tp_clear */ |
|
317 0, /* tp_richcompare */ |
|
318 0, /* tp_weaklistoffset */ |
|
319 0, /* tp_iter */ |
|
320 0, /* tp_iternext */ |
|
321 BufferWithSegments_methods, /* tp_methods */ |
|
322 BufferWithSegments_members, /* tp_members */ |
|
323 0, /* tp_getset */ |
|
324 0, /* tp_base */ |
|
325 0, /* tp_dict */ |
|
326 0, /* tp_descr_get */ |
|
327 0, /* tp_descr_set */ |
|
328 0, /* tp_dictoffset */ |
|
329 (initproc)BufferWithSegments_init, /* tp_init */ |
|
330 0, /* tp_alloc */ |
|
331 PyType_GenericNew, /* tp_new */ |
|
332 }; |
|
333 |
|
334 PyDoc_STRVAR(BufferSegments__doc__, |
|
335 "BufferSegments - Represents segments/offsets within a BufferWithSegments\n" |
|
336 ); |
|
337 |
|
338 static void BufferSegments_dealloc(ZstdBufferSegments* self) { |
|
339 Py_CLEAR(self->parent); |
|
340 PyObject_Del(self); |
|
341 } |
|
342 |
|
343 #if PY_MAJOR_VERSION >= 3 |
|
344 static int BufferSegments_getbuffer(ZstdBufferSegments* self, Py_buffer* view, int flags) { |
|
345 return PyBuffer_FillInfo(view, (PyObject*)self, |
|
346 (void*)self->segments, self->segmentCount * sizeof(BufferSegment), |
|
347 1, flags); |
|
348 } |
|
349 #else |
|
350 static Py_ssize_t BufferSegments_getreadbuffer(ZstdBufferSegments* self, Py_ssize_t segment, void **ptrptr) { |
|
351 if (segment != 0) { |
|
352 PyErr_SetString(PyExc_ValueError, "segment number must be 0"); |
|
353 return -1; |
|
354 } |
|
355 |
|
356 *ptrptr = (void*)self->segments; |
|
357 return self->segmentCount * sizeof(BufferSegment); |
|
358 } |
|
359 |
|
360 static Py_ssize_t BufferSegments_getsegcount(ZstdBufferSegments* self, Py_ssize_t* len) { |
|
361 if (len) { |
|
362 *len = 1; |
|
363 } |
|
364 |
|
365 return 1; |
|
366 } |
|
367 #endif |
|
368 |
|
369 static PyBufferProcs BufferSegments_as_buffer = { |
|
370 #if PY_MAJOR_VERSION >= 3 |
|
371 (getbufferproc)BufferSegments_getbuffer, |
|
372 0 |
|
373 #else |
|
374 (readbufferproc)BufferSegments_getreadbuffer, |
|
375 0, |
|
376 (segcountproc)BufferSegments_getsegcount, |
|
377 0 |
|
378 #endif |
|
379 }; |
|
380 |
|
381 PyTypeObject ZstdBufferSegmentsType = { |
|
382 PyVarObject_HEAD_INIT(NULL, 0) |
|
383 "zstd.BufferSegments", /* tp_name */ |
|
384 sizeof(ZstdBufferSegments),/* tp_basicsize */ |
|
385 0, /* tp_itemsize */ |
|
386 (destructor)BufferSegments_dealloc, /* tp_dealloc */ |
|
387 0, /* tp_print */ |
|
388 0, /* tp_getattr */ |
|
389 0, /* tp_setattr */ |
|
390 0, /* tp_compare */ |
|
391 0, /* tp_repr */ |
|
392 0, /* tp_as_number */ |
|
393 0, /* tp_as_sequence */ |
|
394 0, /* tp_as_mapping */ |
|
395 0, /* tp_hash */ |
|
396 0, /* tp_call */ |
|
397 0, /* tp_str */ |
|
398 0, /* tp_getattro */ |
|
399 0, /* tp_setattro */ |
|
400 &BufferSegments_as_buffer, /* tp_as_buffer */ |
|
401 Py_TPFLAGS_DEFAULT, /* tp_flags */ |
|
402 BufferSegments__doc__, /* tp_doc */ |
|
403 0, /* tp_traverse */ |
|
404 0, /* tp_clear */ |
|
405 0, /* tp_richcompare */ |
|
406 0, /* tp_weaklistoffset */ |
|
407 0, /* tp_iter */ |
|
408 0, /* tp_iternext */ |
|
409 0, /* tp_methods */ |
|
410 0, /* tp_members */ |
|
411 0, /* tp_getset */ |
|
412 0, /* tp_base */ |
|
413 0, /* tp_dict */ |
|
414 0, /* tp_descr_get */ |
|
415 0, /* tp_descr_set */ |
|
416 0, /* tp_dictoffset */ |
|
417 0, /* tp_init */ |
|
418 0, /* tp_alloc */ |
|
419 PyType_GenericNew, /* tp_new */ |
|
420 }; |
|
421 |
|
422 PyDoc_STRVAR(BufferSegment__doc__, |
|
423 "BufferSegment - Represents a segment within a BufferWithSegments\n" |
|
424 ); |
|
425 |
|
426 static void BufferSegment_dealloc(ZstdBufferSegment* self) { |
|
427 Py_CLEAR(self->parent); |
|
428 PyObject_Del(self); |
|
429 } |
|
430 |
|
431 static Py_ssize_t BufferSegment_length(ZstdBufferSegment* self) { |
|
432 return self->dataSize; |
|
433 } |
|
434 |
|
435 #if PY_MAJOR_VERSION >= 3 |
|
436 static int BufferSegment_getbuffer(ZstdBufferSegment* self, Py_buffer* view, int flags) { |
|
437 return PyBuffer_FillInfo(view, (PyObject*)self, |
|
438 self->data, self->dataSize, 1, flags); |
|
439 } |
|
440 #else |
|
441 static Py_ssize_t BufferSegment_getreadbuffer(ZstdBufferSegment* self, Py_ssize_t segment, void **ptrptr) { |
|
442 if (segment != 0) { |
|
443 PyErr_SetString(PyExc_ValueError, "segment number must be 0"); |
|
444 return -1; |
|
445 } |
|
446 |
|
447 *ptrptr = self->data; |
|
448 return self->dataSize; |
|
449 } |
|
450 |
|
451 static Py_ssize_t BufferSegment_getsegcount(ZstdBufferSegment* self, Py_ssize_t* len) { |
|
452 if (len) { |
|
453 *len = 1; |
|
454 } |
|
455 |
|
456 return 1; |
|
457 } |
|
458 #endif |
|
459 |
|
460 PyDoc_STRVAR(BufferSegment_tobytes__doc__, |
|
461 "Obtain a bytes instance for this segment.\n" |
|
462 ); |
|
463 |
|
464 static PyObject* BufferSegment_tobytes(ZstdBufferSegment* self) { |
|
465 return PyBytes_FromStringAndSize(self->data, self->dataSize); |
|
466 } |
|
467 |
|
468 static PySequenceMethods BufferSegment_sq = { |
|
469 (lenfunc)BufferSegment_length, /* sq_length */ |
|
470 0, /* sq_concat */ |
|
471 0, /* sq_repeat */ |
|
472 0, /* sq_item */ |
|
473 0, /* sq_ass_item */ |
|
474 0, /* sq_contains */ |
|
475 0, /* sq_inplace_concat */ |
|
476 0 /* sq_inplace_repeat */ |
|
477 }; |
|
478 |
|
479 static PyBufferProcs BufferSegment_as_buffer = { |
|
480 #if PY_MAJOR_VERSION >= 3 |
|
481 (getbufferproc)BufferSegment_getbuffer, |
|
482 0 |
|
483 #else |
|
484 (readbufferproc)BufferSegment_getreadbuffer, |
|
485 0, |
|
486 (segcountproc)BufferSegment_getsegcount, |
|
487 0 |
|
488 #endif |
|
489 }; |
|
490 |
|
491 static PyMethodDef BufferSegment_methods[] = { |
|
492 { "tobytes", (PyCFunction)BufferSegment_tobytes, |
|
493 METH_NOARGS, BufferSegment_tobytes__doc__ }, |
|
494 { NULL, NULL } |
|
495 }; |
|
496 |
|
497 static PyMemberDef BufferSegment_members[] = { |
|
498 { "offset", T_ULONGLONG, offsetof(ZstdBufferSegment, offset), READONLY, |
|
499 "offset of segment within parent buffer" }, |
|
500 { NULL } |
|
501 }; |
|
502 |
|
503 PyTypeObject ZstdBufferSegmentType = { |
|
504 PyVarObject_HEAD_INIT(NULL, 0) |
|
505 "zstd.BufferSegment", /* tp_name */ |
|
506 sizeof(ZstdBufferSegment),/* tp_basicsize */ |
|
507 0, /* tp_itemsize */ |
|
508 (destructor)BufferSegment_dealloc, /* tp_dealloc */ |
|
509 0, /* tp_print */ |
|
510 0, /* tp_getattr */ |
|
511 0, /* tp_setattr */ |
|
512 0, /* tp_compare */ |
|
513 0, /* tp_repr */ |
|
514 0, /* tp_as_number */ |
|
515 &BufferSegment_sq, /* tp_as_sequence */ |
|
516 0, /* tp_as_mapping */ |
|
517 0, /* tp_hash */ |
|
518 0, /* tp_call */ |
|
519 0, /* tp_str */ |
|
520 0, /* tp_getattro */ |
|
521 0, /* tp_setattro */ |
|
522 &BufferSegment_as_buffer, /* tp_as_buffer */ |
|
523 Py_TPFLAGS_DEFAULT, /* tp_flags */ |
|
524 BufferSegment__doc__, /* tp_doc */ |
|
525 0, /* tp_traverse */ |
|
526 0, /* tp_clear */ |
|
527 0, /* tp_richcompare */ |
|
528 0, /* tp_weaklistoffset */ |
|
529 0, /* tp_iter */ |
|
530 0, /* tp_iternext */ |
|
531 BufferSegment_methods, /* tp_methods */ |
|
532 BufferSegment_members, /* tp_members */ |
|
533 0, /* tp_getset */ |
|
534 0, /* tp_base */ |
|
535 0, /* tp_dict */ |
|
536 0, /* tp_descr_get */ |
|
537 0, /* tp_descr_set */ |
|
538 0, /* tp_dictoffset */ |
|
539 0, /* tp_init */ |
|
540 0, /* tp_alloc */ |
|
541 PyType_GenericNew, /* tp_new */ |
|
542 }; |
|
543 |
|
544 PyDoc_STRVAR(BufferWithSegmentsCollection__doc__, |
|
545 "Represents a collection of BufferWithSegments.\n" |
|
546 ); |
|
547 |
|
548 static void BufferWithSegmentsCollection_dealloc(ZstdBufferWithSegmentsCollection* self) { |
|
549 Py_ssize_t i; |
|
550 |
|
551 if (self->firstElements) { |
|
552 PyMem_Free(self->firstElements); |
|
553 self->firstElements = NULL; |
|
554 } |
|
555 |
|
556 if (self->buffers) { |
|
557 for (i = 0; i < self->bufferCount; i++) { |
|
558 Py_CLEAR(self->buffers[i]); |
|
559 } |
|
560 |
|
561 PyMem_Free(self->buffers); |
|
562 self->buffers = NULL; |
|
563 } |
|
564 |
|
565 PyObject_Del(self); |
|
566 } |
|
567 |
|
568 static int BufferWithSegmentsCollection_init(ZstdBufferWithSegmentsCollection* self, PyObject* args) { |
|
569 Py_ssize_t size; |
|
570 Py_ssize_t i; |
|
571 Py_ssize_t offset = 0; |
|
572 |
|
573 size = PyTuple_Size(args); |
|
574 if (-1 == size) { |
|
575 return -1; |
|
576 } |
|
577 |
|
578 if (0 == size) { |
|
579 PyErr_SetString(PyExc_ValueError, "must pass at least 1 argument"); |
|
580 return -1; |
|
581 } |
|
582 |
|
583 for (i = 0; i < size; i++) { |
|
584 PyObject* item = PyTuple_GET_ITEM(args, i); |
|
585 if (!PyObject_TypeCheck(item, &ZstdBufferWithSegmentsType)) { |
|
586 PyErr_SetString(PyExc_TypeError, "arguments must be BufferWithSegments instances"); |
|
587 return -1; |
|
588 } |
|
589 |
|
590 if (0 == ((ZstdBufferWithSegments*)item)->segmentCount || |
|
591 0 == ((ZstdBufferWithSegments*)item)->dataSize) { |
|
592 PyErr_SetString(PyExc_ValueError, "ZstdBufferWithSegments cannot be empty"); |
|
593 return -1; |
|
594 } |
|
595 } |
|
596 |
|
597 self->buffers = PyMem_Malloc(size * sizeof(ZstdBufferWithSegments*)); |
|
598 if (NULL == self->buffers) { |
|
599 PyErr_NoMemory(); |
|
600 return -1; |
|
601 } |
|
602 |
|
603 self->firstElements = PyMem_Malloc(size * sizeof(Py_ssize_t)); |
|
604 if (NULL == self->firstElements) { |
|
605 PyMem_Free(self->buffers); |
|
606 self->buffers = NULL; |
|
607 PyErr_NoMemory(); |
|
608 return -1; |
|
609 } |
|
610 |
|
611 self->bufferCount = size; |
|
612 |
|
613 for (i = 0; i < size; i++) { |
|
614 ZstdBufferWithSegments* item = (ZstdBufferWithSegments*)PyTuple_GET_ITEM(args, i); |
|
615 |
|
616 self->buffers[i] = item; |
|
617 Py_INCREF(item); |
|
618 |
|
619 if (i > 0) { |
|
620 self->firstElements[i - 1] = offset; |
|
621 } |
|
622 |
|
623 offset += item->segmentCount; |
|
624 } |
|
625 |
|
626 self->firstElements[size - 1] = offset; |
|
627 |
|
628 return 0; |
|
629 } |
|
630 |
|
631 static PyObject* BufferWithSegmentsCollection_size(ZstdBufferWithSegmentsCollection* self) { |
|
632 Py_ssize_t i; |
|
633 Py_ssize_t j; |
|
634 unsigned long long size = 0; |
|
635 |
|
636 for (i = 0; i < self->bufferCount; i++) { |
|
637 for (j = 0; j < self->buffers[i]->segmentCount; j++) { |
|
638 size += self->buffers[i]->segments[j].length; |
|
639 } |
|
640 } |
|
641 |
|
642 return PyLong_FromUnsignedLongLong(size); |
|
643 } |
|
644 |
|
645 Py_ssize_t BufferWithSegmentsCollection_length(ZstdBufferWithSegmentsCollection* self) { |
|
646 return self->firstElements[self->bufferCount - 1]; |
|
647 } |
|
648 |
|
649 static ZstdBufferSegment* BufferWithSegmentsCollection_item(ZstdBufferWithSegmentsCollection* self, Py_ssize_t i) { |
|
650 Py_ssize_t bufferOffset; |
|
651 |
|
652 if (i < 0) { |
|
653 PyErr_SetString(PyExc_IndexError, "offset must be non-negative"); |
|
654 return NULL; |
|
655 } |
|
656 |
|
657 if (i >= BufferWithSegmentsCollection_length(self)) { |
|
658 PyErr_Format(PyExc_IndexError, "offset must be less than %zd", |
|
659 BufferWithSegmentsCollection_length(self)); |
|
660 return NULL; |
|
661 } |
|
662 |
|
663 for (bufferOffset = 0; bufferOffset < self->bufferCount; bufferOffset++) { |
|
664 Py_ssize_t offset = 0; |
|
665 |
|
666 if (i < self->firstElements[bufferOffset]) { |
|
667 if (bufferOffset > 0) { |
|
668 offset = self->firstElements[bufferOffset - 1]; |
|
669 } |
|
670 |
|
671 return BufferWithSegments_item(self->buffers[bufferOffset], i - offset); |
|
672 } |
|
673 } |
|
674 |
|
675 PyErr_SetString(ZstdError, "error resolving segment; this should not happen"); |
|
676 return NULL; |
|
677 } |
|
678 |
|
679 static PySequenceMethods BufferWithSegmentsCollection_sq = { |
|
680 (lenfunc)BufferWithSegmentsCollection_length, /* sq_length */ |
|
681 0, /* sq_concat */ |
|
682 0, /* sq_repeat */ |
|
683 (ssizeargfunc)BufferWithSegmentsCollection_item, /* sq_item */ |
|
684 0, /* sq_ass_item */ |
|
685 0, /* sq_contains */ |
|
686 0, /* sq_inplace_concat */ |
|
687 0 /* sq_inplace_repeat */ |
|
688 }; |
|
689 |
|
690 static PyMethodDef BufferWithSegmentsCollection_methods[] = { |
|
691 { "size", (PyCFunction)BufferWithSegmentsCollection_size, |
|
692 METH_NOARGS, PyDoc_STR("total size in bytes of all segments") }, |
|
693 { NULL, NULL } |
|
694 }; |
|
695 |
|
696 PyTypeObject ZstdBufferWithSegmentsCollectionType = { |
|
697 PyVarObject_HEAD_INIT(NULL, 0) |
|
698 "zstd.BufferWithSegmentsCollection", /* tp_name */ |
|
699 sizeof(ZstdBufferWithSegmentsCollection),/* tp_basicsize */ |
|
700 0, /* tp_itemsize */ |
|
701 (destructor)BufferWithSegmentsCollection_dealloc, /* tp_dealloc */ |
|
702 0, /* tp_print */ |
|
703 0, /* tp_getattr */ |
|
704 0, /* tp_setattr */ |
|
705 0, /* tp_compare */ |
|
706 0, /* tp_repr */ |
|
707 0, /* tp_as_number */ |
|
708 &BufferWithSegmentsCollection_sq, /* tp_as_sequence */ |
|
709 0, /* tp_as_mapping */ |
|
710 0, /* tp_hash */ |
|
711 0, /* tp_call */ |
|
712 0, /* tp_str */ |
|
713 0, /* tp_getattro */ |
|
714 0, /* tp_setattro */ |
|
715 0, /* tp_as_buffer */ |
|
716 Py_TPFLAGS_DEFAULT, /* tp_flags */ |
|
717 BufferWithSegmentsCollection__doc__, /* tp_doc */ |
|
718 0, /* tp_traverse */ |
|
719 0, /* tp_clear */ |
|
720 0, /* tp_richcompare */ |
|
721 0, /* tp_weaklistoffset */ |
|
722 /* TODO implement iterator for performance. */ |
|
723 0, /* tp_iter */ |
|
724 0, /* tp_iternext */ |
|
725 BufferWithSegmentsCollection_methods, /* tp_methods */ |
|
726 0, /* tp_members */ |
|
727 0, /* tp_getset */ |
|
728 0, /* tp_base */ |
|
729 0, /* tp_dict */ |
|
730 0, /* tp_descr_get */ |
|
731 0, /* tp_descr_set */ |
|
732 0, /* tp_dictoffset */ |
|
733 (initproc)BufferWithSegmentsCollection_init, /* tp_init */ |
|
734 0, /* tp_alloc */ |
|
735 PyType_GenericNew, /* tp_new */ |
|
736 }; |
|
737 |
|
738 void bufferutil_module_init(PyObject* mod) { |
|
739 Py_TYPE(&ZstdBufferWithSegmentsType) = &PyType_Type; |
|
740 if (PyType_Ready(&ZstdBufferWithSegmentsType) < 0) { |
|
741 return; |
|
742 } |
|
743 |
|
744 Py_INCREF(&ZstdBufferWithSegmentsType); |
|
745 PyModule_AddObject(mod, "BufferWithSegments", (PyObject*)&ZstdBufferWithSegmentsType); |
|
746 |
|
747 Py_TYPE(&ZstdBufferSegmentsType) = &PyType_Type; |
|
748 if (PyType_Ready(&ZstdBufferSegmentsType) < 0) { |
|
749 return; |
|
750 } |
|
751 |
|
752 Py_INCREF(&ZstdBufferSegmentsType); |
|
753 PyModule_AddObject(mod, "BufferSegments", (PyObject*)&ZstdBufferSegmentsType); |
|
754 |
|
755 Py_TYPE(&ZstdBufferSegmentType) = &PyType_Type; |
|
756 if (PyType_Ready(&ZstdBufferSegmentType) < 0) { |
|
757 return; |
|
758 } |
|
759 |
|
760 Py_INCREF(&ZstdBufferSegmentType); |
|
761 PyModule_AddObject(mod, "BufferSegment", (PyObject*)&ZstdBufferSegmentType); |
|
762 |
|
763 Py_TYPE(&ZstdBufferWithSegmentsCollectionType) = &PyType_Type; |
|
764 if (PyType_Ready(&ZstdBufferWithSegmentsCollectionType) < 0) { |
|
765 return; |
|
766 } |
|
767 |
|
768 Py_INCREF(&ZstdBufferWithSegmentsCollectionType); |
|
769 PyModule_AddObject(mod, "BufferWithSegmentsCollection", (PyObject*)&ZstdBufferWithSegmentsCollectionType); |
|
770 } |