contrib/python-zstandard/zstd.c
changeset 31796 e0dc40530c5a
parent 30895 c32454d69b85
child 31847 39d36c2db68e
equal deleted inserted replaced
31795:2b130e26c3a4 31796:e0dc40530c5a
     6  * of the BSD license. See the LICENSE file for details.
     6  * of the BSD license. See the LICENSE file for details.
     7  */
     7  */
     8 
     8 
     9 /* A Python C extension for Zstandard. */
     9 /* A Python C extension for Zstandard. */
    10 
    10 
       
    11 #if defined(_WIN32)
       
    12 #define WIN32_LEAN_AND_MEAN
       
    13 #include <Windows.h>
       
    14 #endif
       
    15 
    11 #include "python-zstandard.h"
    16 #include "python-zstandard.h"
    12 
    17 
    13 PyObject *ZstdError;
    18 PyObject *ZstdError;
    14 
    19 
    15 PyDoc_STRVAR(estimate_compression_context_size__doc__,
    20 PyDoc_STRVAR(estimate_compression_context_size__doc__,
    47 "A compression dictionary of size ``dict_size`` will be created from the\n"
    52 "A compression dictionary of size ``dict_size`` will be created from the\n"
    48 "iterable of samples provided by ``samples``.\n"
    53 "iterable of samples provided by ``samples``.\n"
    49 "\n"
    54 "\n"
    50 "The raw dictionary content will be returned\n");
    55 "The raw dictionary content will be returned\n");
    51 
    56 
       
    57 PyDoc_STRVAR(train_cover_dictionary__doc__,
       
    58 "train_cover_dictionary(dict_size, samples, k=None, d=None, notifications=0, dict_id=0, level=0)\n"
       
    59 "\n"
       
    60 "Train a dictionary from sample data using the COVER algorithm.\n"
       
    61 "\n"
       
    62 "This behaves like ``train_dictionary()`` except a different algorithm is\n"
       
    63 "used to create the dictionary. The algorithm has 2 parameters: ``k`` and\n"
       
    64 "``d``. These control the *segment size* and *dmer size*. A reasonable range\n"
       
    65 "for ``k`` is ``[16, 2048+]``. A reasonable range for ``d`` is ``[6, 16]``.\n"
       
    66 "``d`` must be less than or equal to ``k``.\n"
       
    67 );
       
    68 
    52 static char zstd_doc[] = "Interface to zstandard";
    69 static char zstd_doc[] = "Interface to zstandard";
    53 
    70 
    54 static PyMethodDef zstd_methods[] = {
    71 static PyMethodDef zstd_methods[] = {
       
    72 	/* TODO remove since it is a method on CompressionParameters. */
    55 	{ "estimate_compression_context_size", (PyCFunction)estimate_compression_context_size,
    73 	{ "estimate_compression_context_size", (PyCFunction)estimate_compression_context_size,
    56 	METH_VARARGS, estimate_compression_context_size__doc__ },
    74 	METH_VARARGS, estimate_compression_context_size__doc__ },
    57 	{ "estimate_decompression_context_size", (PyCFunction)estimate_decompression_context_size,
    75 	{ "estimate_decompression_context_size", (PyCFunction)estimate_decompression_context_size,
    58 	METH_NOARGS, estimate_decompression_context_size__doc__ },
    76 	METH_NOARGS, estimate_decompression_context_size__doc__ },
    59 	{ "get_compression_parameters", (PyCFunction)get_compression_parameters,
    77 	{ "get_compression_parameters", (PyCFunction)get_compression_parameters,
    60 	METH_VARARGS, get_compression_parameters__doc__ },
    78 	METH_VARARGS, get_compression_parameters__doc__ },
    61 	{ "get_frame_parameters", (PyCFunction)get_frame_parameters,
    79 	{ "get_frame_parameters", (PyCFunction)get_frame_parameters,
    62 	METH_VARARGS, get_frame_parameters__doc__ },
    80 	METH_VARARGS, get_frame_parameters__doc__ },
    63 	{ "train_dictionary", (PyCFunction)train_dictionary,
    81 	{ "train_dictionary", (PyCFunction)train_dictionary,
    64 	METH_VARARGS | METH_KEYWORDS, train_dictionary__doc__ },
    82 	METH_VARARGS | METH_KEYWORDS, train_dictionary__doc__ },
       
    83 	{ "train_cover_dictionary", (PyCFunction)train_cover_dictionary,
       
    84 	METH_VARARGS | METH_KEYWORDS, train_cover_dictionary__doc__ },
    65 	{ NULL, NULL }
    85 	{ NULL, NULL }
    66 };
    86 };
    67 
    87 
       
    88 void bufferutil_module_init(PyObject* mod);
    68 void compressobj_module_init(PyObject* mod);
    89 void compressobj_module_init(PyObject* mod);
    69 void compressor_module_init(PyObject* mod);
    90 void compressor_module_init(PyObject* mod);
    70 void compressionparams_module_init(PyObject* mod);
    91 void compressionparams_module_init(PyObject* mod);
    71 void constants_module_init(PyObject* mod);
    92 void constants_module_init(PyObject* mod);
    72 void dictparams_module_init(PyObject* mod);
       
    73 void compressiondict_module_init(PyObject* mod);
    93 void compressiondict_module_init(PyObject* mod);
    74 void compressionwriter_module_init(PyObject* mod);
    94 void compressionwriter_module_init(PyObject* mod);
    75 void compressoriterator_module_init(PyObject* mod);
    95 void compressoriterator_module_init(PyObject* mod);
    76 void decompressor_module_init(PyObject* mod);
    96 void decompressor_module_init(PyObject* mod);
    77 void decompressobj_module_init(PyObject* mod);
    97 void decompressobj_module_init(PyObject* mod);
    98 	if (ZSTD_VERSION_NUMBER != 10103 || ZSTD_versionNumber() != 10103) {
   118 	if (ZSTD_VERSION_NUMBER != 10103 || ZSTD_versionNumber() != 10103) {
    99 		PyErr_SetString(PyExc_ImportError, "zstd C API mismatch; Python bindings not compiled against expected zstd version");
   119 		PyErr_SetString(PyExc_ImportError, "zstd C API mismatch; Python bindings not compiled against expected zstd version");
   100 		return;
   120 		return;
   101 	}
   121 	}
   102 
   122 
       
   123 	bufferutil_module_init(m);
   103 	compressionparams_module_init(m);
   124 	compressionparams_module_init(m);
   104 	dictparams_module_init(m);
       
   105 	compressiondict_module_init(m);
   125 	compressiondict_module_init(m);
   106 	compressobj_module_init(m);
   126 	compressobj_module_init(m);
   107 	compressor_module_init(m);
   127 	compressor_module_init(m);
   108 	compressionwriter_module_init(m);
   128 	compressionwriter_module_init(m);
   109 	compressoriterator_module_init(m);
   129 	compressoriterator_module_init(m);
   141 	if (m) {
   161 	if (m) {
   142 		zstd_module_init(m);
   162 		zstd_module_init(m);
   143 	}
   163 	}
   144 }
   164 }
   145 #endif
   165 #endif
       
   166 
       
   167 /* Attempt to resolve the number of CPUs in the system. */
       
   168 int cpu_count() {
       
   169 	int count = 0;
       
   170 
       
   171 #if defined(_WIN32)
       
   172 	SYSTEM_INFO si;
       
   173 	si.dwNumberOfProcessors = 0;
       
   174 	GetSystemInfo(&si);
       
   175 	count = si.dwNumberOfProcessors;
       
   176 #elif defined(__APPLE__)
       
   177 	int num;
       
   178 	size_t size = sizeof(int);
       
   179 
       
   180 	if (0 == sysctlbyname("hw.logicalcpu", &num, &size, NULL, 0)) {
       
   181 		count = num;
       
   182 	}
       
   183 #elif defined(__linux__)
       
   184 	count = sysconf(_SC_NPROCESSORS_ONLN);
       
   185 #elif defined(__OpenBSD__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__DragonFly__)
       
   186 	int mib[2];
       
   187 	size_t len = sizeof(count);
       
   188 	mib[0] = CTL_HW;
       
   189 	mib[1] = HW_NCPU;
       
   190 	if (0 != sysctl(mib, 2, &count, &len, NULL, 0)) {
       
   191 		count = 0;
       
   192 	}
       
   193 #elif defined(__hpux)
       
   194 	count = mpctl(MPC_GETNUMSPUS, NULL, NULL);
       
   195 #endif
       
   196 
       
   197 	return count;
       
   198 }
       
   199 
       
   200 size_t roundpow2(size_t i) {
       
   201 	i--;
       
   202 	i |= i >> 1;
       
   203 	i |= i >> 2;
       
   204 	i |= i >> 4;
       
   205 	i |= i >> 8;
       
   206 	i |= i >> 16;
       
   207 	i++;
       
   208 
       
   209 	return i;
       
   210 }