18 'common/fse_decompress.c', |
21 'common/fse_decompress.c', |
19 'common/xxhash.c', |
22 'common/xxhash.c', |
20 'common/zstd_common.c', |
23 'common/zstd_common.c', |
21 'compress/fse_compress.c', |
24 'compress/fse_compress.c', |
22 'compress/huf_compress.c', |
25 'compress/huf_compress.c', |
23 'compress/zbuff_compress.c', |
|
24 'compress/zstd_compress.c', |
26 'compress/zstd_compress.c', |
25 'decompress/huf_decompress.c', |
27 'decompress/huf_decompress.c', |
26 'decompress/zbuff_decompress.c', |
|
27 'decompress/zstd_decompress.c', |
28 'decompress/zstd_decompress.c', |
28 'dictBuilder/divsufsort.c', |
29 'dictBuilder/divsufsort.c', |
29 'dictBuilder/zdict.c', |
30 'dictBuilder/zdict.c', |
30 )] |
31 )] |
31 |
32 |
35 'zstd/compress', |
36 'zstd/compress', |
36 'zstd/decompress', |
37 'zstd/decompress', |
37 'zstd/dictBuilder', |
38 'zstd/dictBuilder', |
38 )] |
39 )] |
39 |
40 |
|
41 # cffi can't parse some of the primitives in zstd.h. So we invoke the |
|
42 # preprocessor and feed its output into cffi. |
|
43 compiler = distutils.ccompiler.new_compiler() |
|
44 |
|
45 # Needed for MSVC. |
|
46 if hasattr(compiler, 'initialize'): |
|
47 compiler.initialize() |
|
48 |
|
49 # Distutils doesn't set compiler.preprocessor, so invoke the preprocessor |
|
50 # manually. |
|
51 if compiler.compiler_type == 'unix': |
|
52 args = list(compiler.executables['compiler']) |
|
53 args.extend([ |
|
54 '-E', |
|
55 '-DZSTD_STATIC_LINKING_ONLY', |
|
56 ]) |
|
57 elif compiler.compiler_type == 'msvc': |
|
58 args = [compiler.cc] |
|
59 args.extend([ |
|
60 '/EP', |
|
61 '/DZSTD_STATIC_LINKING_ONLY', |
|
62 ]) |
|
63 else: |
|
64 raise Exception('unsupported compiler type: %s' % compiler.compiler_type) |
|
65 |
|
66 # zstd.h includes <stddef.h>, which is also included by cffi's boilerplate. |
|
67 # This can lead to duplicate declarations. So we strip this include from the |
|
68 # preprocessor invocation. |
|
69 |
40 with open(os.path.join(HERE, 'zstd', 'zstd.h'), 'rb') as fh: |
70 with open(os.path.join(HERE, 'zstd', 'zstd.h'), 'rb') as fh: |
41 zstd_h = fh.read() |
71 lines = [l for l in fh if not l.startswith(b'#include <stddef.h>')] |
|
72 |
|
73 fd, input_file = tempfile.mkstemp(suffix='.h') |
|
74 os.write(fd, b''.join(lines)) |
|
75 os.close(fd) |
|
76 |
|
77 args.append(input_file) |
|
78 |
|
79 try: |
|
80 process = subprocess.Popen(args, stdout=subprocess.PIPE) |
|
81 output = process.communicate()[0] |
|
82 ret = process.poll() |
|
83 if ret: |
|
84 raise Exception('preprocessor exited with error') |
|
85 finally: |
|
86 os.unlink(input_file) |
|
87 |
|
88 def normalize_output(): |
|
89 lines = [] |
|
90 for line in output.splitlines(): |
|
91 # CFFI's parser doesn't like __attribute__ on UNIX compilers. |
|
92 if line.startswith(b'__attribute__ ((visibility ("default"))) '): |
|
93 line = line[len(b'__attribute__ ((visibility ("default"))) '):] |
|
94 |
|
95 lines.append(line) |
|
96 |
|
97 return b'\n'.join(lines) |
42 |
98 |
43 ffi = cffi.FFI() |
99 ffi = cffi.FFI() |
44 ffi.set_source('_zstd_cffi', ''' |
100 ffi.set_source('_zstd_cffi', ''' |
45 /* needed for typedefs like U32 references in zstd.h */ |
|
46 #include "mem.h" |
|
47 #define ZSTD_STATIC_LINKING_ONLY |
101 #define ZSTD_STATIC_LINKING_ONLY |
48 #include "zstd.h" |
102 #include "zstd.h" |
49 ''', |
103 ''', sources=SOURCES, include_dirs=INCLUDE_DIRS) |
50 sources=SOURCES, include_dirs=INCLUDE_DIRS) |
|
51 |
104 |
52 # Rather than define the API definitions from zstd.h inline, munge the |
105 ffi.cdef(normalize_output().decode('latin1')) |
53 # source in a way that cdef() will accept. |
|
54 lines = zstd_h.splitlines() |
|
55 lines = [l.rstrip() for l in lines if l.strip()] |
|
56 |
|
57 # Strip preprocessor directives - they aren't important for our needs. |
|
58 lines = [l for l in lines |
|
59 if not l.startswith((b'#if', b'#else', b'#endif', b'#include'))] |
|
60 |
|
61 # Remove extern C block |
|
62 lines = [l for l in lines if l not in (b'extern "C" {', b'}')] |
|
63 |
|
64 # The version #defines don't parse and aren't necessary. Strip them. |
|
65 lines = [l for l in lines if not l.startswith(( |
|
66 b'#define ZSTD_H_235446', |
|
67 b'#define ZSTD_LIB_VERSION', |
|
68 b'#define ZSTD_QUOTE', |
|
69 b'#define ZSTD_EXPAND_AND_QUOTE', |
|
70 b'#define ZSTD_VERSION_STRING', |
|
71 b'#define ZSTD_VERSION_NUMBER'))] |
|
72 |
|
73 # The C parser also doesn't like some constant defines referencing |
|
74 # other constants. |
|
75 # TODO we pick the 64-bit constants here. We should assert somewhere |
|
76 # we're compiling for 64-bit. |
|
77 def fix_constants(l): |
|
78 if l.startswith(b'#define ZSTD_WINDOWLOG_MAX '): |
|
79 return b'#define ZSTD_WINDOWLOG_MAX 27' |
|
80 elif l.startswith(b'#define ZSTD_CHAINLOG_MAX '): |
|
81 return b'#define ZSTD_CHAINLOG_MAX 28' |
|
82 elif l.startswith(b'#define ZSTD_HASHLOG_MAX '): |
|
83 return b'#define ZSTD_HASHLOG_MAX 27' |
|
84 elif l.startswith(b'#define ZSTD_CHAINLOG_MAX '): |
|
85 return b'#define ZSTD_CHAINLOG_MAX 28' |
|
86 elif l.startswith(b'#define ZSTD_CHAINLOG_MIN '): |
|
87 return b'#define ZSTD_CHAINLOG_MIN 6' |
|
88 elif l.startswith(b'#define ZSTD_SEARCHLOG_MAX '): |
|
89 return b'#define ZSTD_SEARCHLOG_MAX 26' |
|
90 elif l.startswith(b'#define ZSTD_BLOCKSIZE_ABSOLUTEMAX '): |
|
91 return b'#define ZSTD_BLOCKSIZE_ABSOLUTEMAX 131072' |
|
92 else: |
|
93 return l |
|
94 lines = map(fix_constants, lines) |
|
95 |
|
96 # ZSTDLIB_API isn't handled correctly. Strip it. |
|
97 lines = [l for l in lines if not l.startswith(b'# define ZSTDLIB_API')] |
|
98 def strip_api(l): |
|
99 if l.startswith(b'ZSTDLIB_API '): |
|
100 return l[len(b'ZSTDLIB_API '):] |
|
101 else: |
|
102 return l |
|
103 lines = map(strip_api, lines) |
|
104 |
|
105 source = b'\n'.join(lines) |
|
106 ffi.cdef(source.decode('latin1')) |
|
107 |
|
108 |
106 |
109 if __name__ == '__main__': |
107 if __name__ == '__main__': |
110 ffi.compile() |
108 ffi.compile() |