Mercurial > public > mercurial-scm > hg-stable

comparison contrib/python-zstandard/setup_zstd.py @ 40122:73fef626dae3

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
zstandard: vendor python-zstandard 0.10.1 This was just released. The upstream source distribution from PyPI was extracted. Unwanted files were removed. The clang-format ignore list was updated to reflect the new source of files. setup.py was updated to pass a new argument to python-zstandard's function for returning an Extension instance. Upstream had to change to use relative paths because Python 3.7's packaging doesn't seem to like absolute paths when defining sources, includes, etc. The default relative path calculation is relative to setup_zstd.py which is different from the directory of Mercurial's setup.py. The project contains a vendored copy of zstandard 1.3.6. The old version was 1.3.4. The API should be backwards compatible and nothing in core should need adjusted. However, there is a new "chunker" API that we may find useful in places where we want to emit compressed chunks of a fixed size. There are a pair of bug fixes in 0.10.0 with regards to compressobj() and decompressobj() when block flushing is used. I actually found these bugs when introducing these APIs in Mercurial! But existing Mercurial code is not affected because we don't perform block flushing. # no-check-commit because 3rd party code has different style guidelines Differential Revision: https://phab.mercurial-scm.org/D4911
author Gregory Szorc <gregory.szorc@gmail.com>
date Mon, 08 Oct 2018 16:27:40 -0700
parents c0081d3e1598
children 675775c33ab6
comparison
equal deleted inserted replaced
40121:89742f1fa6cb 40122:73fef626dae3
4 # This software may be modified and distributed under the terms 4 # This software may be modified and distributed under the terms
5 # of the BSD license. See the LICENSE file for details. 5 # of the BSD license. See the LICENSE file for details.
6 6
7 import distutils.ccompiler 7 import distutils.ccompiler
8 import os 8 import os
9 import sys
10 9
11 from distutils.extension import Extension 10 from distutils.extension import Extension
12 11
13 12
14 zstd_sources = ['zstd/%s' % p for p in ( 13 zstd_sources = ['zstd/%s' % p for p in (
14 'common/debug.c',
15 'common/entropy_common.c', 15 'common/entropy_common.c',
16 'common/error_private.c', 16 'common/error_private.c',
17 'common/fse_decompress.c', 17 'common/fse_decompress.c',
18 'common/pool.c', 18 'common/pool.c',
19 'common/threading.c', 19 'common/threading.c',
20 'common/xxhash.c', 20 'common/xxhash.c',
21 'common/zstd_common.c', 21 'common/zstd_common.c',
22 'compress/fse_compress.c', 22 'compress/fse_compress.c',
23 'compress/hist.c',
23 'compress/huf_compress.c', 24 'compress/huf_compress.c',
24 'compress/zstd_compress.c', 25 'compress/zstd_compress.c',
25 'compress/zstd_double_fast.c', 26 'compress/zstd_double_fast.c',
26 'compress/zstd_fast.c', 27 'compress/zstd_fast.c',
27 'compress/zstd_lazy.c', 28 'compress/zstd_lazy.c',
30 'compress/zstdmt_compress.c', 31 'compress/zstdmt_compress.c',
31 'decompress/huf_decompress.c', 32 'decompress/huf_decompress.c',
32 'decompress/zstd_decompress.c', 33 'decompress/zstd_decompress.c',
33 'dictBuilder/cover.c', 34 'dictBuilder/cover.c',
34 'dictBuilder/divsufsort.c', 35 'dictBuilder/divsufsort.c',
36 'dictBuilder/fastcover.c',
35 'dictBuilder/zdict.c', 37 'dictBuilder/zdict.c',
36 )] 38 )]
37 39
38 zstd_sources_legacy = ['zstd/%s' % p for p in ( 40 zstd_sources_legacy = ['zstd/%s' % p for p in (
39 'deprecated/zbuff_common.c', 41 'deprecated/zbuff_common.c',
73 'c-ext/bufferutil.c', 75 'c-ext/bufferutil.c',
74 'c-ext/compressiondict.c', 76 'c-ext/compressiondict.c',
75 'c-ext/compressobj.c', 77 'c-ext/compressobj.c',
76 'c-ext/compressor.c', 78 'c-ext/compressor.c',
77 'c-ext/compressoriterator.c', 79 'c-ext/compressoriterator.c',
80 'c-ext/compressionchunker.c',
78 'c-ext/compressionparams.c', 81 'c-ext/compressionparams.c',
79 'c-ext/compressionreader.c', 82 'c-ext/compressionreader.c',
80 'c-ext/compressionwriter.c', 83 'c-ext/compressionwriter.c',
81 'c-ext/constants.c', 84 'c-ext/constants.c',
82 'c-ext/decompressobj.c', 85 'c-ext/decompressobj.c',
91 'c-ext/python-zstandard.h', 94 'c-ext/python-zstandard.h',
92 ] 95 ]
93 96
94 97
95 def get_c_extension(support_legacy=False, system_zstd=False, name='zstd', 98 def get_c_extension(support_legacy=False, system_zstd=False, name='zstd',
96 warnings_as_errors=False): 99 warnings_as_errors=False, root=None):
97 """Obtain a distutils.extension.Extension for the C extension.""" 100 """Obtain a distutils.extension.Extension for the C extension.
98 root = os.path.abspath(os.path.dirname(__file__))
99 101
100 sources = set([os.path.join(root, p) for p in ext_sources]) 102 ``support_legacy`` controls whether to compile in legacy zstd format support.
103
104 ``system_zstd`` controls whether to compile against the system zstd library.
105 For this to work, the system zstd library and headers must match what
106 python-zstandard is coded against exactly.
107
108 ``name`` is the module name of the C extension to produce.
109
110 ``warnings_as_errors`` controls whether compiler warnings are turned into
111 compiler errors.
112
113 ``root`` defines a root path that source should be computed as relative
114 to. This should be the directory with the main ``setup.py`` that is
115 being invoked. If not defined, paths will be relative to this file.
116 """
117 actual_root = os.path.abspath(os.path.dirname(__file__))
118 root = root or actual_root
119
120 sources = set([os.path.join(actual_root, p) for p in ext_sources])
101 if not system_zstd: 121 if not system_zstd:
102 sources.update([os.path.join(root, p) for p in zstd_sources]) 122 sources.update([os.path.join(actual_root, p) for p in zstd_sources])
103 if support_legacy: 123 if support_legacy:
104 sources.update([os.path.join(root, p) for p in zstd_sources_legacy]) 124 sources.update([os.path.join(actual_root, p)
125 for p in zstd_sources_legacy])
105 sources = list(sources) 126 sources = list(sources)
106 127
107 include_dirs = set([os.path.join(root, d) for d in ext_includes]) 128 include_dirs = set([os.path.join(actual_root, d) for d in ext_includes])
108 if not system_zstd: 129 if not system_zstd:
109 include_dirs.update([os.path.join(root, d) for d in zstd_includes]) 130 include_dirs.update([os.path.join(actual_root, d)
131 for d in zstd_includes])
110 if support_legacy: 132 if support_legacy:
111 include_dirs.update([os.path.join(root, d) for d in zstd_includes_legacy]) 133 include_dirs.update([os.path.join(actual_root, d)
134 for d in zstd_includes_legacy])
112 include_dirs = list(include_dirs) 135 include_dirs = list(include_dirs)
113 136
114 depends = [os.path.join(root, p) for p in zstd_depends] 137 depends = [os.path.join(actual_root, p) for p in zstd_depends]
115 138
116 compiler = distutils.ccompiler.new_compiler() 139 compiler = distutils.ccompiler.new_compiler()
117 140
118 # Needed for MSVC. 141 # Needed for MSVC.
119 if hasattr(compiler, 'initialize'): 142 if hasattr(compiler, 'initialize'):
150 else: 173 else:
151 assert False 174 assert False
152 175
153 libraries = ['zstd'] if system_zstd else [] 176 libraries = ['zstd'] if system_zstd else []
154 177
178 # Python 3.7 doesn't like absolute paths. So normalize to relative.
179 sources = [os.path.relpath(p, root) for p in sources]
180 include_dirs = [os.path.relpath(p, root) for p in include_dirs]
181 depends = [os.path.relpath(p, root) for p in depends]
182
155 # TODO compile with optimizations. 183 # TODO compile with optimizations.
156 return Extension(name, sources, 184 return Extension(name, sources,
157 include_dirs=include_dirs, 185 include_dirs=include_dirs,
158 depends=depends, 186 depends=depends,
159 extra_compile_args=extra_args, 187 extra_compile_args=extra_args,