--- a/Makefile Sat Feb 25 12:48:50 2017 +0900
+++ b/Makefile Tue Feb 28 11:13:25 2017 -0800
@@ -262,5 +262,9 @@
.PHONY: help all local build doc cleanbutpackages clean install install-bin \
install-doc install-home install-home-bin install-home-doc \
dist dist-notests check tests check-code update-pot \
- osx fedora20 docker-fedora20 fedora21 docker-fedora21 \
+ osx deb ppa docker-debian-jessie \
+ docker-ubuntu-trusty docker-ubuntu-trusty-ppa \
+ docker-ubuntu-xenial docker-ubuntu-xenial-ppa \
+ docker-ubuntu-yakkety docker-ubuntu-yakkety-ppa \
+ fedora20 docker-fedora20 fedora21 docker-fedora21 \
centos5 docker-centos5 centos6 docker-centos6 centos7 docker-centos7
--- a/contrib/check-code.py Sat Feb 25 12:48:50 2017 +0900
+++ b/contrib/check-code.py Tue Feb 28 11:13:25 2017 -0800
@@ -237,7 +237,7 @@
(r'lambda\s*\(.*,.*\)',
"tuple parameter unpacking not available in Python 3+"),
(r'(?<!def)\s+(cmp)\(', "cmp is not available in Python 3+"),
- (r'\breduce\s*\(.*', "reduce is not available in Python 3+"),
+ (r'(?<!\.)\breduce\s*\(.*', "reduce is not available in Python 3+"),
(r'\bdict\(.*=', 'dict() is different in Py2 and 3 and is slower than {}',
'dict-from-generator'),
(r'\.has_key\b', "dict.has_key is not available in Python 3+"),
--- a/contrib/chg/chg.c Sat Feb 25 12:48:50 2017 +0900
+++ b/contrib/chg/chg.c Tue Feb 28 11:13:25 2017 -0800
@@ -128,6 +128,24 @@
abortmsg("insecure sockdir %s", sockdir);
}
+/*
+ * Check if a socket directory exists and is only owned by the current user.
+ * Return 1 if so, 0 if not. This is used to check if XDG_RUNTIME_DIR can be
+ * used or not. According to the specification [1], XDG_RUNTIME_DIR should be
+ * ignored if the directory is not owned by the user with mode 0700.
+ * [1]: https://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html
+ */
+static int checkruntimedir(const char *sockdir)
+{
+ struct stat st;
+ int r = lstat(sockdir, &st);
+ if (r < 0) /* ex. does not exist */
+ return 0;
+ if (!S_ISDIR(st.st_mode)) /* ex. is a file, not a directory */
+ return 0;
+ return st.st_uid == geteuid() && (st.st_mode & 0777) == 0700;
+}
+
static void getdefaultsockdir(char sockdir[], size_t size)
{
/* by default, put socket file in secure directory
@@ -135,7 +153,7 @@
* (permission of socket file may be ignored on some Unices) */
const char *runtimedir = getenv("XDG_RUNTIME_DIR");
int r;
- if (runtimedir) {
+ if (runtimedir && checkruntimedir(runtimedir)) {
r = snprintf(sockdir, size, "%s/chg", runtimedir);
} else {
const char *tmpdir = getenv("TMPDIR");
--- a/contrib/hgperf Sat Feb 25 12:48:50 2017 +0900
+++ b/contrib/hgperf Tue Feb 28 11:13:25 2017 -0800
@@ -55,17 +55,15 @@
import mercurial.util
import mercurial.dispatch
-import time
-
def timer(func, title=None):
results = []
- begin = time.time()
+ begin = mercurial.util.timer()
count = 0
while True:
ostart = os.times()
- cstart = time.time()
+ cstart = mercurial.util.timer()
r = func()
- cstop = time.time()
+ cstop = mercurial.util.timer()
ostop = os.times()
count += 1
a, b = ostart, ostop
--- a/contrib/perf.py Sat Feb 25 12:48:50 2017 +0900
+++ b/contrib/perf.py Tue Feb 28 11:13:25 2017 -0800
@@ -190,13 +190,13 @@
def _timer(fm, func, title=None):
results = []
- begin = time.time()
+ begin = util.timer()
count = 0
while True:
ostart = os.times()
- cstart = time.time()
+ cstart = util.timer()
r = func()
- cstop = time.time()
+ cstop = util.timer()
ostop = os.times()
count += 1
a, b = ostart, ostop
@@ -993,6 +993,26 @@
node = r.lookup(rev)
rev = r.rev(node)
+ def getrawchunks(data, chain):
+ start = r.start
+ length = r.length
+ inline = r._inline
+ iosize = r._io.size
+ buffer = util.buffer
+ offset = start(chain[0])
+
+ chunks = []
+ ladd = chunks.append
+
+ for rev in chain:
+ chunkstart = start(rev)
+ if inline:
+ chunkstart += (rev + 1) * iosize
+ chunklength = length(rev)
+ ladd(buffer(data, chunkstart - offset, chunklength))
+
+ return chunks
+
def dodeltachain(rev):
if not cache:
r.clearcaches()
@@ -1003,24 +1023,15 @@
r.clearcaches()
r._chunkraw(chain[0], chain[-1])
- def dodecompress(data, chain):
+ def dorawchunks(data, chain):
if not cache:
r.clearcaches()
-
- start = r.start
- length = r.length
- inline = r._inline
- iosize = r._io.size
- buffer = util.buffer
- offset = start(chain[0])
+ getrawchunks(data, chain)
- for rev in chain:
- chunkstart = start(rev)
- if inline:
- chunkstart += (rev + 1) * iosize
- chunklength = length(rev)
- b = buffer(data, chunkstart - offset, chunklength)
- r.decompress(b)
+ def dodecompress(chunks):
+ decomp = r.decompress
+ for chunk in chunks:
+ decomp(chunk)
def dopatch(text, bins):
if not cache:
@@ -1039,6 +1050,7 @@
chain = r._deltachain(rev)[0]
data = r._chunkraw(chain[0], chain[-1])[1]
+ rawchunks = getrawchunks(data, chain)
bins = r._chunks(chain)
text = str(bins[0])
bins = bins[1:]
@@ -1048,7 +1060,8 @@
(lambda: dorevision(), 'full'),
(lambda: dodeltachain(rev), 'deltachain'),
(lambda: doread(chain), 'read'),
- (lambda: dodecompress(data, chain), 'decompress'),
+ (lambda: dorawchunks(data, chain), 'rawchunks'),
+ (lambda: dodecompress(rawchunks), 'decompress'),
(lambda: dopatch(text, bins), 'patch'),
(lambda: dohash(text), 'hash'),
]
@@ -1256,6 +1269,17 @@
timer(fn, title=title)
fm.end()
+@command('perfwrite', formatteropts)
+def perfwrite(ui, repo, **opts):
+ """microbenchmark ui.write
+ """
+ timer, fm = gettimer(ui, opts)
+ def write():
+ for i in range(100000):
+ ui.write(('Testing write performance\n'))
+ timer(write)
+ fm.end()
+
def uisetup(ui):
if (util.safehasattr(cmdutil, 'openrevlog') and
not util.safehasattr(commands, 'debugrevlogopts')):
--- a/contrib/python-zstandard/NEWS.rst Sat Feb 25 12:48:50 2017 +0900
+++ b/contrib/python-zstandard/NEWS.rst Tue Feb 28 11:13:25 2017 -0800
@@ -1,6 +1,33 @@
Version History
===============
+0.7.0 (released 2017-02-07)
+---------------------------
+
+* Added zstd.get_frame_parameters() to obtain info about a zstd frame.
+* Added ZstdDecompressor.decompress_content_dict_chain() for efficient
+ decompression of *content-only dictionary chains*.
+* CFFI module fully implemented; all tests run against both C extension and
+ CFFI implementation.
+* Vendored version of zstd updated to 1.1.3.
+* Use ZstdDecompressor.decompress() now uses ZSTD_createDDict_byReference()
+ to avoid extra memory allocation of dict data.
+* Add function names to error messages (by using ":name" in PyArg_Parse*
+ functions).
+* Reuse decompression context across operations. Previously, we created a
+ new ZSTD_DCtx for each decompress(). This was measured to slow down
+ decompression by 40-200MB/s. The API guarantees say ZstdDecompressor
+ is not thread safe. So we reuse the ZSTD_DCtx across operations and make
+ things faster in the process.
+* ZstdCompressor.write_to()'s compress() and flush() methods now return number
+ of bytes written.
+* ZstdDecompressor.write_to()'s write() method now returns the number of bytes
+ written to the underlying output object.
+* CompressionParameters instances now expose their values as attributes.
+* CompressionParameters instances no longer are subscriptable nor behave
+ as tuples (backwards incompatible). Use attributes to obtain values.
+* DictParameters instances now expose their values as attributes.
+
0.6.0 (released 2017-01-14)
---------------------------
--- a/contrib/python-zstandard/README.rst Sat Feb 25 12:48:50 2017 +0900
+++ b/contrib/python-zstandard/README.rst Tue Feb 28 11:13:25 2017 -0800
@@ -4,10 +4,11 @@
This project provides Python bindings for interfacing with the
`Zstandard <http://www.zstd.net>`_ compression library. A C extension
-and CFFI interface is provided.
+and CFFI interface are provided.
-The primary goal of the extension is to provide a Pythonic interface to
-the underlying C API. This means exposing most of the features and flexibility
+The primary goal of the project is to provide a rich interface to the
+underlying C API through a Pythonic interface while not sacrificing
+performance. This means exposing most of the features and flexibility
of the C API while not sacrificing usability or safety that Python provides.
The canonical home for this project is
@@ -23,6 +24,9 @@
may be some backwards incompatible changes before 1.0. Though the author
does not intend to make any major changes to the Python API.
+This project is vendored and distributed with Mercurial 4.1, where it is
+used in a production capacity.
+
There is continuous integration for Python versions 2.6, 2.7, and 3.3+
on Linux x86_x64 and Windows x86 and x86_64. The author is reasonably
confident the extension is stable and works as advertised on these
@@ -48,14 +52,15 @@
support compression without the framing headers. But the author doesn't
believe it a high priority at this time.
-The CFFI bindings are half-baked and need to be finished.
+The CFFI bindings are feature complete and all tests run against both
+the C extension and CFFI bindings to ensure behavior parity.
Requirements
============
-This extension is designed to run with Python 2.6, 2.7, 3.3, 3.4, and 3.5
-on common platforms (Linux, Windows, and OS X). Only x86_64 is currently
-well-tested as an architecture.
+This extension is designed to run with Python 2.6, 2.7, 3.3, 3.4, 3.5, and
+3.6 on common platforms (Linux, Windows, and OS X). Only x86_64 is
+currently well-tested as an architecture.
Installing
==========
@@ -106,15 +111,11 @@
Comparison to Other Python Bindings
===================================
-https://pypi.python.org/pypi/zstd is an alternative Python binding to
+https://pypi.python.org/pypi/zstd is an alternate Python binding to
Zstandard. At the time this was written, the latest release of that
-package (1.0.0.2) had the following significant differences from this package:
-
-* It only exposes the simple API for compression and decompression operations.
- This extension exposes the streaming API, dictionary training, and more.
-* It adds a custom framing header to compressed data and there is no way to
- disable it. This means that data produced with that module cannot be used by
- other Zstandard implementations.
+package (1.1.2) only exposed the simple APIs for compression and decompression.
+This package exposes much more of the zstd API, including streaming and
+dictionary compression. This package also has CFFI support.
Bundling of Zstandard Source Code
=================================
@@ -260,6 +261,10 @@
compressor's internal state into the output object. This may result in 0 or
more ``write()`` calls to the output object.
+Both ``write()`` and ``flush()`` return the number of bytes written to the
+object's ``write()``. In many cases, small inputs do not accumulate enough
+data to cause a write and ``write()`` will return ``0``.
+
If the size of the data being fed to this streaming compressor is known,
you can declare it before compression begins::
@@ -476,6 +481,10 @@
the decompressor by calling ``write(data)`` and decompressed output is written
to the output object by calling its ``write(data)`` method.
+Calls to ``write()`` will return the number of bytes written to the output
+object. Not all inputs will result in bytes being written, so return values
+of ``0`` are possible.
+
The size of chunks being ``write()`` to the destination can be specified::
dctx = zstd.ZstdDecompressor()
@@ -576,6 +585,53 @@
data = dobj.decompress(compressed_chunk_0)
data = dobj.decompress(compressed_chunk_1)
+Content-Only Dictionary Chain Decompression
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+``decompress_content_dict_chain(frames)`` performs decompression of a list of
+zstd frames produced using chained *content-only* dictionary compression. Such
+a list of frames is produced by compressing discrete inputs where each
+non-initial input is compressed with a *content-only* dictionary consisting
+of the content of the previous input.
+
+For example, say you have the following inputs::
+
+ inputs = [b'input 1', b'input 2', b'input 3']
+
+The zstd frame chain consists of:
+
+1. ``b'input 1'`` compressed in standalone/discrete mode
+2. ``b'input 2'`` compressed using ``b'input 1'`` as a *content-only* dictionary
+3. ``b'input 3'`` compressed using ``b'input 2'`` as a *content-only* dictionary
+
+Each zstd frame **must** have the content size written.
+
+The following Python code can be used to produce a *content-only dictionary
+chain*::
+
+ def make_chain(inputs):
+ frames = []
+
+ # First frame is compressed in standalone/discrete mode.
+ zctx = zstd.ZstdCompressor(write_content_size=True)
+ frames.append(zctx.compress(inputs[0]))
+
+ # Subsequent frames use the previous fulltext as a content-only dictionary
+ for i, raw in enumerate(inputs[1:]):
+ dict_data = zstd.ZstdCompressionDict(inputs[i])
+ zctx = zstd.ZstdCompressor(write_content_size=True, dict_data=dict_data)
+ frames.append(zctx.compress(raw))
+
+ return frames
+
+``decompress_content_dict_chain()`` returns the uncompressed data of the last
+element in the input chain.
+
+It is possible to implement *content-only dictionary chain* decompression
+on top of other Python APIs. However, this function will likely be significantly
+faster, especially for long input chains, as it avoids the overhead of
+instantiating and passing around intermediate objects between C and Python.
+
Choosing an API
---------------
@@ -634,6 +690,13 @@
dict_data = zstd.ZstdCompressionDict(data)
+It is possible to construct a dictionary from *any* data. Unless the
+data begins with a magic header, the dictionary will be treated as
+*content-only*. *Content-only* dictionaries allow compression operations
+that follow to reference raw data within the content. For one use of
+*content-only* dictionaries, see
+``ZstdDecompressor.decompress_content_dict_chain()``.
+
More interestingly, instances can be created by *training* on sample data::
dict_data = zstd.train_dictionary(size, samples)
@@ -700,19 +763,57 @@
cctx = zstd.ZstdCompressor(compression_params=params)
-The members of the ``CompressionParameters`` tuple are as follows::
+The members/attributes of ``CompressionParameters`` instances are as follows::
-* 0 - Window log
-* 1 - Chain log
-* 2 - Hash log
-* 3 - Search log
-* 4 - Search length
-* 5 - Target length
-* 6 - Strategy (one of the ``zstd.STRATEGY_`` constants)
+* window_log
+* chain_log
+* hash_log
+* search_log
+* search_length
+* target_length
+* strategy
+
+This is the order the arguments are passed to the constructor if not using
+named arguments.
You'll need to read the Zstandard documentation for what these parameters
do.
+Frame Inspection
+----------------
+
+Data emitted from zstd compression is encapsulated in a *frame*. This frame
+begins with a 4 byte *magic number* header followed by 2 to 14 bytes describing
+the frame in more detail. For more info, see
+https://github.com/facebook/zstd/blob/master/doc/zstd_compression_format.md.
+
+``zstd.get_frame_parameters(data)`` parses a zstd *frame* header from a bytes
+instance and return a ``FrameParameters`` object describing the frame.
+
+Depending on which fields are present in the frame and their values, the
+length of the frame parameters varies. If insufficient bytes are passed
+in to fully parse the frame parameters, ``ZstdError`` is raised. To ensure
+frame parameters can be parsed, pass in at least 18 bytes.
+
+``FrameParameters`` instances have the following attributes:
+
+content_size
+ Integer size of original, uncompressed content. This will be ``0`` if the
+ original content size isn't written to the frame (controlled with the
+ ``write_content_size`` argument to ``ZstdCompressor``) or if the input
+ content size was ``0``.
+
+window_size
+ Integer size of maximum back-reference distance in compressed data.
+
+dict_id
+ Integer of dictionary ID used for compression. ``0`` if no dictionary
+ ID was used or if the dictionary ID was ``0``.
+
+has_checksum
+ Bool indicating whether a 4 byte content checksum is stored at the end
+ of the frame.
+
Misc Functionality
------------------
@@ -776,19 +877,32 @@
TARGETLENGTH_MAX
Maximum value for compression parameter
STRATEGY_FAST
- Compression strategory
+ Compression strategy
STRATEGY_DFAST
- Compression strategory
+ Compression strategy
STRATEGY_GREEDY
- Compression strategory
+ Compression strategy
STRATEGY_LAZY
- Compression strategory
+ Compression strategy
STRATEGY_LAZY2
- Compression strategory
+ Compression strategy
STRATEGY_BTLAZY2
- Compression strategory
+ Compression strategy
STRATEGY_BTOPT
- Compression strategory
+ Compression strategy
+
+Performance Considerations
+--------------------------
+
+The ``ZstdCompressor`` and ``ZstdDecompressor`` types maintain state to a
+persistent compression or decompression *context*. Reusing a ``ZstdCompressor``
+or ``ZstdDecompressor`` instance for multiple operations is faster than
+instantiating a new ``ZstdCompressor`` or ``ZstdDecompressor`` for each
+operation. The differences are magnified as the size of data decreases. For
+example, the difference between *context* reuse and non-reuse for 100,000
+100 byte inputs will be significant (possiby over 10x faster to reuse contexts)
+whereas 10 1,000,000 byte inputs will be more similar in speed (because the
+time spent doing compression dwarfs time spent creating new *contexts*).
Note on Zstandard's *Experimental* API
======================================
--- a/contrib/python-zstandard/c-ext/compressiondict.c Sat Feb 25 12:48:50 2017 +0900
+++ b/contrib/python-zstandard/c-ext/compressiondict.c Tue Feb 28 11:13:25 2017 -0800
@@ -28,7 +28,8 @@
void* dict;
ZstdCompressionDict* result;
- if (!PyArg_ParseTupleAndKeywords(args, kwargs, "nO!|O!", kwlist,
+ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "nO!|O!:train_dictionary",
+ kwlist,
&capacity,
&PyList_Type, &samples,
(PyObject*)&DictParametersType, ¶meters)) {
@@ -57,7 +58,6 @@
sampleItem = PyList_GetItem(samples, sampleIndex);
if (!PyBytes_Check(sampleItem)) {
PyErr_SetString(PyExc_ValueError, "samples must be bytes");
- /* TODO probably need to perform DECREF here */
return NULL;
}
samplesSize += PyBytes_GET_SIZE(sampleItem);
@@ -133,10 +133,11 @@
self->dictSize = 0;
#if PY_MAJOR_VERSION >= 3
- if (!PyArg_ParseTuple(args, "y#", &source, &sourceSize)) {
+ if (!PyArg_ParseTuple(args, "y#:ZstdCompressionDict",
#else
- if (!PyArg_ParseTuple(args, "s#", &source, &sourceSize)) {
+ if (!PyArg_ParseTuple(args, "s#:ZstdCompressionDict",
#endif
+ &source, &sourceSize)) {
return -1;
}
--- a/contrib/python-zstandard/c-ext/compressionparams.c Sat Feb 25 12:48:50 2017 +0900
+++ b/contrib/python-zstandard/c-ext/compressionparams.c Tue Feb 28 11:13:25 2017 -0800
@@ -25,7 +25,8 @@
ZSTD_compressionParameters params;
CompressionParametersObject* result;
- if (!PyArg_ParseTuple(args, "i|Kn", &compressionLevel, &sourceSize, &dictSize)) {
+ if (!PyArg_ParseTuple(args, "i|Kn:get_compression_parameters",
+ &compressionLevel, &sourceSize, &dictSize)) {
return NULL;
}
@@ -47,12 +48,85 @@
return result;
}
+static int CompressionParameters_init(CompressionParametersObject* self, PyObject* args, PyObject* kwargs) {
+ static char* kwlist[] = {
+ "window_log",
+ "chain_log",
+ "hash_log",
+ "search_log",
+ "search_length",
+ "target_length",
+ "strategy",
+ NULL
+ };
+
+ unsigned windowLog;
+ unsigned chainLog;
+ unsigned hashLog;
+ unsigned searchLog;
+ unsigned searchLength;
+ unsigned targetLength;
+ unsigned strategy;
+
+ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "IIIIIII:CompressionParameters",
+ kwlist, &windowLog, &chainLog, &hashLog, &searchLog, &searchLength,
+ &targetLength, &strategy)) {
+ return -1;
+ }
+
+ if (windowLog < ZSTD_WINDOWLOG_MIN || windowLog > ZSTD_WINDOWLOG_MAX) {
+ PyErr_SetString(PyExc_ValueError, "invalid window log value");
+ return -1;
+ }
+
+ if (chainLog < ZSTD_CHAINLOG_MIN || chainLog > ZSTD_CHAINLOG_MAX) {
+ PyErr_SetString(PyExc_ValueError, "invalid chain log value");
+ return -1;
+ }
+
+ if (hashLog < ZSTD_HASHLOG_MIN || hashLog > ZSTD_HASHLOG_MAX) {
+ PyErr_SetString(PyExc_ValueError, "invalid hash log value");
+ return -1;
+ }
+
+ if (searchLog < ZSTD_SEARCHLOG_MIN || searchLog > ZSTD_SEARCHLOG_MAX) {
+ PyErr_SetString(PyExc_ValueError, "invalid search log value");
+ return -1;
+ }
+
+ if (searchLength < ZSTD_SEARCHLENGTH_MIN || searchLength > ZSTD_SEARCHLENGTH_MAX) {
+ PyErr_SetString(PyExc_ValueError, "invalid search length value");
+ return -1;
+ }
+
+ if (targetLength < ZSTD_TARGETLENGTH_MIN || targetLength > ZSTD_TARGETLENGTH_MAX) {
+ PyErr_SetString(PyExc_ValueError, "invalid target length value");
+ return -1;
+ }
+
+ if (strategy < ZSTD_fast || strategy > ZSTD_btopt) {
+ PyErr_SetString(PyExc_ValueError, "invalid strategy value");
+ return -1;
+ }
+
+ self->windowLog = windowLog;
+ self->chainLog = chainLog;
+ self->hashLog = hashLog;
+ self->searchLog = searchLog;
+ self->searchLength = searchLength;
+ self->targetLength = targetLength;
+ self->strategy = strategy;
+
+ return 0;
+}
+
PyObject* estimate_compression_context_size(PyObject* self, PyObject* args) {
CompressionParametersObject* params;
ZSTD_compressionParameters zparams;
PyObject* result;
- if (!PyArg_ParseTuple(args, "O!", &CompressionParametersType, ¶ms)) {
+ if (!PyArg_ParseTuple(args, "O!:estimate_compression_context_size",
+ &CompressionParametersType, ¶ms)) {
return NULL;
}
@@ -64,113 +138,33 @@
PyDoc_STRVAR(CompressionParameters__doc__,
"CompressionParameters: low-level control over zstd compression");
-static PyObject* CompressionParameters_new(PyTypeObject* subtype, PyObject* args, PyObject* kwargs) {
- CompressionParametersObject* self;
- unsigned windowLog;
- unsigned chainLog;
- unsigned hashLog;
- unsigned searchLog;
- unsigned searchLength;
- unsigned targetLength;
- unsigned strategy;
-
- if (!PyArg_ParseTuple(args, "IIIIIII", &windowLog, &chainLog, &hashLog, &searchLog,
- &searchLength, &targetLength, &strategy)) {
- return NULL;
- }
-
- if (windowLog < ZSTD_WINDOWLOG_MIN || windowLog > ZSTD_WINDOWLOG_MAX) {
- PyErr_SetString(PyExc_ValueError, "invalid window log value");
- return NULL;
- }
-
- if (chainLog < ZSTD_CHAINLOG_MIN || chainLog > ZSTD_CHAINLOG_MAX) {
- PyErr_SetString(PyExc_ValueError, "invalid chain log value");
- return NULL;
- }
-
- if (hashLog < ZSTD_HASHLOG_MIN || hashLog > ZSTD_HASHLOG_MAX) {
- PyErr_SetString(PyExc_ValueError, "invalid hash log value");
- return NULL;
- }
-
- if (searchLog < ZSTD_SEARCHLOG_MIN || searchLog > ZSTD_SEARCHLOG_MAX) {
- PyErr_SetString(PyExc_ValueError, "invalid search log value");
- return NULL;
- }
-
- if (searchLength < ZSTD_SEARCHLENGTH_MIN || searchLength > ZSTD_SEARCHLENGTH_MAX) {
- PyErr_SetString(PyExc_ValueError, "invalid search length value");
- return NULL;
- }
-
- if (targetLength < ZSTD_TARGETLENGTH_MIN || targetLength > ZSTD_TARGETLENGTH_MAX) {
- PyErr_SetString(PyExc_ValueError, "invalid target length value");
- return NULL;
- }
-
- if (strategy < ZSTD_fast || strategy > ZSTD_btopt) {
- PyErr_SetString(PyExc_ValueError, "invalid strategy value");
- return NULL;
- }
-
- self = (CompressionParametersObject*)subtype->tp_alloc(subtype, 1);
- if (!self) {
- return NULL;
- }
-
- self->windowLog = windowLog;
- self->chainLog = chainLog;
- self->hashLog = hashLog;
- self->searchLog = searchLog;
- self->searchLength = searchLength;
- self->targetLength = targetLength;
- self->strategy = strategy;
-
- return (PyObject*)self;
-}
-
static void CompressionParameters_dealloc(PyObject* self) {
PyObject_Del(self);
}
-static Py_ssize_t CompressionParameters_length(PyObject* self) {
- return 7;
-}
-
-static PyObject* CompressionParameters_item(PyObject* o, Py_ssize_t i) {
- CompressionParametersObject* self = (CompressionParametersObject*)o;
-
- switch (i) {
- case 0:
- return PyLong_FromLong(self->windowLog);
- case 1:
- return PyLong_FromLong(self->chainLog);
- case 2:
- return PyLong_FromLong(self->hashLog);
- case 3:
- return PyLong_FromLong(self->searchLog);
- case 4:
- return PyLong_FromLong(self->searchLength);
- case 5:
- return PyLong_FromLong(self->targetLength);
- case 6:
- return PyLong_FromLong(self->strategy);
- default:
- PyErr_SetString(PyExc_IndexError, "index out of range");
- return NULL;
- }
-}
-
-static PySequenceMethods CompressionParameters_sq = {
- CompressionParameters_length, /* sq_length */
- 0, /* sq_concat */
- 0, /* sq_repeat */
- CompressionParameters_item, /* sq_item */
- 0, /* sq_ass_item */
- 0, /* sq_contains */
- 0, /* sq_inplace_concat */
- 0 /* sq_inplace_repeat */
+static PyMemberDef CompressionParameters_members[] = {
+ { "window_log", T_UINT,
+ offsetof(CompressionParametersObject, windowLog), READONLY,
+ "window log" },
+ { "chain_log", T_UINT,
+ offsetof(CompressionParametersObject, chainLog), READONLY,
+ "chain log" },
+ { "hash_log", T_UINT,
+ offsetof(CompressionParametersObject, hashLog), READONLY,
+ "hash log" },
+ { "search_log", T_UINT,
+ offsetof(CompressionParametersObject, searchLog), READONLY,
+ "search log" },
+ { "search_length", T_UINT,
+ offsetof(CompressionParametersObject, searchLength), READONLY,
+ "search length" },
+ { "target_length", T_UINT,
+ offsetof(CompressionParametersObject, targetLength), READONLY,
+ "target length" },
+ { "strategy", T_INT,
+ offsetof(CompressionParametersObject, strategy), READONLY,
+ "strategy" },
+ { NULL }
};
PyTypeObject CompressionParametersType = {
@@ -185,7 +179,7 @@
0, /* tp_compare */
0, /* tp_repr */
0, /* tp_as_number */
- &CompressionParameters_sq, /* tp_as_sequence */
+ 0, /* tp_as_sequence */
0, /* tp_as_mapping */
0, /* tp_hash */
0, /* tp_call */
@@ -193,7 +187,7 @@
0, /* tp_getattro */
0, /* tp_setattro */
0, /* tp_as_buffer */
- Py_TPFLAGS_DEFAULT, /* tp_flags */
+ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
CompressionParameters__doc__, /* tp_doc */
0, /* tp_traverse */
0, /* tp_clear */
@@ -202,16 +196,16 @@
0, /* tp_iter */
0, /* tp_iternext */
0, /* tp_methods */
- 0, /* tp_members */
+ CompressionParameters_members, /* tp_members */
0, /* tp_getset */
0, /* tp_base */
0, /* tp_dict */
0, /* tp_descr_get */
0, /* tp_descr_set */
0, /* tp_dictoffset */
- 0, /* tp_init */
+ (initproc)CompressionParameters_init, /* tp_init */
0, /* tp_alloc */
- CompressionParameters_new, /* tp_new */
+ PyType_GenericNew, /* tp_new */
};
void compressionparams_module_init(PyObject* mod) {
--- a/contrib/python-zstandard/c-ext/compressionwriter.c Sat Feb 25 12:48:50 2017 +0900
+++ b/contrib/python-zstandard/c-ext/compressionwriter.c Tue Feb 28 11:13:25 2017 -0800
@@ -52,7 +52,7 @@
ZSTD_outBuffer output;
PyObject* res;
- if (!PyArg_ParseTuple(args, "OOO", &exc_type, &exc_value, &exc_tb)) {
+ if (!PyArg_ParseTuple(args, "OOO:__exit__", &exc_type, &exc_value, &exc_tb)) {
return NULL;
}
@@ -119,11 +119,12 @@
ZSTD_inBuffer input;
ZSTD_outBuffer output;
PyObject* res;
+ Py_ssize_t totalWrite = 0;
#if PY_MAJOR_VERSION >= 3
- if (!PyArg_ParseTuple(args, "y#", &source, &sourceSize)) {
+ if (!PyArg_ParseTuple(args, "y#:write", &source, &sourceSize)) {
#else
- if (!PyArg_ParseTuple(args, "s#", &source, &sourceSize)) {
+ if (!PyArg_ParseTuple(args, "s#:write", &source, &sourceSize)) {
#endif
return NULL;
}
@@ -164,20 +165,21 @@
#endif
output.dst, output.pos);
Py_XDECREF(res);
+ totalWrite += output.pos;
}
output.pos = 0;
}
PyMem_Free(output.dst);
- /* TODO return bytes written */
- Py_RETURN_NONE;
+ return PyLong_FromSsize_t(totalWrite);
}
static PyObject* ZstdCompressionWriter_flush(ZstdCompressionWriter* self, PyObject* args) {
size_t zresult;
ZSTD_outBuffer output;
PyObject* res;
+ Py_ssize_t totalWrite = 0;
if (!self->entered) {
PyErr_SetString(ZstdError, "flush must be called from an active context manager");
@@ -215,14 +217,14 @@
#endif
output.dst, output.pos);
Py_XDECREF(res);
+ totalWrite += output.pos;
}
output.pos = 0;
}
PyMem_Free(output.dst);
- /* TODO return bytes written */
- Py_RETURN_NONE;
+ return PyLong_FromSsize_t(totalWrite);
}
static PyMethodDef ZstdCompressionWriter_methods[] = {
--- a/contrib/python-zstandard/c-ext/compressobj.c Sat Feb 25 12:48:50 2017 +0900
+++ b/contrib/python-zstandard/c-ext/compressobj.c Tue Feb 28 11:13:25 2017 -0800
@@ -42,9 +42,9 @@
}
#if PY_MAJOR_VERSION >= 3
- if (!PyArg_ParseTuple(args, "y#", &source, &sourceSize)) {
+ if (!PyArg_ParseTuple(args, "y#:compress", &source, &sourceSize)) {
#else
- if (!PyArg_ParseTuple(args, "s#", &source, &sourceSize)) {
+ if (!PyArg_ParseTuple(args, "s#:compress", &source, &sourceSize)) {
#endif
return NULL;
}
@@ -98,7 +98,7 @@
PyObject* result = NULL;
Py_ssize_t resultSize = 0;
- if (!PyArg_ParseTuple(args, "|i", &flushMode)) {
+ if (!PyArg_ParseTuple(args, "|i:flush", &flushMode)) {
return NULL;
}
--- a/contrib/python-zstandard/c-ext/compressor.c Sat Feb 25 12:48:50 2017 +0900
+++ b/contrib/python-zstandard/c-ext/compressor.c Tue Feb 28 11:13:25 2017 -0800
@@ -16,7 +16,7 @@
Py_BEGIN_ALLOW_THREADS
memset(&zmem, 0, sizeof(zmem));
compressor->cdict = ZSTD_createCDict_advanced(compressor->dict->dictData,
- compressor->dict->dictSize, *zparams, zmem);
+ compressor->dict->dictSize, 1, *zparams, zmem);
Py_END_ALLOW_THREADS
if (!compressor->cdict) {
@@ -128,8 +128,8 @@
self->cparams = NULL;
self->cdict = NULL;
- if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|iO!O!OOO", kwlist,
- &level, &ZstdCompressionDictType, &dict,
+ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|iO!O!OOO:ZstdCompressor",
+ kwlist, &level, &ZstdCompressionDictType, &dict,
&CompressionParametersType, ¶ms,
&writeChecksum, &writeContentSize, &writeDictID)) {
return -1;
@@ -243,8 +243,8 @@
PyObject* totalReadPy;
PyObject* totalWritePy;
- if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OO|nkk", kwlist, &source, &dest, &sourceSize,
- &inSize, &outSize)) {
+ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OO|nkk:copy_stream", kwlist,
+ &source, &dest, &sourceSize, &inSize, &outSize)) {
return NULL;
}
@@ -402,9 +402,9 @@
ZSTD_parameters zparams;
#if PY_MAJOR_VERSION >= 3
- if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y#|O",
+ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y#|O:compress",
#else
- if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s#|O",
+ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s#|O:compress",
#endif
kwlist, &source, &sourceSize, &allowEmpty)) {
return NULL;
@@ -512,7 +512,7 @@
return NULL;
}
- if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|n", kwlist, &inSize)) {
+ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|n:compressobj", kwlist, &inSize)) {
return NULL;
}
@@ -574,8 +574,8 @@
size_t outSize = ZSTD_CStreamOutSize();
ZstdCompressorIterator* result;
- if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|nkk", kwlist, &reader, &sourceSize,
- &inSize, &outSize)) {
+ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|nkk:read_from", kwlist,
+ &reader, &sourceSize, &inSize, &outSize)) {
return NULL;
}
@@ -693,8 +693,8 @@
Py_ssize_t sourceSize = 0;
size_t outSize = ZSTD_CStreamOutSize();
- if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|nk", kwlist, &writer, &sourceSize,
- &outSize)) {
+ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|nk:write_to", kwlist,
+ &writer, &sourceSize, &outSize)) {
return NULL;
}
--- a/contrib/python-zstandard/c-ext/decompressionwriter.c Sat Feb 25 12:48:50 2017 +0900
+++ b/contrib/python-zstandard/c-ext/decompressionwriter.c Tue Feb 28 11:13:25 2017 -0800
@@ -71,11 +71,12 @@
ZSTD_inBuffer input;
ZSTD_outBuffer output;
PyObject* res;
+ Py_ssize_t totalWrite = 0;
#if PY_MAJOR_VERSION >= 3
- if (!PyArg_ParseTuple(args, "y#", &source, &sourceSize)) {
+ if (!PyArg_ParseTuple(args, "y#:write", &source, &sourceSize)) {
#else
- if (!PyArg_ParseTuple(args, "s#", &source, &sourceSize)) {
+ if (!PyArg_ParseTuple(args, "s#:write", &source, &sourceSize)) {
#endif
return NULL;
}
@@ -116,15 +117,15 @@
#endif
output.dst, output.pos);
Py_XDECREF(res);
+ totalWrite += output.pos;
output.pos = 0;
}
}
PyMem_Free(output.dst);
- /* TODO return bytes written */
- Py_RETURN_NONE;
- }
+ return PyLong_FromSsize_t(totalWrite);
+}
static PyMethodDef ZstdDecompressionWriter_methods[] = {
{ "__enter__", (PyCFunction)ZstdDecompressionWriter_enter, METH_NOARGS,
--- a/contrib/python-zstandard/c-ext/decompressobj.c Sat Feb 25 12:48:50 2017 +0900
+++ b/contrib/python-zstandard/c-ext/decompressobj.c Tue Feb 28 11:13:25 2017 -0800
@@ -41,9 +41,9 @@
}
#if PY_MAJOR_VERSION >= 3
- if (!PyArg_ParseTuple(args, "y#",
+ if (!PyArg_ParseTuple(args, "y#:decompress",
#else
- if (!PyArg_ParseTuple(args, "s#",
+ if (!PyArg_ParseTuple(args, "s#:decompress",
#endif
&source, &sourceSize)) {
return NULL;
--- a/contrib/python-zstandard/c-ext/decompressor.c Sat Feb 25 12:48:50 2017 +0900
+++ b/contrib/python-zstandard/c-ext/decompressor.c Tue Feb 28 11:13:25 2017 -0800
@@ -59,23 +59,19 @@
ZstdCompressionDict* dict = NULL;
- self->refdctx = NULL;
+ self->dctx = NULL;
self->dict = NULL;
self->ddict = NULL;
- if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!", kwlist,
+ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!:ZstdDecompressor", kwlist,
&ZstdCompressionDictType, &dict)) {
return -1;
}
- /* Instead of creating a ZSTD_DCtx for every decompression operation,
- we create an instance at object creation time and recycle it via
- ZSTD_copyDCTx() on each use. This means each use is a malloc+memcpy
- instead of a malloc+init. */
/* TODO lazily initialize the reference ZSTD_DCtx on first use since
not instances of ZstdDecompressor will use a ZSTD_DCtx. */
- self->refdctx = ZSTD_createDCtx();
- if (!self->refdctx) {
+ self->dctx = ZSTD_createDCtx();
+ if (!self->dctx) {
PyErr_NoMemory();
goto except;
}
@@ -88,17 +84,17 @@
return 0;
except:
- if (self->refdctx) {
- ZSTD_freeDCtx(self->refdctx);
- self->refdctx = NULL;
+ if (self->dctx) {
+ ZSTD_freeDCtx(self->dctx);
+ self->dctx = NULL;
}
return -1;
}
static void Decompressor_dealloc(ZstdDecompressor* self) {
- if (self->refdctx) {
- ZSTD_freeDCtx(self->refdctx);
+ if (self->dctx) {
+ ZSTD_freeDCtx(self->dctx);
}
Py_XDECREF(self->dict);
@@ -150,8 +146,8 @@
PyObject* totalReadPy;
PyObject* totalWritePy;
- if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OO|kk", kwlist, &source,
- &dest, &inSize, &outSize)) {
+ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OO|kk:copy_stream", kwlist,
+ &source, &dest, &inSize, &outSize)) {
return NULL;
}
@@ -243,7 +239,7 @@
Py_DecRef(totalReadPy);
Py_DecRef(totalWritePy);
- finally:
+finally:
if (output.dst) {
PyMem_Free(output.dst);
}
@@ -291,28 +287,19 @@
unsigned long long decompressedSize;
size_t destCapacity;
PyObject* result = NULL;
- ZSTD_DCtx* dctx = NULL;
void* dictData = NULL;
size_t dictSize = 0;
size_t zresult;
#if PY_MAJOR_VERSION >= 3
- if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y#|n", kwlist,
+ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y#|n:decompress",
#else
- if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s#|n", kwlist,
+ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s#|n:decompress",
#endif
- &source, &sourceSize, &maxOutputSize)) {
+ kwlist, &source, &sourceSize, &maxOutputSize)) {
return NULL;
}
- dctx = PyMem_Malloc(ZSTD_sizeof_DCtx(self->refdctx));
- if (!dctx) {
- PyErr_NoMemory();
- return NULL;
- }
-
- ZSTD_copyDCtx(dctx, self->refdctx);
-
if (self->dict) {
dictData = self->dict->dictData;
dictSize = self->dict->dictSize;
@@ -320,12 +307,12 @@
if (dictData && !self->ddict) {
Py_BEGIN_ALLOW_THREADS
- self->ddict = ZSTD_createDDict(dictData, dictSize);
+ self->ddict = ZSTD_createDDict_byReference(dictData, dictSize);
Py_END_ALLOW_THREADS
if (!self->ddict) {
PyErr_SetString(ZstdError, "could not create decompression dict");
- goto except;
+ return NULL;
}
}
@@ -335,7 +322,7 @@
if (0 == maxOutputSize) {
PyErr_SetString(ZstdError, "input data invalid or missing content size "
"in frame header");
- goto except;
+ return NULL;
}
else {
result = PyBytes_FromStringAndSize(NULL, maxOutputSize);
@@ -348,45 +335,39 @@
}
if (!result) {
- goto except;
+ return NULL;
}
Py_BEGIN_ALLOW_THREADS
if (self->ddict) {
- zresult = ZSTD_decompress_usingDDict(dctx, PyBytes_AsString(result), destCapacity,
+ zresult = ZSTD_decompress_usingDDict(self->dctx,
+ PyBytes_AsString(result), destCapacity,
source, sourceSize, self->ddict);
}
else {
- zresult = ZSTD_decompressDCtx(dctx, PyBytes_AsString(result), destCapacity, source, sourceSize);
+ zresult = ZSTD_decompressDCtx(self->dctx,
+ PyBytes_AsString(result), destCapacity, source, sourceSize);
}
Py_END_ALLOW_THREADS
if (ZSTD_isError(zresult)) {
PyErr_Format(ZstdError, "decompression error: %s", ZSTD_getErrorName(zresult));
- goto except;
+ Py_DecRef(result);
+ return NULL;
}
else if (decompressedSize && zresult != decompressedSize) {
PyErr_Format(ZstdError, "decompression error: decompressed %zu bytes; expected %llu",
zresult, decompressedSize);
- goto except;
+ Py_DecRef(result);
+ return NULL;
}
else if (zresult < destCapacity) {
if (_PyBytes_Resize(&result, zresult)) {
- goto except;
+ Py_DecRef(result);
+ return NULL;
}
}
- goto finally;
-
-except:
- Py_DecRef(result);
- result = NULL;
-
-finally:
- if (dctx) {
- PyMem_FREE(dctx);
- }
-
return result;
}
@@ -455,8 +436,8 @@
ZstdDecompressorIterator* result;
size_t skipBytes = 0;
- if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|kkk", kwlist, &reader,
- &inSize, &outSize, &skipBytes)) {
+ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|kkk:read_from", kwlist,
+ &reader, &inSize, &outSize, &skipBytes)) {
return NULL;
}
@@ -534,19 +515,14 @@
goto finally;
except:
- if (result->reader) {
- Py_DECREF(result->reader);
- result->reader = NULL;
- }
+ Py_CLEAR(result->reader);
if (result->buffer) {
PyBuffer_Release(result->buffer);
- Py_DECREF(result->buffer);
- result->buffer = NULL;
+ Py_CLEAR(result->buffer);
}
- Py_DECREF(result);
- result = NULL;
+ Py_CLEAR(result);
finally:
@@ -577,7 +553,8 @@
size_t outSize = ZSTD_DStreamOutSize();
ZstdDecompressionWriter* result;
- if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|k", kwlist, &writer, &outSize)) {
+ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|k:write_to", kwlist,
+ &writer, &outSize)) {
return NULL;
}
@@ -605,6 +582,200 @@
return result;
}
+PyDoc_STRVAR(Decompressor_decompress_content_dict_chain__doc__,
+"Decompress a series of chunks using the content dictionary chaining technique\n"
+);
+
+static PyObject* Decompressor_decompress_content_dict_chain(PyObject* self, PyObject* args, PyObject* kwargs) {
+ static char* kwlist[] = {
+ "frames",
+ NULL
+ };
+
+ PyObject* chunks;
+ Py_ssize_t chunksLen;
+ Py_ssize_t chunkIndex;
+ char parity = 0;
+ PyObject* chunk;
+ char* chunkData;
+ Py_ssize_t chunkSize;
+ ZSTD_DCtx* dctx = NULL;
+ size_t zresult;
+ ZSTD_frameParams frameParams;
+ void* buffer1 = NULL;
+ size_t buffer1Size = 0;
+ size_t buffer1ContentSize = 0;
+ void* buffer2 = NULL;
+ size_t buffer2Size = 0;
+ size_t buffer2ContentSize = 0;
+ void* destBuffer = NULL;
+ PyObject* result = NULL;
+
+ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O!:decompress_content_dict_chain",
+ kwlist, &PyList_Type, &chunks)) {
+ return NULL;
+ }
+
+ chunksLen = PyList_Size(chunks);
+ if (!chunksLen) {
+ PyErr_SetString(PyExc_ValueError, "empty input chain");
+ return NULL;
+ }
+
+ /* The first chunk should not be using a dictionary. We handle it specially. */
+ chunk = PyList_GetItem(chunks, 0);
+ if (!PyBytes_Check(chunk)) {
+ PyErr_SetString(PyExc_ValueError, "chunk 0 must be bytes");
+ return NULL;
+ }
+
+ /* We require that all chunks be zstd frames and that they have content size set. */
+ PyBytes_AsStringAndSize(chunk, &chunkData, &chunkSize);
+ zresult = ZSTD_getFrameParams(&frameParams, (void*)chunkData, chunkSize);
+ if (ZSTD_isError(zresult)) {
+ PyErr_SetString(PyExc_ValueError, "chunk 0 is not a valid zstd frame");
+ return NULL;
+ }
+ else if (zresult) {
+ PyErr_SetString(PyExc_ValueError, "chunk 0 is too small to contain a zstd frame");
+ return NULL;
+ }
+
+ if (0 == frameParams.frameContentSize) {
+ PyErr_SetString(PyExc_ValueError, "chunk 0 missing content size in frame");
+ return NULL;
+ }
+
+ dctx = ZSTD_createDCtx();
+ if (!dctx) {
+ PyErr_NoMemory();
+ goto finally;
+ }
+
+ buffer1Size = frameParams.frameContentSize;
+ buffer1 = PyMem_Malloc(buffer1Size);
+ if (!buffer1) {
+ goto finally;
+ }
+
+ Py_BEGIN_ALLOW_THREADS
+ zresult = ZSTD_decompressDCtx(dctx, buffer1, buffer1Size, chunkData, chunkSize);
+ Py_END_ALLOW_THREADS
+ if (ZSTD_isError(zresult)) {
+ PyErr_Format(ZstdError, "could not decompress chunk 0: %s", ZSTD_getErrorName(zresult));
+ goto finally;
+ }
+
+ buffer1ContentSize = zresult;
+
+ /* Special case of a simple chain. */
+ if (1 == chunksLen) {
+ result = PyBytes_FromStringAndSize(buffer1, buffer1Size);
+ goto finally;
+ }
+
+ /* This should ideally look at next chunk. But this is slightly simpler. */
+ buffer2Size = frameParams.frameContentSize;
+ buffer2 = PyMem_Malloc(buffer2Size);
+ if (!buffer2) {
+ goto finally;
+ }
+
+ /* For each subsequent chunk, use the previous fulltext as a content dictionary.
+ Our strategy is to have 2 buffers. One holds the previous fulltext (to be
+ used as a content dictionary) and the other holds the new fulltext. The
+ buffers grow when needed but never decrease in size. This limits the
+ memory allocator overhead.
+ */
+ for (chunkIndex = 1; chunkIndex < chunksLen; chunkIndex++) {
+ chunk = PyList_GetItem(chunks, chunkIndex);
+ if (!PyBytes_Check(chunk)) {
+ PyErr_Format(PyExc_ValueError, "chunk %zd must be bytes", chunkIndex);
+ goto finally;
+ }
+
+ PyBytes_AsStringAndSize(chunk, &chunkData, &chunkSize);
+ zresult = ZSTD_getFrameParams(&frameParams, (void*)chunkData, chunkSize);
+ if (ZSTD_isError(zresult)) {
+ PyErr_Format(PyExc_ValueError, "chunk %zd is not a valid zstd frame", chunkIndex);
+ goto finally;
+ }
+ else if (zresult) {
+ PyErr_Format(PyExc_ValueError, "chunk %zd is too small to contain a zstd frame", chunkIndex);
+ goto finally;
+ }
+
+ if (0 == frameParams.frameContentSize) {
+ PyErr_Format(PyExc_ValueError, "chunk %zd missing content size in frame", chunkIndex);
+ goto finally;
+ }
+
+ parity = chunkIndex % 2;
+
+ /* This could definitely be abstracted to reduce code duplication. */
+ if (parity) {
+ /* Resize destination buffer to hold larger content. */
+ if (buffer2Size < frameParams.frameContentSize) {
+ buffer2Size = frameParams.frameContentSize;
+ destBuffer = PyMem_Realloc(buffer2, buffer2Size);
+ if (!destBuffer) {
+ goto finally;
+ }
+ buffer2 = destBuffer;
+ }
+
+ Py_BEGIN_ALLOW_THREADS
+ zresult = ZSTD_decompress_usingDict(dctx, buffer2, buffer2Size,
+ chunkData, chunkSize, buffer1, buffer1ContentSize);
+ Py_END_ALLOW_THREADS
+ if (ZSTD_isError(zresult)) {
+ PyErr_Format(ZstdError, "could not decompress chunk %zd: %s",
+ chunkIndex, ZSTD_getErrorName(zresult));
+ goto finally;
+ }
+ buffer2ContentSize = zresult;
+ }
+ else {
+ if (buffer1Size < frameParams.frameContentSize) {
+ buffer1Size = frameParams.frameContentSize;
+ destBuffer = PyMem_Realloc(buffer1, buffer1Size);
+ if (!destBuffer) {
+ goto finally;
+ }
+ buffer1 = destBuffer;
+ }
+
+ Py_BEGIN_ALLOW_THREADS
+ zresult = ZSTD_decompress_usingDict(dctx, buffer1, buffer1Size,
+ chunkData, chunkSize, buffer2, buffer2ContentSize);
+ Py_END_ALLOW_THREADS
+ if (ZSTD_isError(zresult)) {
+ PyErr_Format(ZstdError, "could not decompress chunk %zd: %s",
+ chunkIndex, ZSTD_getErrorName(zresult));
+ goto finally;
+ }
+ buffer1ContentSize = zresult;
+ }
+ }
+
+ result = PyBytes_FromStringAndSize(parity ? buffer2 : buffer1,
+ parity ? buffer2ContentSize : buffer1ContentSize);
+
+finally:
+ if (buffer2) {
+ PyMem_Free(buffer2);
+ }
+ if (buffer1) {
+ PyMem_Free(buffer1);
+ }
+
+ if (dctx) {
+ ZSTD_freeDCtx(dctx);
+ }
+
+ return result;
+}
+
static PyMethodDef Decompressor_methods[] = {
{ "copy_stream", (PyCFunction)Decompressor_copy_stream, METH_VARARGS | METH_KEYWORDS,
Decompressor_copy_stream__doc__ },
@@ -616,6 +787,8 @@
Decompressor_read_from__doc__ },
{ "write_to", (PyCFunction)Decompressor_write_to, METH_VARARGS | METH_KEYWORDS,
Decompressor_write_to__doc__ },
+ { "decompress_content_dict_chain", (PyCFunction)Decompressor_decompress_content_dict_chain,
+ METH_VARARGS | METH_KEYWORDS, Decompressor_decompress_content_dict_chain__doc__ },
{ NULL, NULL }
};
--- a/contrib/python-zstandard/c-ext/dictparams.c Sat Feb 25 12:48:50 2017 +0900
+++ b/contrib/python-zstandard/c-ext/dictparams.c Tue Feb 28 11:13:25 2017 -0800
@@ -18,8 +18,8 @@
unsigned notificationLevel;
unsigned dictID;
- if (!PyArg_ParseTuple(args, "IiII", &selectivityLevel, &compressionLevel,
- ¬ificationLevel, &dictID)) {
+ if (!PyArg_ParseTuple(args, "IiII:DictParameters",
+ &selectivityLevel, &compressionLevel, ¬ificationLevel, &dictID)) {
return NULL;
}
@@ -40,6 +40,22 @@
PyObject_Del(self);
}
+static PyMemberDef DictParameters_members[] = {
+ { "selectivity_level", T_UINT,
+ offsetof(DictParametersObject, selectivityLevel), READONLY,
+ "selectivity level" },
+ { "compression_level", T_INT,
+ offsetof(DictParametersObject, compressionLevel), READONLY,
+ "compression level" },
+ { "notification_level", T_UINT,
+ offsetof(DictParametersObject, notificationLevel), READONLY,
+ "notification level" },
+ { "dict_id", T_UINT,
+ offsetof(DictParametersObject, dictID), READONLY,
+ "dictionary ID" },
+ { NULL }
+};
+
static Py_ssize_t DictParameters_length(PyObject* self) {
return 4;
}
@@ -102,7 +118,7 @@
0, /* tp_iter */
0, /* tp_iternext */
0, /* tp_methods */
- 0, /* tp_members */
+ DictParameters_members, /* tp_members */
0, /* tp_getset */
0, /* tp_base */
0, /* tp_dict */
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/contrib/python-zstandard/c-ext/frameparams.c Tue Feb 28 11:13:25 2017 -0800
@@ -0,0 +1,132 @@
+/**
+* Copyright (c) 2017-present, Gregory Szorc
+* All rights reserved.
+*
+* This software may be modified and distributed under the terms
+* of the BSD license. See the LICENSE file for details.
+*/
+
+#include "python-zstandard.h"
+
+extern PyObject* ZstdError;
+
+PyDoc_STRVAR(FrameParameters__doc__,
+ "FrameParameters: information about a zstd frame");
+
+FrameParametersObject* get_frame_parameters(PyObject* self, PyObject* args) {
+ const char* source;
+ Py_ssize_t sourceSize;
+ ZSTD_frameParams params;
+ FrameParametersObject* result = NULL;
+ size_t zresult;
+
+#if PY_MAJOR_VERSION >= 3
+ if (!PyArg_ParseTuple(args, "y#:get_frame_parameters",
+#else
+ if (!PyArg_ParseTuple(args, "s#:get_frame_parameters",
+#endif
+ &source, &sourceSize)) {
+ return NULL;
+ }
+
+ /* Needed for Python 2 to reject unicode */
+ if (!PyBytes_Check(PyTuple_GET_ITEM(args, 0))) {
+ PyErr_SetString(PyExc_TypeError, "argument must be bytes");
+ return NULL;
+ }
+
+ zresult = ZSTD_getFrameParams(¶ms, (void*)source, sourceSize);
+
+ if (ZSTD_isError(zresult)) {
+ PyErr_Format(ZstdError, "cannot get frame parameters: %s", ZSTD_getErrorName(zresult));
+ return NULL;
+ }
+
+ if (zresult) {
+ PyErr_Format(ZstdError, "not enough data for frame parameters; need %zu bytes", zresult);
+ return NULL;
+ }
+
+ result = PyObject_New(FrameParametersObject, &FrameParametersType);
+ if (!result) {
+ return NULL;
+ }
+
+ result->frameContentSize = params.frameContentSize;
+ result->windowSize = params.windowSize;
+ result->dictID = params.dictID;
+ result->checksumFlag = params.checksumFlag ? 1 : 0;
+
+ return result;
+}
+
+static void FrameParameters_dealloc(PyObject* self) {
+ PyObject_Del(self);
+}
+
+static PyMemberDef FrameParameters_members[] = {
+ { "content_size", T_ULONGLONG,
+ offsetof(FrameParametersObject, frameContentSize), READONLY,
+ "frame content size" },
+ { "window_size", T_UINT,
+ offsetof(FrameParametersObject, windowSize), READONLY,
+ "window size" },
+ { "dict_id", T_UINT,
+ offsetof(FrameParametersObject, dictID), READONLY,
+ "dictionary ID" },
+ { "has_checksum", T_BOOL,
+ offsetof(FrameParametersObject, checksumFlag), READONLY,
+ "checksum flag" },
+ { NULL }
+};
+
+PyTypeObject FrameParametersType = {
+ PyVarObject_HEAD_INIT(NULL, 0)
+ "FrameParameters", /* tp_name */
+ sizeof(FrameParametersObject), /* tp_basicsize */
+ 0, /* tp_itemsize */
+ (destructor)FrameParameters_dealloc, /* tp_dealloc */
+ 0, /* tp_print */
+ 0, /* tp_getattr */
+ 0, /* tp_setattr */
+ 0, /* tp_compare */
+ 0, /* tp_repr */
+ 0, /* tp_as_number */
+ 0, /* tp_as_sequence */
+ 0, /* tp_as_mapping */
+ 0, /* tp_hash */
+ 0, /* tp_call */
+ 0, /* tp_str */
+ 0, /* tp_getattro */
+ 0, /* tp_setattro */
+ 0, /* tp_as_buffer */
+ Py_TPFLAGS_DEFAULT, /* tp_flags */
+ FrameParameters__doc__, /* tp_doc */
+ 0, /* tp_traverse */
+ 0, /* tp_clear */
+ 0, /* tp_richcompare */
+ 0, /* tp_weaklistoffset */
+ 0, /* tp_iter */
+ 0, /* tp_iternext */
+ 0, /* tp_methods */
+ FrameParameters_members, /* tp_members */
+ 0, /* tp_getset */
+ 0, /* tp_base */
+ 0, /* tp_dict */
+ 0, /* tp_descr_get */
+ 0, /* tp_descr_set */
+ 0, /* tp_dictoffset */
+ 0, /* tp_init */
+ 0, /* tp_alloc */
+ 0, /* tp_new */
+};
+
+void frameparams_module_init(PyObject* mod) {
+ Py_TYPE(&FrameParametersType) = &PyType_Type;
+ if (PyType_Ready(&FrameParametersType) < 0) {
+ return;
+ }
+
+ Py_IncRef((PyObject*)&FrameParametersType);
+ PyModule_AddObject(mod, "FrameParameters", (PyObject*)&FrameParametersType);
+}
--- a/contrib/python-zstandard/c-ext/python-zstandard.h Sat Feb 25 12:48:50 2017 +0900
+++ b/contrib/python-zstandard/c-ext/python-zstandard.h Tue Feb 28 11:13:25 2017 -0800
@@ -8,6 +8,7 @@
#define PY_SSIZE_T_CLEAN
#include <Python.h>
+#include "structmember.h"
#define ZSTD_STATIC_LINKING_ONLY
#define ZDICT_STATIC_LINKING_ONLY
@@ -15,7 +16,7 @@
#include "zstd.h"
#include "zdict.h"
-#define PYTHON_ZSTANDARD_VERSION "0.6.0"
+#define PYTHON_ZSTANDARD_VERSION "0.7.0"
typedef enum {
compressorobj_flush_finish,
@@ -37,6 +38,16 @@
typedef struct {
PyObject_HEAD
+ unsigned long long frameContentSize;
+ unsigned windowSize;
+ unsigned dictID;
+ char checksumFlag;
+} FrameParametersObject;
+
+extern PyTypeObject FrameParametersType;
+
+typedef struct {
+ PyObject_HEAD
unsigned selectivityLevel;
int compressionLevel;
unsigned notificationLevel;
@@ -115,7 +126,7 @@
typedef struct {
PyObject_HEAD
- ZSTD_DCtx* refdctx;
+ ZSTD_DCtx* dctx;
ZstdCompressionDict* dict;
ZSTD_DDict* ddict;
@@ -172,6 +183,7 @@
void ztopy_compression_parameters(CompressionParametersObject* params, ZSTD_compressionParameters* zparams);
CompressionParametersObject* get_compression_parameters(PyObject* self, PyObject* args);
+FrameParametersObject* get_frame_parameters(PyObject* self, PyObject* args);
PyObject* estimate_compression_context_size(PyObject* self, PyObject* args);
ZSTD_CStream* CStream_from_ZstdCompressor(ZstdCompressor* compressor, Py_ssize_t sourceSize);
ZSTD_DStream* DStream_from_ZstdDecompressor(ZstdDecompressor* decompressor);
--- a/contrib/python-zstandard/make_cffi.py Sat Feb 25 12:48:50 2017 +0900
+++ b/contrib/python-zstandard/make_cffi.py Tue Feb 28 11:13:25 2017 -0800
@@ -9,6 +9,7 @@
import cffi
import distutils.ccompiler
import os
+import re
import subprocess
import tempfile
@@ -19,6 +20,8 @@
'common/entropy_common.c',
'common/error_private.c',
'common/fse_decompress.c',
+ 'common/pool.c',
+ 'common/threading.c',
'common/xxhash.c',
'common/zstd_common.c',
'compress/fse_compress.c',
@@ -26,10 +29,17 @@
'compress/zstd_compress.c',
'decompress/huf_decompress.c',
'decompress/zstd_decompress.c',
+ 'dictBuilder/cover.c',
'dictBuilder/divsufsort.c',
'dictBuilder/zdict.c',
)]
+HEADERS = [os.path.join(HERE, 'zstd', *p) for p in (
+ ('zstd.h',),
+ ('common', 'pool.h'),
+ ('dictBuilder', 'zdict.h'),
+)]
+
INCLUDE_DIRS = [os.path.join(HERE, d) for d in (
'zstd',
'zstd/common',
@@ -53,56 +63,92 @@
args.extend([
'-E',
'-DZSTD_STATIC_LINKING_ONLY',
+ '-DZDICT_STATIC_LINKING_ONLY',
])
elif compiler.compiler_type == 'msvc':
args = [compiler.cc]
args.extend([
'/EP',
'/DZSTD_STATIC_LINKING_ONLY',
+ '/DZDICT_STATIC_LINKING_ONLY',
])
else:
raise Exception('unsupported compiler type: %s' % compiler.compiler_type)
-# zstd.h includes <stddef.h>, which is also included by cffi's boilerplate.
-# This can lead to duplicate declarations. So we strip this include from the
-# preprocessor invocation.
+def preprocess(path):
+ # zstd.h includes <stddef.h>, which is also included by cffi's boilerplate.
+ # This can lead to duplicate declarations. So we strip this include from the
+ # preprocessor invocation.
+ with open(path, 'rb') as fh:
+ lines = [l for l in fh if not l.startswith(b'#include <stddef.h>')]
-with open(os.path.join(HERE, 'zstd', 'zstd.h'), 'rb') as fh:
- lines = [l for l in fh if not l.startswith(b'#include <stddef.h>')]
-
-fd, input_file = tempfile.mkstemp(suffix='.h')
-os.write(fd, b''.join(lines))
-os.close(fd)
+ fd, input_file = tempfile.mkstemp(suffix='.h')
+ os.write(fd, b''.join(lines))
+ os.close(fd)
-args.append(input_file)
+ try:
+ process = subprocess.Popen(args + [input_file], stdout=subprocess.PIPE)
+ output = process.communicate()[0]
+ ret = process.poll()
+ if ret:
+ raise Exception('preprocessor exited with error')
-try:
- process = subprocess.Popen(args, stdout=subprocess.PIPE)
- output = process.communicate()[0]
- ret = process.poll()
- if ret:
- raise Exception('preprocessor exited with error')
-finally:
- os.unlink(input_file)
+ return output
+ finally:
+ os.unlink(input_file)
-def normalize_output():
+
+def normalize_output(output):
lines = []
for line in output.splitlines():
# CFFI's parser doesn't like __attribute__ on UNIX compilers.
if line.startswith(b'__attribute__ ((visibility ("default"))) '):
line = line[len(b'__attribute__ ((visibility ("default"))) '):]
+ if line.startswith(b'__attribute__((deprecated('):
+ continue
+ elif b'__declspec(deprecated(' in line:
+ continue
+
lines.append(line)
return b'\n'.join(lines)
+
ffi = cffi.FFI()
ffi.set_source('_zstd_cffi', '''
+#include "mem.h"
#define ZSTD_STATIC_LINKING_ONLY
#include "zstd.h"
+#define ZDICT_STATIC_LINKING_ONLY
+#include "pool.h"
+#include "zdict.h"
''', sources=SOURCES, include_dirs=INCLUDE_DIRS)
-ffi.cdef(normalize_output().decode('latin1'))
+DEFINE = re.compile(b'^\\#define ([a-zA-Z0-9_]+) ')
+
+sources = []
+
+for header in HEADERS:
+ preprocessed = preprocess(header)
+ sources.append(normalize_output(preprocessed))
+
+ # Do another pass over source and find constants that were preprocessed
+ # away.
+ with open(header, 'rb') as fh:
+ for line in fh:
+ line = line.strip()
+ m = DEFINE.match(line)
+ if not m:
+ continue
+
+ # The parser doesn't like some constants with complex values.
+ if m.group(1) in (b'ZSTD_LIB_VERSION', b'ZSTD_VERSION_STRING'):
+ continue
+
+ sources.append(m.group(0) + b' ...')
+
+ffi.cdef(u'\n'.join(s.decode('latin1') for s in sources))
if __name__ == '__main__':
ffi.compile()
--- a/contrib/python-zstandard/setup.py Sat Feb 25 12:48:50 2017 +0900
+++ b/contrib/python-zstandard/setup.py Tue Feb 28 11:13:25 2017 -0800
@@ -62,6 +62,7 @@
'Programming Language :: Python :: 3.3',
'Programming Language :: Python :: 3.4',
'Programming Language :: Python :: 3.5',
+ 'Programming Language :: Python :: 3.6',
],
keywords='zstandard zstd compression',
ext_modules=extensions,
--- a/contrib/python-zstandard/setup_zstd.py Sat Feb 25 12:48:50 2017 +0900
+++ b/contrib/python-zstandard/setup_zstd.py Tue Feb 28 11:13:25 2017 -0800
@@ -12,6 +12,8 @@
'common/entropy_common.c',
'common/error_private.c',
'common/fse_decompress.c',
+ 'common/pool.c',
+ 'common/threading.c',
'common/xxhash.c',
'common/zstd_common.c',
'compress/fse_compress.c',
@@ -19,11 +21,13 @@
'compress/zstd_compress.c',
'decompress/huf_decompress.c',
'decompress/zstd_decompress.c',
+ 'dictBuilder/cover.c',
'dictBuilder/divsufsort.c',
'dictBuilder/zdict.c',
)]
zstd_sources_legacy = ['zstd/%s' % p for p in (
+ 'deprecated/zbuff_common.c',
'deprecated/zbuff_compress.c',
'deprecated/zbuff_decompress.c',
'legacy/zstd_v01.c',
@@ -63,6 +67,7 @@
'c-ext/decompressoriterator.c',
'c-ext/decompressionwriter.c',
'c-ext/dictparams.c',
+ 'c-ext/frameparams.c',
]
zstd_depends = [
--- a/contrib/python-zstandard/tests/common.py Sat Feb 25 12:48:50 2017 +0900
+++ b/contrib/python-zstandard/tests/common.py Tue Feb 28 11:13:25 2017 -0800
@@ -1,4 +1,50 @@
+import inspect
import io
+import types
+
+
+def make_cffi(cls):
+ """Decorator to add CFFI versions of each test method."""
+
+ try:
+ import zstd_cffi
+ except ImportError:
+ return cls
+
+ # If CFFI version is available, dynamically construct test methods
+ # that use it.
+
+ for attr in dir(cls):
+ fn = getattr(cls, attr)
+ if not inspect.ismethod(fn) and not inspect.isfunction(fn):
+ continue
+
+ if not fn.__name__.startswith('test_'):
+ continue
+
+ name = '%s_cffi' % fn.__name__
+
+ # Replace the "zstd" symbol with the CFFI module instance. Then copy
+ # the function object and install it in a new attribute.
+ if isinstance(fn, types.FunctionType):
+ globs = dict(fn.__globals__)
+ globs['zstd'] = zstd_cffi
+ new_fn = types.FunctionType(fn.__code__, globs, name,
+ fn.__defaults__, fn.__closure__)
+ new_method = new_fn
+ else:
+ globs = dict(fn.__func__.func_globals)
+ globs['zstd'] = zstd_cffi
+ new_fn = types.FunctionType(fn.__func__.func_code, globs, name,
+ fn.__func__.func_defaults,
+ fn.__func__.func_closure)
+ new_method = types.UnboundMethodType(new_fn, fn.im_self,
+ fn.im_class)
+
+ setattr(cls, name, new_method)
+
+ return cls
+
class OpCountingBytesIO(io.BytesIO):
def __init__(self, *args, **kwargs):
--- a/contrib/python-zstandard/tests/test_cffi.py Sat Feb 25 12:48:50 2017 +0900
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,35 +0,0 @@
-import io
-
-try:
- import unittest2 as unittest
-except ImportError:
- import unittest
-
-import zstd
-
-try:
- import zstd_cffi
-except ImportError:
- raise unittest.SkipTest('cffi version of zstd not available')
-
-
-class TestCFFIWriteToToCDecompressor(unittest.TestCase):
- def test_simple(self):
- orig = io.BytesIO()
- orig.write(b'foo')
- orig.write(b'bar')
- orig.write(b'foobar' * 16384)
-
- dest = io.BytesIO()
- cctx = zstd_cffi.ZstdCompressor()
- with cctx.write_to(dest) as compressor:
- compressor.write(orig.getvalue())
-
- uncompressed = io.BytesIO()
- dctx = zstd.ZstdDecompressor()
- with dctx.write_to(uncompressed) as decompressor:
- decompressor.write(dest.getvalue())
-
- self.assertEqual(uncompressed.getvalue(), orig.getvalue())
-
-
--- a/contrib/python-zstandard/tests/test_compressor.py Sat Feb 25 12:48:50 2017 +0900
+++ b/contrib/python-zstandard/tests/test_compressor.py Tue Feb 28 11:13:25 2017 -0800
@@ -10,7 +10,10 @@
import zstd
-from .common import OpCountingBytesIO
+from .common import (
+ make_cffi,
+ OpCountingBytesIO,
+)
if sys.version_info[0] >= 3:
@@ -19,6 +22,7 @@
next = lambda it: it.next()
+@make_cffi
class TestCompressor(unittest.TestCase):
def test_level_bounds(self):
with self.assertRaises(ValueError):
@@ -28,18 +32,17 @@
zstd.ZstdCompressor(level=23)
+@make_cffi
class TestCompressor_compress(unittest.TestCase):
def test_compress_empty(self):
cctx = zstd.ZstdCompressor(level=1)
- cctx.compress(b'')
-
- cctx = zstd.ZstdCompressor(level=22)
- cctx.compress(b'')
-
- def test_compress_empty(self):
- cctx = zstd.ZstdCompressor(level=1)
- self.assertEqual(cctx.compress(b''),
- b'\x28\xb5\x2f\xfd\x00\x48\x01\x00\x00')
+ result = cctx.compress(b'')
+ self.assertEqual(result, b'\x28\xb5\x2f\xfd\x00\x48\x01\x00\x00')
+ params = zstd.get_frame_parameters(result)
+ self.assertEqual(params.content_size, 0)
+ self.assertEqual(params.window_size, 524288)
+ self.assertEqual(params.dict_id, 0)
+ self.assertFalse(params.has_checksum, 0)
# TODO should be temporary until https://github.com/facebook/zstd/issues/506
# is fixed.
@@ -59,6 +62,13 @@
self.assertEqual(len(result), 999)
self.assertEqual(result[0:4], b'\x28\xb5\x2f\xfd')
+ # This matches the test for read_from() below.
+ cctx = zstd.ZstdCompressor(level=1)
+ result = cctx.compress(b'f' * zstd.COMPRESSION_RECOMMENDED_INPUT_SIZE + b'o')
+ self.assertEqual(result, b'\x28\xb5\x2f\xfd\x00\x40\x54\x00\x00'
+ b'\x10\x66\x66\x01\x00\xfb\xff\x39\xc0'
+ b'\x02\x09\x00\x00\x6f')
+
def test_write_checksum(self):
cctx = zstd.ZstdCompressor(level=1)
no_checksum = cctx.compress(b'foobar')
@@ -67,6 +77,12 @@
self.assertEqual(len(with_checksum), len(no_checksum) + 4)
+ no_params = zstd.get_frame_parameters(no_checksum)
+ with_params = zstd.get_frame_parameters(with_checksum)
+
+ self.assertFalse(no_params.has_checksum)
+ self.assertTrue(with_params.has_checksum)
+
def test_write_content_size(self):
cctx = zstd.ZstdCompressor(level=1)
no_size = cctx.compress(b'foobar' * 256)
@@ -75,6 +91,11 @@
self.assertEqual(len(with_size), len(no_size) + 1)
+ no_params = zstd.get_frame_parameters(no_size)
+ with_params = zstd.get_frame_parameters(with_size)
+ self.assertEqual(no_params.content_size, 0)
+ self.assertEqual(with_params.content_size, 1536)
+
def test_no_dict_id(self):
samples = []
for i in range(128):
@@ -92,6 +113,11 @@
self.assertEqual(len(with_dict_id), len(no_dict_id) + 4)
+ no_params = zstd.get_frame_parameters(no_dict_id)
+ with_params = zstd.get_frame_parameters(with_dict_id)
+ self.assertEqual(no_params.dict_id, 0)
+ self.assertEqual(with_params.dict_id, 1584102229)
+
def test_compress_dict_multiple(self):
samples = []
for i in range(128):
@@ -107,6 +133,7 @@
cctx.compress(b'foo bar foobar foo bar foobar')
+@make_cffi
class TestCompressor_compressobj(unittest.TestCase):
def test_compressobj_empty(self):
cctx = zstd.ZstdCompressor(level=1)
@@ -127,6 +154,12 @@
self.assertEqual(len(result), 999)
self.assertEqual(result[0:4], b'\x28\xb5\x2f\xfd')
+ params = zstd.get_frame_parameters(result)
+ self.assertEqual(params.content_size, 0)
+ self.assertEqual(params.window_size, 1048576)
+ self.assertEqual(params.dict_id, 0)
+ self.assertFalse(params.has_checksum)
+
def test_write_checksum(self):
cctx = zstd.ZstdCompressor(level=1)
cobj = cctx.compressobj()
@@ -135,6 +168,15 @@
cobj = cctx.compressobj()
with_checksum = cobj.compress(b'foobar') + cobj.flush()
+ no_params = zstd.get_frame_parameters(no_checksum)
+ with_params = zstd.get_frame_parameters(with_checksum)
+ self.assertEqual(no_params.content_size, 0)
+ self.assertEqual(with_params.content_size, 0)
+ self.assertEqual(no_params.dict_id, 0)
+ self.assertEqual(with_params.dict_id, 0)
+ self.assertFalse(no_params.has_checksum)
+ self.assertTrue(with_params.has_checksum)
+
self.assertEqual(len(with_checksum), len(no_checksum) + 4)
def test_write_content_size(self):
@@ -145,6 +187,15 @@
cobj = cctx.compressobj(size=len(b'foobar' * 256))
with_size = cobj.compress(b'foobar' * 256) + cobj.flush()
+ no_params = zstd.get_frame_parameters(no_size)
+ with_params = zstd.get_frame_parameters(with_size)
+ self.assertEqual(no_params.content_size, 0)
+ self.assertEqual(with_params.content_size, 1536)
+ self.assertEqual(no_params.dict_id, 0)
+ self.assertEqual(with_params.dict_id, 0)
+ self.assertFalse(no_params.has_checksum)
+ self.assertFalse(with_params.has_checksum)
+
self.assertEqual(len(with_size), len(no_size) + 1)
def test_compress_after_finished(self):
@@ -187,6 +238,7 @@
self.assertEqual(header, b'\x01\x00\x00')
+@make_cffi
class TestCompressor_copy_stream(unittest.TestCase):
def test_no_read(self):
source = object()
@@ -229,6 +281,12 @@
self.assertEqual(r, 255 * 16384)
self.assertEqual(w, 999)
+ params = zstd.get_frame_parameters(dest.getvalue())
+ self.assertEqual(params.content_size, 0)
+ self.assertEqual(params.window_size, 1048576)
+ self.assertEqual(params.dict_id, 0)
+ self.assertFalse(params.has_checksum)
+
def test_write_checksum(self):
source = io.BytesIO(b'foobar')
no_checksum = io.BytesIO()
@@ -244,6 +302,15 @@
self.assertEqual(len(with_checksum.getvalue()),
len(no_checksum.getvalue()) + 4)
+ no_params = zstd.get_frame_parameters(no_checksum.getvalue())
+ with_params = zstd.get_frame_parameters(with_checksum.getvalue())
+ self.assertEqual(no_params.content_size, 0)
+ self.assertEqual(with_params.content_size, 0)
+ self.assertEqual(no_params.dict_id, 0)
+ self.assertEqual(with_params.dict_id, 0)
+ self.assertFalse(no_params.has_checksum)
+ self.assertTrue(with_params.has_checksum)
+
def test_write_content_size(self):
source = io.BytesIO(b'foobar' * 256)
no_size = io.BytesIO()
@@ -268,6 +335,15 @@
self.assertEqual(len(with_size.getvalue()),
len(no_size.getvalue()) + 1)
+ no_params = zstd.get_frame_parameters(no_size.getvalue())
+ with_params = zstd.get_frame_parameters(with_size.getvalue())
+ self.assertEqual(no_params.content_size, 0)
+ self.assertEqual(with_params.content_size, 1536)
+ self.assertEqual(no_params.dict_id, 0)
+ self.assertEqual(with_params.dict_id, 0)
+ self.assertFalse(no_params.has_checksum)
+ self.assertFalse(with_params.has_checksum)
+
def test_read_write_size(self):
source = OpCountingBytesIO(b'foobarfoobar')
dest = OpCountingBytesIO()
@@ -288,18 +364,25 @@
return buffer.getvalue()
+@make_cffi
class TestCompressor_write_to(unittest.TestCase):
def test_empty(self):
- self.assertEqual(compress(b'', 1),
- b'\x28\xb5\x2f\xfd\x00\x48\x01\x00\x00')
+ result = compress(b'', 1)
+ self.assertEqual(result, b'\x28\xb5\x2f\xfd\x00\x48\x01\x00\x00')
+
+ params = zstd.get_frame_parameters(result)
+ self.assertEqual(params.content_size, 0)
+ self.assertEqual(params.window_size, 524288)
+ self.assertEqual(params.dict_id, 0)
+ self.assertFalse(params.has_checksum)
def test_multiple_compress(self):
buffer = io.BytesIO()
cctx = zstd.ZstdCompressor(level=5)
with cctx.write_to(buffer) as compressor:
- compressor.write(b'foo')
- compressor.write(b'bar')
- compressor.write(b'x' * 8192)
+ self.assertEqual(compressor.write(b'foo'), 0)
+ self.assertEqual(compressor.write(b'bar'), 0)
+ self.assertEqual(compressor.write(b'x' * 8192), 0)
result = buffer.getvalue()
self.assertEqual(result,
@@ -318,11 +401,23 @@
buffer = io.BytesIO()
cctx = zstd.ZstdCompressor(level=9, dict_data=d)
with cctx.write_to(buffer) as compressor:
- compressor.write(b'foo')
- compressor.write(b'bar')
- compressor.write(b'foo' * 16384)
+ self.assertEqual(compressor.write(b'foo'), 0)
+ self.assertEqual(compressor.write(b'bar'), 0)
+ self.assertEqual(compressor.write(b'foo' * 16384), 634)
compressed = buffer.getvalue()
+
+ params = zstd.get_frame_parameters(compressed)
+ self.assertEqual(params.content_size, 0)
+ self.assertEqual(params.window_size, 1024)
+ self.assertEqual(params.dict_id, d.dict_id())
+ self.assertFalse(params.has_checksum)
+
+ self.assertEqual(compressed[0:32],
+ b'\x28\xb5\x2f\xfd\x03\x00\x55\x7b\x6b\x5e\x54\x00'
+ b'\x00\x00\x02\xfc\xf4\xa5\xba\x23\x3f\x85\xb3\x54'
+ b'\x00\x00\x18\x6f\x6f\x66\x01\x00')
+
h = hashlib.sha1(compressed).hexdigest()
self.assertEqual(h, '1c5bcd25181bcd8c1a73ea8773323e0056129f92')
@@ -332,11 +427,18 @@
buffer = io.BytesIO()
cctx = zstd.ZstdCompressor(compression_params=params)
with cctx.write_to(buffer) as compressor:
- compressor.write(b'foo')
- compressor.write(b'bar')
- compressor.write(b'foobar' * 16384)
+ self.assertEqual(compressor.write(b'foo'), 0)
+ self.assertEqual(compressor.write(b'bar'), 0)
+ self.assertEqual(compressor.write(b'foobar' * 16384), 0)
compressed = buffer.getvalue()
+
+ params = zstd.get_frame_parameters(compressed)
+ self.assertEqual(params.content_size, 0)
+ self.assertEqual(params.window_size, 1048576)
+ self.assertEqual(params.dict_id, 0)
+ self.assertFalse(params.has_checksum)
+
h = hashlib.sha1(compressed).hexdigest()
self.assertEqual(h, '1ae31f270ed7de14235221a604b31ecd517ebd99')
@@ -344,12 +446,21 @@
no_checksum = io.BytesIO()
cctx = zstd.ZstdCompressor(level=1)
with cctx.write_to(no_checksum) as compressor:
- compressor.write(b'foobar')
+ self.assertEqual(compressor.write(b'foobar'), 0)
with_checksum = io.BytesIO()
cctx = zstd.ZstdCompressor(level=1, write_checksum=True)
with cctx.write_to(with_checksum) as compressor:
- compressor.write(b'foobar')
+ self.assertEqual(compressor.write(b'foobar'), 0)
+
+ no_params = zstd.get_frame_parameters(no_checksum.getvalue())
+ with_params = zstd.get_frame_parameters(with_checksum.getvalue())
+ self.assertEqual(no_params.content_size, 0)
+ self.assertEqual(with_params.content_size, 0)
+ self.assertEqual(no_params.dict_id, 0)
+ self.assertEqual(with_params.dict_id, 0)
+ self.assertFalse(no_params.has_checksum)
+ self.assertTrue(with_params.has_checksum)
self.assertEqual(len(with_checksum.getvalue()),
len(no_checksum.getvalue()) + 4)
@@ -358,12 +469,12 @@
no_size = io.BytesIO()
cctx = zstd.ZstdCompressor(level=1)
with cctx.write_to(no_size) as compressor:
- compressor.write(b'foobar' * 256)
+ self.assertEqual(compressor.write(b'foobar' * 256), 0)
with_size = io.BytesIO()
cctx = zstd.ZstdCompressor(level=1, write_content_size=True)
with cctx.write_to(with_size) as compressor:
- compressor.write(b'foobar' * 256)
+ self.assertEqual(compressor.write(b'foobar' * 256), 0)
# Source size is not known in streaming mode, so header not
# written.
@@ -373,7 +484,16 @@
# Declaring size will write the header.
with_size = io.BytesIO()
with cctx.write_to(with_size, size=len(b'foobar' * 256)) as compressor:
- compressor.write(b'foobar' * 256)
+ self.assertEqual(compressor.write(b'foobar' * 256), 0)
+
+ no_params = zstd.get_frame_parameters(no_size.getvalue())
+ with_params = zstd.get_frame_parameters(with_size.getvalue())
+ self.assertEqual(no_params.content_size, 0)
+ self.assertEqual(with_params.content_size, 1536)
+ self.assertEqual(no_params.dict_id, 0)
+ self.assertEqual(with_params.dict_id, 0)
+ self.assertFalse(no_params.has_checksum)
+ self.assertFalse(with_params.has_checksum)
self.assertEqual(len(with_size.getvalue()),
len(no_size.getvalue()) + 1)
@@ -390,12 +510,21 @@
with_dict_id = io.BytesIO()
cctx = zstd.ZstdCompressor(level=1, dict_data=d)
with cctx.write_to(with_dict_id) as compressor:
- compressor.write(b'foobarfoobar')
+ self.assertEqual(compressor.write(b'foobarfoobar'), 0)
cctx = zstd.ZstdCompressor(level=1, dict_data=d, write_dict_id=False)
no_dict_id = io.BytesIO()
with cctx.write_to(no_dict_id) as compressor:
- compressor.write(b'foobarfoobar')
+ self.assertEqual(compressor.write(b'foobarfoobar'), 0)
+
+ no_params = zstd.get_frame_parameters(no_dict_id.getvalue())
+ with_params = zstd.get_frame_parameters(with_dict_id.getvalue())
+ self.assertEqual(no_params.content_size, 0)
+ self.assertEqual(with_params.content_size, 0)
+ self.assertEqual(no_params.dict_id, 0)
+ self.assertEqual(with_params.dict_id, d.dict_id())
+ self.assertFalse(no_params.has_checksum)
+ self.assertFalse(with_params.has_checksum)
self.assertEqual(len(with_dict_id.getvalue()),
len(no_dict_id.getvalue()) + 4)
@@ -412,9 +541,9 @@
cctx = zstd.ZstdCompressor(level=3)
dest = OpCountingBytesIO()
with cctx.write_to(dest, write_size=1) as compressor:
- compressor.write(b'foo')
- compressor.write(b'bar')
- compressor.write(b'foobar')
+ self.assertEqual(compressor.write(b'foo'), 0)
+ self.assertEqual(compressor.write(b'bar'), 0)
+ self.assertEqual(compressor.write(b'foobar'), 0)
self.assertEqual(len(dest.getvalue()), dest._write_count)
@@ -422,15 +551,15 @@
cctx = zstd.ZstdCompressor(level=3)
dest = OpCountingBytesIO()
with cctx.write_to(dest) as compressor:
- compressor.write(b'foo')
+ self.assertEqual(compressor.write(b'foo'), 0)
self.assertEqual(dest._write_count, 0)
- compressor.flush()
+ self.assertEqual(compressor.flush(), 12)
self.assertEqual(dest._write_count, 1)
- compressor.write(b'bar')
+ self.assertEqual(compressor.write(b'bar'), 0)
self.assertEqual(dest._write_count, 1)
- compressor.flush()
+ self.assertEqual(compressor.flush(), 6)
self.assertEqual(dest._write_count, 2)
- compressor.write(b'baz')
+ self.assertEqual(compressor.write(b'baz'), 0)
self.assertEqual(dest._write_count, 3)
@@ -438,10 +567,10 @@
cctx = zstd.ZstdCompressor(level=3, write_checksum=True)
dest = OpCountingBytesIO()
with cctx.write_to(dest) as compressor:
- compressor.write(b'foobar' * 8192)
+ self.assertEqual(compressor.write(b'foobar' * 8192), 0)
count = dest._write_count
offset = dest.tell()
- compressor.flush()
+ self.assertEqual(compressor.flush(), 23)
self.assertGreater(dest._write_count, count)
self.assertGreater(dest.tell(), offset)
offset = dest.tell()
@@ -456,18 +585,22 @@
self.assertEqual(header, b'\x01\x00\x00')
+@make_cffi
class TestCompressor_read_from(unittest.TestCase):
def test_type_validation(self):
cctx = zstd.ZstdCompressor()
# Object with read() works.
- cctx.read_from(io.BytesIO())
+ for chunk in cctx.read_from(io.BytesIO()):
+ pass
# Buffer protocol works.
- cctx.read_from(b'foobar')
+ for chunk in cctx.read_from(b'foobar'):
+ pass
with self.assertRaisesRegexp(ValueError, 'must pass an object with a read'):
- cctx.read_from(True)
+ for chunk in cctx.read_from(True):
+ pass
def test_read_empty(self):
cctx = zstd.ZstdCompressor(level=1)
@@ -521,6 +654,12 @@
# We should get the same output as the one-shot compression mechanism.
self.assertEqual(b''.join(chunks), cctx.compress(source.getvalue()))
+ params = zstd.get_frame_parameters(b''.join(chunks))
+ self.assertEqual(params.content_size, 0)
+ self.assertEqual(params.window_size, 262144)
+ self.assertEqual(params.dict_id, 0)
+ self.assertFalse(params.has_checksum)
+
# Now check the buffer protocol.
it = cctx.read_from(source.getvalue())
chunks = list(it)
--- a/contrib/python-zstandard/tests/test_data_structures.py Sat Feb 25 12:48:50 2017 +0900
+++ b/contrib/python-zstandard/tests/test_data_structures.py Tue Feb 28 11:13:25 2017 -0800
@@ -13,6 +13,12 @@
import zstd
+from . common import (
+ make_cffi,
+)
+
+
+@make_cffi
class TestCompressionParameters(unittest.TestCase):
def test_init_bad_arg_type(self):
with self.assertRaises(TypeError):
@@ -42,7 +48,81 @@
p = zstd.get_compression_parameters(1)
self.assertIsInstance(p, zstd.CompressionParameters)
- self.assertEqual(p[0], 19)
+ self.assertEqual(p.window_log, 19)
+
+ def test_members(self):
+ p = zstd.CompressionParameters(10, 6, 7, 4, 5, 8, 1)
+ self.assertEqual(p.window_log, 10)
+ self.assertEqual(p.chain_log, 6)
+ self.assertEqual(p.hash_log, 7)
+ self.assertEqual(p.search_log, 4)
+ self.assertEqual(p.search_length, 5)
+ self.assertEqual(p.target_length, 8)
+ self.assertEqual(p.strategy, 1)
+
+
+@make_cffi
+class TestFrameParameters(unittest.TestCase):
+ def test_invalid_type(self):
+ with self.assertRaises(TypeError):
+ zstd.get_frame_parameters(None)
+
+ with self.assertRaises(TypeError):
+ zstd.get_frame_parameters(u'foobarbaz')
+
+ def test_invalid_input_sizes(self):
+ with self.assertRaisesRegexp(zstd.ZstdError, 'not enough data for frame'):
+ zstd.get_frame_parameters(b'')
+
+ with self.assertRaisesRegexp(zstd.ZstdError, 'not enough data for frame'):
+ zstd.get_frame_parameters(zstd.FRAME_HEADER)
+
+ def test_invalid_frame(self):
+ with self.assertRaisesRegexp(zstd.ZstdError, 'Unknown frame descriptor'):
+ zstd.get_frame_parameters(b'foobarbaz')
+
+ def test_attributes(self):
+ params = zstd.get_frame_parameters(zstd.FRAME_HEADER + b'\x00\x00')
+ self.assertEqual(params.content_size, 0)
+ self.assertEqual(params.window_size, 1024)
+ self.assertEqual(params.dict_id, 0)
+ self.assertFalse(params.has_checksum)
+
+ # Lowest 2 bits indicate a dictionary and length. Here, the dict id is 1 byte.
+ params = zstd.get_frame_parameters(zstd.FRAME_HEADER + b'\x01\x00\xff')
+ self.assertEqual(params.content_size, 0)
+ self.assertEqual(params.window_size, 1024)
+ self.assertEqual(params.dict_id, 255)
+ self.assertFalse(params.has_checksum)
+
+ # Lowest 3rd bit indicates if checksum is present.
+ params = zstd.get_frame_parameters(zstd.FRAME_HEADER + b'\x04\x00')
+ self.assertEqual(params.content_size, 0)
+ self.assertEqual(params.window_size, 1024)
+ self.assertEqual(params.dict_id, 0)
+ self.assertTrue(params.has_checksum)
+
+ # Upper 2 bits indicate content size.
+ params = zstd.get_frame_parameters(zstd.FRAME_HEADER + b'\x40\x00\xff\x00')
+ self.assertEqual(params.content_size, 511)
+ self.assertEqual(params.window_size, 1024)
+ self.assertEqual(params.dict_id, 0)
+ self.assertFalse(params.has_checksum)
+
+ # Window descriptor is 2nd byte after frame header.
+ params = zstd.get_frame_parameters(zstd.FRAME_HEADER + b'\x00\x40')
+ self.assertEqual(params.content_size, 0)
+ self.assertEqual(params.window_size, 262144)
+ self.assertEqual(params.dict_id, 0)
+ self.assertFalse(params.has_checksum)
+
+ # Set multiple things.
+ params = zstd.get_frame_parameters(zstd.FRAME_HEADER + b'\x45\x40\x0f\x10\x00')
+ self.assertEqual(params.content_size, 272)
+ self.assertEqual(params.window_size, 262144)
+ self.assertEqual(params.dict_id, 15)
+ self.assertTrue(params.has_checksum)
+
if hypothesis:
s_windowlog = strategies.integers(min_value=zstd.WINDOWLOG_MIN,
@@ -65,6 +145,8 @@
zstd.STRATEGY_BTLAZY2,
zstd.STRATEGY_BTOPT))
+
+ @make_cffi
class TestCompressionParametersHypothesis(unittest.TestCase):
@hypothesis.given(s_windowlog, s_chainlog, s_hashlog, s_searchlog,
s_searchlength, s_targetlength, s_strategy)
@@ -73,9 +155,6 @@
p = zstd.CompressionParameters(windowlog, chainlog, hashlog,
searchlog, searchlength,
targetlength, strategy)
- self.assertEqual(tuple(p),
- (windowlog, chainlog, hashlog, searchlog,
- searchlength, targetlength, strategy))
# Verify we can instantiate a compressor with the supplied values.
# ZSTD_checkCParams moves the goal posts on us from what's advertised
--- a/contrib/python-zstandard/tests/test_decompressor.py Sat Feb 25 12:48:50 2017 +0900
+++ b/contrib/python-zstandard/tests/test_decompressor.py Tue Feb 28 11:13:25 2017 -0800
@@ -10,7 +10,10 @@
import zstd
-from .common import OpCountingBytesIO
+from .common import (
+ make_cffi,
+ OpCountingBytesIO,
+)
if sys.version_info[0] >= 3:
@@ -19,6 +22,7 @@
next = lambda it: it.next()
+@make_cffi
class TestDecompressor_decompress(unittest.TestCase):
def test_empty_input(self):
dctx = zstd.ZstdDecompressor()
@@ -119,6 +123,7 @@
self.assertEqual(decompressed, sources[i])
+@make_cffi
class TestDecompressor_copy_stream(unittest.TestCase):
def test_no_read(self):
source = object()
@@ -180,6 +185,7 @@
self.assertEqual(dest._write_count, len(dest.getvalue()))
+@make_cffi
class TestDecompressor_decompressobj(unittest.TestCase):
def test_simple(self):
data = zstd.ZstdCompressor(level=1).compress(b'foobar')
@@ -207,6 +213,7 @@
return buffer.getvalue()
+@make_cffi
class TestDecompressor_write_to(unittest.TestCase):
def test_empty_roundtrip(self):
cctx = zstd.ZstdCompressor()
@@ -256,14 +263,14 @@
buffer = io.BytesIO()
cctx = zstd.ZstdCompressor(dict_data=d)
with cctx.write_to(buffer) as compressor:
- compressor.write(orig)
+ self.assertEqual(compressor.write(orig), 1544)
compressed = buffer.getvalue()
buffer = io.BytesIO()
dctx = zstd.ZstdDecompressor(dict_data=d)
with dctx.write_to(buffer) as decompressor:
- decompressor.write(compressed)
+ self.assertEqual(decompressor.write(compressed), len(orig))
self.assertEqual(buffer.getvalue(), orig)
@@ -291,6 +298,7 @@
self.assertEqual(dest._write_count, len(dest.getvalue()))
+@make_cffi
class TestDecompressor_read_from(unittest.TestCase):
def test_type_validation(self):
dctx = zstd.ZstdDecompressor()
@@ -302,7 +310,7 @@
dctx.read_from(b'foobar')
with self.assertRaisesRegexp(ValueError, 'must pass an object with a read'):
- dctx.read_from(True)
+ b''.join(dctx.read_from(True))
def test_empty_input(self):
dctx = zstd.ZstdDecompressor()
@@ -351,7 +359,7 @@
dctx = zstd.ZstdDecompressor()
with self.assertRaisesRegexp(ValueError, 'skip_bytes must be smaller than read_size'):
- dctx.read_from(b'', skip_bytes=1, read_size=1)
+ b''.join(dctx.read_from(b'', skip_bytes=1, read_size=1))
with self.assertRaisesRegexp(ValueError, 'skip_bytes larger than first input chunk'):
b''.join(dctx.read_from(b'foobar', skip_bytes=10))
@@ -476,3 +484,94 @@
self.assertEqual(len(chunk), 1)
self.assertEqual(source._read_count, len(source.getvalue()))
+
+
+@make_cffi
+class TestDecompressor_content_dict_chain(unittest.TestCase):
+ def test_bad_inputs_simple(self):
+ dctx = zstd.ZstdDecompressor()
+
+ with self.assertRaises(TypeError):
+ dctx.decompress_content_dict_chain(b'foo')
+
+ with self.assertRaises(TypeError):
+ dctx.decompress_content_dict_chain((b'foo', b'bar'))
+
+ with self.assertRaisesRegexp(ValueError, 'empty input chain'):
+ dctx.decompress_content_dict_chain([])
+
+ with self.assertRaisesRegexp(ValueError, 'chunk 0 must be bytes'):
+ dctx.decompress_content_dict_chain([u'foo'])
+
+ with self.assertRaisesRegexp(ValueError, 'chunk 0 must be bytes'):
+ dctx.decompress_content_dict_chain([True])
+
+ with self.assertRaisesRegexp(ValueError, 'chunk 0 is too small to contain a zstd frame'):
+ dctx.decompress_content_dict_chain([zstd.FRAME_HEADER])
+
+ with self.assertRaisesRegexp(ValueError, 'chunk 0 is not a valid zstd frame'):
+ dctx.decompress_content_dict_chain([b'foo' * 8])
+
+ no_size = zstd.ZstdCompressor().compress(b'foo' * 64)
+
+ with self.assertRaisesRegexp(ValueError, 'chunk 0 missing content size in frame'):
+ dctx.decompress_content_dict_chain([no_size])
+
+ # Corrupt first frame.
+ frame = zstd.ZstdCompressor(write_content_size=True).compress(b'foo' * 64)
+ frame = frame[0:12] + frame[15:]
+ with self.assertRaisesRegexp(zstd.ZstdError, 'could not decompress chunk 0'):
+ dctx.decompress_content_dict_chain([frame])
+
+ def test_bad_subsequent_input(self):
+ initial = zstd.ZstdCompressor(write_content_size=True).compress(b'foo' * 64)
+
+ dctx = zstd.ZstdDecompressor()
+
+ with self.assertRaisesRegexp(ValueError, 'chunk 1 must be bytes'):
+ dctx.decompress_content_dict_chain([initial, u'foo'])
+
+ with self.assertRaisesRegexp(ValueError, 'chunk 1 must be bytes'):
+ dctx.decompress_content_dict_chain([initial, None])
+
+ with self.assertRaisesRegexp(ValueError, 'chunk 1 is too small to contain a zstd frame'):
+ dctx.decompress_content_dict_chain([initial, zstd.FRAME_HEADER])
+
+ with self.assertRaisesRegexp(ValueError, 'chunk 1 is not a valid zstd frame'):
+ dctx.decompress_content_dict_chain([initial, b'foo' * 8])
+
+ no_size = zstd.ZstdCompressor().compress(b'foo' * 64)
+
+ with self.assertRaisesRegexp(ValueError, 'chunk 1 missing content size in frame'):
+ dctx.decompress_content_dict_chain([initial, no_size])
+
+ # Corrupt second frame.
+ cctx = zstd.ZstdCompressor(write_content_size=True, dict_data=zstd.ZstdCompressionDict(b'foo' * 64))
+ frame = cctx.compress(b'bar' * 64)
+ frame = frame[0:12] + frame[15:]
+
+ with self.assertRaisesRegexp(zstd.ZstdError, 'could not decompress chunk 1'):
+ dctx.decompress_content_dict_chain([initial, frame])
+
+ def test_simple(self):
+ original = [
+ b'foo' * 64,
+ b'foobar' * 64,
+ b'baz' * 64,
+ b'foobaz' * 64,
+ b'foobarbaz' * 64,
+ ]
+
+ chunks = []
+ chunks.append(zstd.ZstdCompressor(write_content_size=True).compress(original[0]))
+ for i, chunk in enumerate(original[1:]):
+ d = zstd.ZstdCompressionDict(original[i])
+ cctx = zstd.ZstdCompressor(dict_data=d, write_content_size=True)
+ chunks.append(cctx.compress(chunk))
+
+ for i in range(1, len(original)):
+ chain = chunks[0:i]
+ expected = original[i - 1]
+ dctx = zstd.ZstdDecompressor()
+ decompressed = dctx.decompress_content_dict_chain(chain)
+ self.assertEqual(decompressed, expected)
--- a/contrib/python-zstandard/tests/test_estimate_sizes.py Sat Feb 25 12:48:50 2017 +0900
+++ b/contrib/python-zstandard/tests/test_estimate_sizes.py Tue Feb 28 11:13:25 2017 -0800
@@ -5,7 +5,12 @@
import zstd
+from . common import (
+ make_cffi,
+)
+
+@make_cffi
class TestSizes(unittest.TestCase):
def test_decompression_size(self):
size = zstd.estimate_decompression_context_size()
--- a/contrib/python-zstandard/tests/test_module_attributes.py Sat Feb 25 12:48:50 2017 +0900
+++ b/contrib/python-zstandard/tests/test_module_attributes.py Tue Feb 28 11:13:25 2017 -0800
@@ -7,9 +7,15 @@
import zstd
+from . common import (
+ make_cffi,
+)
+
+
+@make_cffi
class TestModuleAttributes(unittest.TestCase):
def test_version(self):
- self.assertEqual(zstd.ZSTD_VERSION, (1, 1, 2))
+ self.assertEqual(zstd.ZSTD_VERSION, (1, 1, 3))
def test_constants(self):
self.assertEqual(zstd.MAX_COMPRESSION_LEVEL, 22)
@@ -45,4 +51,4 @@
)
for a in attrs:
- self.assertTrue(hasattr(zstd, a))
+ self.assertTrue(hasattr(zstd, a), a)
--- a/contrib/python-zstandard/tests/test_roundtrip.py Sat Feb 25 12:48:50 2017 +0900
+++ b/contrib/python-zstandard/tests/test_roundtrip.py Tue Feb 28 11:13:25 2017 -0800
@@ -13,10 +13,14 @@
import zstd
+from .common import (
+ make_cffi,
+)
compression_levels = strategies.integers(min_value=1, max_value=22)
+@make_cffi
class TestRoundTrip(unittest.TestCase):
@hypothesis.given(strategies.binary(), compression_levels)
def test_compress_write_to(self, data, level):
--- a/contrib/python-zstandard/tests/test_train_dictionary.py Sat Feb 25 12:48:50 2017 +0900
+++ b/contrib/python-zstandard/tests/test_train_dictionary.py Tue Feb 28 11:13:25 2017 -0800
@@ -7,6 +7,9 @@
import zstd
+from . common import (
+ make_cffi,
+)
if sys.version_info[0] >= 3:
int_type = int
@@ -14,6 +17,7 @@
int_type = long
+@make_cffi
class TestTrainDictionary(unittest.TestCase):
def test_no_args(self):
with self.assertRaises(TypeError):
--- a/contrib/python-zstandard/zstd.c Sat Feb 25 12:48:50 2017 +0900
+++ b/contrib/python-zstandard/zstd.c Tue Feb 28 11:13:25 2017 -0800
@@ -34,6 +34,11 @@
"Obtains a ``CompressionParameters`` instance from a compression level and\n"
"optional input size and dictionary size");
+PyDoc_STRVAR(get_frame_parameters__doc__,
+"get_frame_parameters(data)\n"
+"\n"
+"Obtains a ``FrameParameters`` instance by parsing data.\n");
+
PyDoc_STRVAR(train_dictionary__doc__,
"train_dictionary(dict_size, samples)\n"
"\n"
@@ -53,6 +58,8 @@
METH_NOARGS, estimate_decompression_context_size__doc__ },
{ "get_compression_parameters", (PyCFunction)get_compression_parameters,
METH_VARARGS, get_compression_parameters__doc__ },
+ { "get_frame_parameters", (PyCFunction)get_frame_parameters,
+ METH_VARARGS, get_frame_parameters__doc__ },
{ "train_dictionary", (PyCFunction)train_dictionary,
METH_VARARGS | METH_KEYWORDS, train_dictionary__doc__ },
{ NULL, NULL }
@@ -70,6 +77,7 @@
void decompressobj_module_init(PyObject* mod);
void decompressionwriter_module_init(PyObject* mod);
void decompressoriterator_module_init(PyObject* mod);
+void frameparams_module_init(PyObject* mod);
void zstd_module_init(PyObject* m) {
/* python-zstandard relies on unstable zstd C API features. This means
@@ -87,7 +95,7 @@
We detect this mismatch here and refuse to load the module if this
scenario is detected.
*/
- if (ZSTD_VERSION_NUMBER != 10102 || ZSTD_versionNumber() != 10102) {
+ if (ZSTD_VERSION_NUMBER != 10103 || ZSTD_versionNumber() != 10103) {
PyErr_SetString(PyExc_ImportError, "zstd C API mismatch; Python bindings not compiled against expected zstd version");
return;
}
@@ -104,6 +112,7 @@
decompressobj_module_init(m);
decompressionwriter_module_init(m);
decompressoriterator_module_init(m);
+ frameparams_module_init(m);
}
#if PY_MAJOR_VERSION >= 3
--- a/contrib/python-zstandard/zstd/common/mem.h Sat Feb 25 12:48:50 2017 +0900
+++ b/contrib/python-zstandard/zstd/common/mem.h Tue Feb 28 11:13:25 2017 -0800
@@ -39,7 +39,7 @@
#endif
/* code only tested on 32 and 64 bits systems */
-#define MEM_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; }
+#define MEM_STATIC_ASSERT(c) { enum { MEM_static_assert = 1/(int)(!!(c)) }; }
MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); }
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/contrib/python-zstandard/zstd/common/pool.c Tue Feb 28 11:13:25 2017 -0800
@@ -0,0 +1,194 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+
+/* ====== Dependencies ======= */
+#include <stddef.h> /* size_t */
+#include <stdlib.h> /* malloc, calloc, free */
+#include "pool.h"
+
+/* ====== Compiler specifics ====== */
+#if defined(_MSC_VER)
+# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */
+#endif
+
+
+#ifdef ZSTD_MULTITHREAD
+
+#include "threading.h" /* pthread adaptation */
+
+/* A job is a function and an opaque argument */
+typedef struct POOL_job_s {
+ POOL_function function;
+ void *opaque;
+} POOL_job;
+
+struct POOL_ctx_s {
+ /* Keep track of the threads */
+ pthread_t *threads;
+ size_t numThreads;
+
+ /* The queue is a circular buffer */
+ POOL_job *queue;
+ size_t queueHead;
+ size_t queueTail;
+ size_t queueSize;
+ /* The mutex protects the queue */
+ pthread_mutex_t queueMutex;
+ /* Condition variable for pushers to wait on when the queue is full */
+ pthread_cond_t queuePushCond;
+ /* Condition variables for poppers to wait on when the queue is empty */
+ pthread_cond_t queuePopCond;
+ /* Indicates if the queue is shutting down */
+ int shutdown;
+};
+
+/* POOL_thread() :
+ Work thread for the thread pool.
+ Waits for jobs and executes them.
+ @returns : NULL on failure else non-null.
+*/
+static void* POOL_thread(void* opaque) {
+ POOL_ctx* const ctx = (POOL_ctx*)opaque;
+ if (!ctx) { return NULL; }
+ for (;;) {
+ /* Lock the mutex and wait for a non-empty queue or until shutdown */
+ pthread_mutex_lock(&ctx->queueMutex);
+ while (ctx->queueHead == ctx->queueTail && !ctx->shutdown) {
+ pthread_cond_wait(&ctx->queuePopCond, &ctx->queueMutex);
+ }
+ /* empty => shutting down: so stop */
+ if (ctx->queueHead == ctx->queueTail) {
+ pthread_mutex_unlock(&ctx->queueMutex);
+ return opaque;
+ }
+ /* Pop a job off the queue */
+ { POOL_job const job = ctx->queue[ctx->queueHead];
+ ctx->queueHead = (ctx->queueHead + 1) % ctx->queueSize;
+ /* Unlock the mutex, signal a pusher, and run the job */
+ pthread_mutex_unlock(&ctx->queueMutex);
+ pthread_cond_signal(&ctx->queuePushCond);
+ job.function(job.opaque);
+ }
+ }
+ /* Unreachable */
+}
+
+POOL_ctx *POOL_create(size_t numThreads, size_t queueSize) {
+ POOL_ctx *ctx;
+ /* Check the parameters */
+ if (!numThreads || !queueSize) { return NULL; }
+ /* Allocate the context and zero initialize */
+ ctx = (POOL_ctx *)calloc(1, sizeof(POOL_ctx));
+ if (!ctx) { return NULL; }
+ /* Initialize the job queue.
+ * It needs one extra space since one space is wasted to differentiate empty
+ * and full queues.
+ */
+ ctx->queueSize = queueSize + 1;
+ ctx->queue = (POOL_job *)malloc(ctx->queueSize * sizeof(POOL_job));
+ ctx->queueHead = 0;
+ ctx->queueTail = 0;
+ pthread_mutex_init(&ctx->queueMutex, NULL);
+ pthread_cond_init(&ctx->queuePushCond, NULL);
+ pthread_cond_init(&ctx->queuePopCond, NULL);
+ ctx->shutdown = 0;
+ /* Allocate space for the thread handles */
+ ctx->threads = (pthread_t *)malloc(numThreads * sizeof(pthread_t));
+ ctx->numThreads = 0;
+ /* Check for errors */
+ if (!ctx->threads || !ctx->queue) { POOL_free(ctx); return NULL; }
+ /* Initialize the threads */
+ { size_t i;
+ for (i = 0; i < numThreads; ++i) {
+ if (pthread_create(&ctx->threads[i], NULL, &POOL_thread, ctx)) {
+ ctx->numThreads = i;
+ POOL_free(ctx);
+ return NULL;
+ } }
+ ctx->numThreads = numThreads;
+ }
+ return ctx;
+}
+
+/*! POOL_join() :
+ Shutdown the queue, wake any sleeping threads, and join all of the threads.
+*/
+static void POOL_join(POOL_ctx *ctx) {
+ /* Shut down the queue */
+ pthread_mutex_lock(&ctx->queueMutex);
+ ctx->shutdown = 1;
+ pthread_mutex_unlock(&ctx->queueMutex);
+ /* Wake up sleeping threads */
+ pthread_cond_broadcast(&ctx->queuePushCond);
+ pthread_cond_broadcast(&ctx->queuePopCond);
+ /* Join all of the threads */
+ { size_t i;
+ for (i = 0; i < ctx->numThreads; ++i) {
+ pthread_join(ctx->threads[i], NULL);
+ } }
+}
+
+void POOL_free(POOL_ctx *ctx) {
+ if (!ctx) { return; }
+ POOL_join(ctx);
+ pthread_mutex_destroy(&ctx->queueMutex);
+ pthread_cond_destroy(&ctx->queuePushCond);
+ pthread_cond_destroy(&ctx->queuePopCond);
+ if (ctx->queue) free(ctx->queue);
+ if (ctx->threads) free(ctx->threads);
+ free(ctx);
+}
+
+void POOL_add(void *ctxVoid, POOL_function function, void *opaque) {
+ POOL_ctx *ctx = (POOL_ctx *)ctxVoid;
+ if (!ctx) { return; }
+
+ pthread_mutex_lock(&ctx->queueMutex);
+ { POOL_job const job = {function, opaque};
+ /* Wait until there is space in the queue for the new job */
+ size_t newTail = (ctx->queueTail + 1) % ctx->queueSize;
+ while (ctx->queueHead == newTail && !ctx->shutdown) {
+ pthread_cond_wait(&ctx->queuePushCond, &ctx->queueMutex);
+ newTail = (ctx->queueTail + 1) % ctx->queueSize;
+ }
+ /* The queue is still going => there is space */
+ if (!ctx->shutdown) {
+ ctx->queue[ctx->queueTail] = job;
+ ctx->queueTail = newTail;
+ }
+ }
+ pthread_mutex_unlock(&ctx->queueMutex);
+ pthread_cond_signal(&ctx->queuePopCond);
+}
+
+#else /* ZSTD_MULTITHREAD not defined */
+/* No multi-threading support */
+
+/* We don't need any data, but if it is empty malloc() might return NULL. */
+struct POOL_ctx_s {
+ int data;
+};
+
+POOL_ctx *POOL_create(size_t numThreads, size_t queueSize) {
+ (void)numThreads;
+ (void)queueSize;
+ return (POOL_ctx *)malloc(sizeof(POOL_ctx));
+}
+
+void POOL_free(POOL_ctx *ctx) {
+ if (ctx) free(ctx);
+}
+
+void POOL_add(void *ctx, POOL_function function, void *opaque) {
+ (void)ctx;
+ function(opaque);
+}
+
+#endif /* ZSTD_MULTITHREAD */
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/contrib/python-zstandard/zstd/common/pool.h Tue Feb 28 11:13:25 2017 -0800
@@ -0,0 +1,56 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+#ifndef POOL_H
+#define POOL_H
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+
+#include <stddef.h> /* size_t */
+
+typedef struct POOL_ctx_s POOL_ctx;
+
+/*! POOL_create() :
+ Create a thread pool with at most `numThreads` threads.
+ `numThreads` must be at least 1.
+ The maximum number of queued jobs before blocking is `queueSize`.
+ `queueSize` must be at least 1.
+ @return : The POOL_ctx pointer on success else NULL.
+*/
+POOL_ctx *POOL_create(size_t numThreads, size_t queueSize);
+
+/*! POOL_free() :
+ Free a thread pool returned by POOL_create().
+*/
+void POOL_free(POOL_ctx *ctx);
+
+/*! POOL_function :
+ The function type that can be added to a thread pool.
+*/
+typedef void (*POOL_function)(void *);
+/*! POOL_add_function :
+ The function type for a generic thread pool add function.
+*/
+typedef void (*POOL_add_function)(void *, POOL_function, void *);
+
+/*! POOL_add() :
+ Add the job `function(opaque)` to the thread pool.
+ Possibly blocks until there is room in the queue.
+ Note : The function may be executed asynchronously, so `opaque` must live until the function has been completed.
+*/
+void POOL_add(void *ctx, POOL_function function, void *opaque);
+
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/contrib/python-zstandard/zstd/common/threading.c Tue Feb 28 11:13:25 2017 -0800
@@ -0,0 +1,79 @@
+
+/**
+ * Copyright (c) 2016 Tino Reichardt
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ *
+ * You can contact the author at:
+ * - zstdmt source repository: https://github.com/mcmilk/zstdmt
+ */
+
+/**
+ * This file will hold wrapper for systems, which do not support pthreads
+ */
+
+/* ====== Compiler specifics ====== */
+#if defined(_MSC_VER)
+# pragma warning(disable : 4206) /* disable: C4206: translation unit is empty (when ZSTD_MULTITHREAD is not defined) */
+#endif
+
+
+#if defined(ZSTD_MULTITHREAD) && defined(_WIN32)
+
+/**
+ * Windows minimalist Pthread Wrapper, based on :
+ * http://www.cse.wustl.edu/~schmidt/win32-cv-1.html
+ */
+
+
+/* === Dependencies === */
+#include <process.h>
+#include <errno.h>
+#include "threading.h"
+
+
+/* === Implementation === */
+
+static unsigned __stdcall worker(void *arg)
+{
+ pthread_t* const thread = (pthread_t*) arg;
+ thread->arg = thread->start_routine(thread->arg);
+ return 0;
+}
+
+int pthread_create(pthread_t* thread, const void* unused,
+ void* (*start_routine) (void*), void* arg)
+{
+ (void)unused;
+ thread->arg = arg;
+ thread->start_routine = start_routine;
+ thread->handle = (HANDLE) _beginthreadex(NULL, 0, worker, thread, 0, NULL);
+
+ if (!thread->handle)
+ return errno;
+ else
+ return 0;
+}
+
+int _pthread_join(pthread_t * thread, void **value_ptr)
+{
+ DWORD result;
+
+ if (!thread->handle) return 0;
+
+ result = WaitForSingleObject(thread->handle, INFINITE);
+ switch (result) {
+ case WAIT_OBJECT_0:
+ if (value_ptr) *value_ptr = thread->arg;
+ return 0;
+ case WAIT_ABANDONED:
+ return EINVAL;
+ default:
+ return GetLastError();
+ }
+}
+
+#endif /* ZSTD_MULTITHREAD */
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/contrib/python-zstandard/zstd/common/threading.h Tue Feb 28 11:13:25 2017 -0800
@@ -0,0 +1,104 @@
+
+/**
+ * Copyright (c) 2016 Tino Reichardt
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ *
+ * You can contact the author at:
+ * - zstdmt source repository: https://github.com/mcmilk/zstdmt
+ */
+
+#ifndef THREADING_H_938743
+#define THREADING_H_938743
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+#if defined(ZSTD_MULTITHREAD) && defined(_WIN32)
+
+/**
+ * Windows minimalist Pthread Wrapper, based on :
+ * http://www.cse.wustl.edu/~schmidt/win32-cv-1.html
+ */
+#ifdef WINVER
+# undef WINVER
+#endif
+#define WINVER 0x0600
+
+#ifdef _WIN32_WINNT
+# undef _WIN32_WINNT
+#endif
+#define _WIN32_WINNT 0x0600
+
+#ifndef WIN32_LEAN_AND_MEAN
+# define WIN32_LEAN_AND_MEAN
+#endif
+
+#include <windows.h>
+
+/* mutex */
+#define pthread_mutex_t CRITICAL_SECTION
+#define pthread_mutex_init(a,b) InitializeCriticalSection((a))
+#define pthread_mutex_destroy(a) DeleteCriticalSection((a))
+#define pthread_mutex_lock(a) EnterCriticalSection((a))
+#define pthread_mutex_unlock(a) LeaveCriticalSection((a))
+
+/* condition variable */
+#define pthread_cond_t CONDITION_VARIABLE
+#define pthread_cond_init(a, b) InitializeConditionVariable((a))
+#define pthread_cond_destroy(a) /* No delete */
+#define pthread_cond_wait(a, b) SleepConditionVariableCS((a), (b), INFINITE)
+#define pthread_cond_signal(a) WakeConditionVariable((a))
+#define pthread_cond_broadcast(a) WakeAllConditionVariable((a))
+
+/* pthread_create() and pthread_join() */
+typedef struct {
+ HANDLE handle;
+ void* (*start_routine)(void*);
+ void* arg;
+} pthread_t;
+
+int pthread_create(pthread_t* thread, const void* unused,
+ void* (*start_routine) (void*), void* arg);
+
+#define pthread_join(a, b) _pthread_join(&(a), (b))
+int _pthread_join(pthread_t* thread, void** value_ptr);
+
+/**
+ * add here more wrappers as required
+ */
+
+
+#elif defined(ZSTD_MULTITHREAD) /* posix assumed ; need a better detection mathod */
+/* === POSIX Systems === */
+# include <pthread.h>
+
+#else /* ZSTD_MULTITHREAD not defined */
+/* No multithreading support */
+
+#define pthread_mutex_t int /* #define rather than typedef, as sometimes pthread support is implicit, resulting in duplicated symbols */
+#define pthread_mutex_init(a,b)
+#define pthread_mutex_destroy(a)
+#define pthread_mutex_lock(a)
+#define pthread_mutex_unlock(a)
+
+#define pthread_cond_t int
+#define pthread_cond_init(a,b)
+#define pthread_cond_destroy(a)
+#define pthread_cond_wait(a,b)
+#define pthread_cond_signal(a)
+#define pthread_cond_broadcast(a)
+
+/* do not use pthread_t */
+
+#endif /* ZSTD_MULTITHREAD */
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* THREADING_H_938743 */
--- a/contrib/python-zstandard/zstd/common/zstd_common.c Sat Feb 25 12:48:50 2017 +0900
+++ b/contrib/python-zstandard/zstd/common/zstd_common.c Tue Feb 28 11:13:25 2017 -0800
@@ -43,10 +43,6 @@
* provides error code string from enum */
const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorName(code); }
-/* --- ZBUFF Error Management (deprecated) --- */
-unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); }
-const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); }
-
/*=**************************************************************
* Custom allocator
--- a/contrib/python-zstandard/zstd/common/zstd_errors.h Sat Feb 25 12:48:50 2017 +0900
+++ b/contrib/python-zstandard/zstd/common/zstd_errors.h Tue Feb 28 11:13:25 2017 -0800
@@ -18,6 +18,20 @@
#include <stddef.h> /* size_t */
+/* ===== ZSTDERRORLIB_API : control library symbols visibility ===== */
+#if defined(__GNUC__) && (__GNUC__ >= 4)
+# define ZSTDERRORLIB_VISIBILITY __attribute__ ((visibility ("default")))
+#else
+# define ZSTDERRORLIB_VISIBILITY
+#endif
+#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
+# define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBILITY
+#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
+# define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
+#else
+# define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBILITY
+#endif
+
/*-****************************************
* error codes list
******************************************/
@@ -49,8 +63,8 @@
/*! ZSTD_getErrorCode() :
convert a `size_t` function result into a `ZSTD_ErrorCode` enum type,
which can be used to compare directly with enum list published into "error_public.h" */
-ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult);
-const char* ZSTD_getErrorString(ZSTD_ErrorCode code);
+ZSTDERRORLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult);
+ZSTDERRORLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code);
#if defined (__cplusplus)
--- a/contrib/python-zstandard/zstd/common/zstd_internal.h Sat Feb 25 12:48:50 2017 +0900
+++ b/contrib/python-zstandard/zstd/common/zstd_internal.h Tue Feb 28 11:13:25 2017 -0800
@@ -267,4 +267,13 @@
}
+/* hidden functions */
+
+/* ZSTD_invalidateRepCodes() :
+ * ensures next compression will not use repcodes from previous block.
+ * Note : only works with regular variant;
+ * do not use with extDict variant ! */
+void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx);
+
+
#endif /* ZSTD_CCOMMON_H_MODULE */
--- a/contrib/python-zstandard/zstd/compress/zstd_compress.c Sat Feb 25 12:48:50 2017 +0900
+++ b/contrib/python-zstandard/zstd/compress/zstd_compress.c Tue Feb 28 11:13:25 2017 -0800
@@ -51,8 +51,7 @@
/*-*************************************
* Context memory management
***************************************/
-struct ZSTD_CCtx_s
-{
+struct ZSTD_CCtx_s {
const BYTE* nextSrc; /* next block here to continue on current prefix */
const BYTE* base; /* All regular indexes relative to this position */
const BYTE* dictBase; /* extDict indexes relative to this position */
@@ -61,10 +60,11 @@
U32 nextToUpdate; /* index from which to continue dictionary update */
U32 nextToUpdate3; /* index from which to continue dictionary update */
U32 hashLog3; /* dispatch table : larger == faster, more memory */
- U32 loadedDictEnd;
+ U32 loadedDictEnd; /* index of end of dictionary */
+ U32 forceWindow; /* force back-references to respect limit of 1<<wLog, even for dictionary */
ZSTD_compressionStage_e stage;
U32 rep[ZSTD_REP_NUM];
- U32 savedRep[ZSTD_REP_NUM];
+ U32 repToConfirm[ZSTD_REP_NUM];
U32 dictID;
ZSTD_parameters params;
void* workSpace;
@@ -101,7 +101,7 @@
cctx = (ZSTD_CCtx*) ZSTD_malloc(sizeof(ZSTD_CCtx), customMem);
if (!cctx) return NULL;
memset(cctx, 0, sizeof(ZSTD_CCtx));
- memcpy(&(cctx->customMem), &customMem, sizeof(customMem));
+ cctx->customMem = customMem;
return cctx;
}
@@ -119,6 +119,15 @@
return sizeof(*cctx) + cctx->workSpaceSize;
}
+size_t ZSTD_setCCtxParameter(ZSTD_CCtx* cctx, ZSTD_CCtxParameter param, unsigned value)
+{
+ switch(param)
+ {
+ case ZSTD_p_forceWindow : cctx->forceWindow = value>0; cctx->loadedDictEnd = 0; return 0;
+ default: return ERROR(parameter_unknown);
+ }
+}
+
const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) /* hidden interface */
{
return &(ctx->seqStore);
@@ -318,6 +327,14 @@
}
}
+/* ZSTD_invalidateRepCodes() :
+ * ensures next compression will not use repcodes from previous block.
+ * Note : only works with regular variant;
+ * do not use with extDict variant ! */
+void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx) {
+ int i;
+ for (i=0; i<ZSTD_REP_NUM; i++) cctx->rep[i] = 0;
+}
/*! ZSTD_copyCCtx() :
* Duplicate an existing context `srcCCtx` into another one `dstCCtx`.
@@ -735,12 +752,19 @@
if ((size_t)(op-ostart) >= maxCSize) return 0; }
/* confirm repcodes */
- { int i; for (i=0; i<ZSTD_REP_NUM; i++) zc->rep[i] = zc->savedRep[i]; }
+ { int i; for (i=0; i<ZSTD_REP_NUM; i++) zc->rep[i] = zc->repToConfirm[i]; }
return op - ostart;
}
+#if 0 /* for debug */
+# define STORESEQ_DEBUG
+#include <stdio.h> /* fprintf */
+U32 g_startDebug = 0;
+const BYTE* g_start = NULL;
+#endif
+
/*! ZSTD_storeSeq() :
Store a sequence (literal length, literals, offset code and match length code) into seqStore_t.
`offsetCode` : distance to match, or 0 == repCode.
@@ -748,13 +772,14 @@
*/
MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const void* literals, U32 offsetCode, size_t matchCode)
{
-#if 0 /* for debug */
- static const BYTE* g_start = NULL;
- const U32 pos = (U32)((const BYTE*)literals - g_start);
- if (g_start==NULL) g_start = (const BYTE*)literals;
- //if ((pos > 1) && (pos < 50000))
- printf("Cpos %6u :%5u literals & match %3u bytes at distance %6u \n",
- pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode);
+#ifdef STORESEQ_DEBUG
+ if (g_startDebug) {
+ const U32 pos = (U32)((const BYTE*)literals - g_start);
+ if (g_start==NULL) g_start = (const BYTE*)literals;
+ if ((pos > 1895000) && (pos < 1895300))
+ fprintf(stderr, "Cpos %6u :%5u literals & match %3u bytes at distance %6u \n",
+ pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode);
+ }
#endif
/* copy Literals */
ZSTD_wildcopy(seqStorePtr->lit, literals, litLength);
@@ -1004,8 +1029,8 @@
} } }
/* save reps for next block */
- cctx->savedRep[0] = offset_1 ? offset_1 : offsetSaved;
- cctx->savedRep[1] = offset_2 ? offset_2 : offsetSaved;
+ cctx->repToConfirm[0] = offset_1 ? offset_1 : offsetSaved;
+ cctx->repToConfirm[1] = offset_2 ? offset_2 : offsetSaved;
/* Last Literals */
{ size_t const lastLLSize = iend - anchor;
@@ -1119,7 +1144,7 @@
} } }
/* save reps for next block */
- ctx->savedRep[0] = offset_1; ctx->savedRep[1] = offset_2;
+ ctx->repToConfirm[0] = offset_1; ctx->repToConfirm[1] = offset_2;
/* Last Literals */
{ size_t const lastLLSize = iend - anchor;
@@ -1273,8 +1298,8 @@
} } }
/* save reps for next block */
- cctx->savedRep[0] = offset_1 ? offset_1 : offsetSaved;
- cctx->savedRep[1] = offset_2 ? offset_2 : offsetSaved;
+ cctx->repToConfirm[0] = offset_1 ? offset_1 : offsetSaved;
+ cctx->repToConfirm[1] = offset_2 ? offset_2 : offsetSaved;
/* Last Literals */
{ size_t const lastLLSize = iend - anchor;
@@ -1423,7 +1448,7 @@
} } }
/* save reps for next block */
- ctx->savedRep[0] = offset_1; ctx->savedRep[1] = offset_2;
+ ctx->repToConfirm[0] = offset_1; ctx->repToConfirm[1] = offset_2;
/* Last Literals */
{ size_t const lastLLSize = iend - anchor;
@@ -1955,8 +1980,8 @@
} }
/* Save reps for next block */
- ctx->savedRep[0] = offset_1 ? offset_1 : savedOffset;
- ctx->savedRep[1] = offset_2 ? offset_2 : savedOffset;
+ ctx->repToConfirm[0] = offset_1 ? offset_1 : savedOffset;
+ ctx->repToConfirm[1] = offset_2 ? offset_2 : savedOffset;
/* Last Literals */
{ size_t const lastLLSize = iend - anchor;
@@ -2150,7 +2175,7 @@
} }
/* Save reps for next block */
- ctx->savedRep[0] = offset_1; ctx->savedRep[1] = offset_2;
+ ctx->repToConfirm[0] = offset_1; ctx->repToConfirm[1] = offset_2;
/* Last Literals */
{ size_t const lastLLSize = iend - anchor;
@@ -2409,12 +2434,14 @@
cctx->nextSrc = ip + srcSize;
- { size_t const cSize = frame ?
+ if (srcSize) {
+ size_t const cSize = frame ?
ZSTD_compress_generic (cctx, dst, dstCapacity, src, srcSize, lastFrameChunk) :
ZSTD_compressBlock_internal (cctx, dst, dstCapacity, src, srcSize);
if (ZSTD_isError(cSize)) return cSize;
return cSize + fhSize;
- }
+ } else
+ return fhSize;
}
@@ -2450,7 +2477,7 @@
zc->dictBase = zc->base;
zc->base += ip - zc->nextSrc;
zc->nextToUpdate = zc->dictLimit;
- zc->loadedDictEnd = (U32)(iend - zc->base);
+ zc->loadedDictEnd = zc->forceWindow ? 0 : (U32)(iend - zc->base);
zc->nextSrc = iend;
if (srcSize <= HASH_READ_SIZE) return 0;
@@ -2557,9 +2584,9 @@
}
if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted);
- cctx->rep[0] = MEM_readLE32(dictPtr+0); if (cctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted);
- cctx->rep[1] = MEM_readLE32(dictPtr+4); if (cctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted);
- cctx->rep[2] = MEM_readLE32(dictPtr+8); if (cctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted);
+ cctx->rep[0] = MEM_readLE32(dictPtr+0); if (cctx->rep[0] == 0 || cctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted);
+ cctx->rep[1] = MEM_readLE32(dictPtr+4); if (cctx->rep[1] == 0 || cctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted);
+ cctx->rep[2] = MEM_readLE32(dictPtr+8); if (cctx->rep[2] == 0 || cctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted);
dictPtr += 12;
{ U32 offcodeMax = MaxOff;
@@ -2594,7 +2621,6 @@
}
}
-
/*! ZSTD_compressBegin_internal() :
* @return : 0, or an error code */
static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
@@ -2626,9 +2652,9 @@
}
-size_t ZSTD_compressBegin(ZSTD_CCtx* zc, int compressionLevel)
+size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel)
{
- return ZSTD_compressBegin_usingDict(zc, NULL, 0, compressionLevel);
+ return ZSTD_compressBegin_usingDict(cctx, NULL, 0, compressionLevel);
}
@@ -2733,7 +2759,8 @@
/* ===== Dictionary API ===== */
struct ZSTD_CDict_s {
- void* dictContent;
+ void* dictBuffer;
+ const void* dictContent;
size_t dictContentSize;
ZSTD_CCtx* refContext;
}; /* typedef'd tp ZSTD_CDict within "zstd.h" */
@@ -2741,39 +2768,45 @@
size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict)
{
if (cdict==NULL) return 0; /* support sizeof on NULL */
- return ZSTD_sizeof_CCtx(cdict->refContext) + cdict->dictContentSize;
+ return ZSTD_sizeof_CCtx(cdict->refContext) + (cdict->dictBuffer ? cdict->dictContentSize : 0) + sizeof(*cdict);
}
-ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, ZSTD_parameters params, ZSTD_customMem customMem)
+ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, unsigned byReference,
+ ZSTD_parameters params, ZSTD_customMem customMem)
{
if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem;
if (!customMem.customAlloc || !customMem.customFree) return NULL;
{ ZSTD_CDict* const cdict = (ZSTD_CDict*) ZSTD_malloc(sizeof(ZSTD_CDict), customMem);
- void* const dictContent = ZSTD_malloc(dictSize, customMem);
ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(customMem);
- if (!dictContent || !cdict || !cctx) {
- ZSTD_free(dictContent, customMem);
+ if (!cdict || !cctx) {
ZSTD_free(cdict, customMem);
ZSTD_free(cctx, customMem);
return NULL;
}
- if (dictSize) {
- memcpy(dictContent, dict, dictSize);
+ if ((byReference) || (!dictBuffer) || (!dictSize)) {
+ cdict->dictBuffer = NULL;
+ cdict->dictContent = dictBuffer;
+ } else {
+ void* const internalBuffer = ZSTD_malloc(dictSize, customMem);
+ if (!internalBuffer) { ZSTD_free(cctx, customMem); ZSTD_free(cdict, customMem); return NULL; }
+ memcpy(internalBuffer, dictBuffer, dictSize);
+ cdict->dictBuffer = internalBuffer;
+ cdict->dictContent = internalBuffer;
}
- { size_t const errorCode = ZSTD_compressBegin_advanced(cctx, dictContent, dictSize, params, 0);
+
+ { size_t const errorCode = ZSTD_compressBegin_advanced(cctx, cdict->dictContent, dictSize, params, 0);
if (ZSTD_isError(errorCode)) {
- ZSTD_free(dictContent, customMem);
+ ZSTD_free(cdict->dictBuffer, customMem);
+ ZSTD_free(cctx, customMem);
ZSTD_free(cdict, customMem);
- ZSTD_free(cctx, customMem);
return NULL;
} }
- cdict->dictContent = dictContent;
+ cdict->refContext = cctx;
cdict->dictContentSize = dictSize;
- cdict->refContext = cctx;
return cdict;
}
}
@@ -2783,7 +2816,15 @@
ZSTD_customMem const allocator = { NULL, NULL, NULL };
ZSTD_parameters params = ZSTD_getParams(compressionLevel, 0, dictSize);
params.fParams.contentSizeFlag = 1;
- return ZSTD_createCDict_advanced(dict, dictSize, params, allocator);
+ return ZSTD_createCDict_advanced(dict, dictSize, 0, params, allocator);
+}
+
+ZSTD_CDict* ZSTD_createCDict_byReference(const void* dict, size_t dictSize, int compressionLevel)
+{
+ ZSTD_customMem const allocator = { NULL, NULL, NULL };
+ ZSTD_parameters params = ZSTD_getParams(compressionLevel, 0, dictSize);
+ params.fParams.contentSizeFlag = 1;
+ return ZSTD_createCDict_advanced(dict, dictSize, 1, params, allocator);
}
size_t ZSTD_freeCDict(ZSTD_CDict* cdict)
@@ -2791,7 +2832,7 @@
if (cdict==NULL) return 0; /* support free on NULL */
{ ZSTD_customMem const cMem = cdict->refContext->customMem;
ZSTD_freeCCtx(cdict->refContext);
- ZSTD_free(cdict->dictContent, cMem);
+ ZSTD_free(cdict->dictBuffer, cMem);
ZSTD_free(cdict, cMem);
return 0;
}
@@ -2801,7 +2842,7 @@
return ZSTD_getParamsFromCCtx(cdict->refContext);
}
-size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, U64 pledgedSrcSize)
+size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize)
{
if (cdict->dictContentSize) CHECK_F(ZSTD_copyCCtx(cctx, cdict->refContext, pledgedSrcSize))
else CHECK_F(ZSTD_compressBegin_advanced(cctx, NULL, 0, cdict->refContext->params, pledgedSrcSize));
@@ -2900,7 +2941,7 @@
size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
{
- if (zcs->inBuffSize==0) return ERROR(stage_wrong); /* zcs has not been init at least once */
+ if (zcs->inBuffSize==0) return ERROR(stage_wrong); /* zcs has not been init at least once => can't reset */
if (zcs->cdict) CHECK_F(ZSTD_compressBegin_usingCDict(zcs->cctx, zcs->cdict, pledgedSrcSize))
else CHECK_F(ZSTD_compressBegin_advanced(zcs->cctx, NULL, 0, zcs->params, pledgedSrcSize));
@@ -2937,9 +2978,9 @@
if (zcs->outBuff == NULL) return ERROR(memory_allocation);
}
- if (dict) {
+ if (dict && dictSize >= 8) {
ZSTD_freeCDict(zcs->cdictLocal);
- zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, params, zcs->customMem);
+ zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, 0, params, zcs->customMem);
if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);
zcs->cdict = zcs->cdictLocal;
} else zcs->cdict = NULL;
@@ -2956,6 +2997,7 @@
ZSTD_parameters const params = ZSTD_getParamsFromCDict(cdict);
size_t const initError = ZSTD_initCStream_advanced(zcs, NULL, 0, params, 0);
zcs->cdict = cdict;
+ zcs->cctx->dictID = params.fParams.noDictIDFlag ? 0 : cdict->refContext->dictID;
return initError;
}
@@ -2967,7 +3009,8 @@
size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize)
{
- ZSTD_parameters const params = ZSTD_getParams(compressionLevel, pledgedSrcSize, 0);
+ ZSTD_parameters params = ZSTD_getParams(compressionLevel, pledgedSrcSize, 0);
+ if (pledgedSrcSize) params.fParams.contentSizeFlag = 1;
return ZSTD_initCStream_advanced(zcs, NULL, 0, params, pledgedSrcSize);
}
--- a/contrib/python-zstandard/zstd/compress/zstd_opt.h Sat Feb 25 12:48:50 2017 +0900
+++ b/contrib/python-zstandard/zstd/compress/zstd_opt.h Tue Feb 28 11:13:25 2017 -0800
@@ -38,7 +38,7 @@
ssPtr->cachedLiterals = NULL;
ssPtr->cachedPrice = ssPtr->cachedLitLength = 0;
- ssPtr->staticPrices = 0;
+ ssPtr->staticPrices = 0;
if (ssPtr->litLengthSum == 0) {
if (srcSize <= 1024) ssPtr->staticPrices = 1;
@@ -56,7 +56,7 @@
for (u=0; u<=MaxLit; u++) {
ssPtr->litFreq[u] = 1 + (ssPtr->litFreq[u]>>ZSTD_FREQ_DIV);
- ssPtr->litSum += ssPtr->litFreq[u];
+ ssPtr->litSum += ssPtr->litFreq[u];
}
for (u=0; u<=MaxLL; u++)
ssPtr->litLengthFreq[u] = 1;
@@ -634,7 +634,7 @@
} } /* for (cur=0; cur < last_pos; ) */
/* Save reps for next block */
- { int i; for (i=0; i<ZSTD_REP_NUM; i++) ctx->savedRep[i] = rep[i]; }
+ { int i; for (i=0; i<ZSTD_REP_NUM; i++) ctx->repToConfirm[i] = rep[i]; }
/* Last Literals */
{ size_t const lastLLSize = iend - anchor;
@@ -825,7 +825,7 @@
match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, inr, iend, maxSearches, mls, matches, minMatch);
- if (match_num > 0 && matches[match_num-1].len > sufficient_len) {
+ if (match_num > 0 && (matches[match_num-1].len > sufficient_len || cur + matches[match_num-1].len >= ZSTD_OPT_NUM)) {
best_mlen = matches[match_num-1].len;
best_off = matches[match_num-1].off;
last_pos = cur + 1;
@@ -835,7 +835,7 @@
/* set prices using matches at position = cur */
for (u = 0; u < match_num; u++) {
mlen = (u>0) ? matches[u-1].len+1 : best_mlen;
- best_mlen = (cur + matches[u].len < ZSTD_OPT_NUM) ? matches[u].len : ZSTD_OPT_NUM - cur;
+ best_mlen = matches[u].len;
while (mlen <= best_mlen) {
if (opt[cur].mlen == 1) {
@@ -907,7 +907,7 @@
} } /* for (cur=0; cur < last_pos; ) */
/* Save reps for next block */
- { int i; for (i=0; i<ZSTD_REP_NUM; i++) ctx->savedRep[i] = rep[i]; }
+ { int i; for (i=0; i<ZSTD_REP_NUM; i++) ctx->repToConfirm[i] = rep[i]; }
/* Last Literals */
{ size_t lastLLSize = iend - anchor;
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/contrib/python-zstandard/zstd/compress/zstdmt_compress.c Tue Feb 28 11:13:25 2017 -0800
@@ -0,0 +1,740 @@
+/**
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+
+/* ====== Tuning parameters ====== */
+#define ZSTDMT_NBTHREADS_MAX 128
+
+
+/* ====== Compiler specifics ====== */
+#if defined(_MSC_VER)
+# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */
+#endif
+
+
+/* ====== Dependencies ====== */
+#include <stdlib.h> /* malloc */
+#include <string.h> /* memcpy */
+#include "pool.h" /* threadpool */
+#include "threading.h" /* mutex */
+#include "zstd_internal.h" /* MIN, ERROR, ZSTD_*, ZSTD_highbit32 */
+#include "zstdmt_compress.h"
+#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
+#include "xxhash.h"
+
+
+/* ====== Debug ====== */
+#if 0
+
+# include <stdio.h>
+# include <unistd.h>
+# include <sys/times.h>
+ static unsigned g_debugLevel = 3;
+# define DEBUGLOGRAW(l, ...) if (l<=g_debugLevel) { fprintf(stderr, __VA_ARGS__); }
+# define DEBUGLOG(l, ...) if (l<=g_debugLevel) { fprintf(stderr, __FILE__ ": "); fprintf(stderr, __VA_ARGS__); fprintf(stderr, " \n"); }
+
+# define DEBUG_PRINTHEX(l,p,n) { \
+ unsigned debug_u; \
+ for (debug_u=0; debug_u<(n); debug_u++) \
+ DEBUGLOGRAW(l, "%02X ", ((const unsigned char*)(p))[debug_u]); \
+ DEBUGLOGRAW(l, " \n"); \
+}
+
+static unsigned long long GetCurrentClockTimeMicroseconds()
+{
+ static clock_t _ticksPerSecond = 0;
+ if (_ticksPerSecond <= 0) _ticksPerSecond = sysconf(_SC_CLK_TCK);
+
+ struct tms junk; clock_t newTicks = (clock_t) times(&junk);
+ return ((((unsigned long long)newTicks)*(1000000))/_ticksPerSecond);
+}
+
+#define MUTEX_WAIT_TIME_DLEVEL 5
+#define PTHREAD_MUTEX_LOCK(mutex) \
+if (g_debugLevel>=MUTEX_WAIT_TIME_DLEVEL) { \
+ unsigned long long beforeTime = GetCurrentClockTimeMicroseconds(); \
+ pthread_mutex_lock(mutex); \
+ unsigned long long afterTime = GetCurrentClockTimeMicroseconds(); \
+ unsigned long long elapsedTime = (afterTime-beforeTime); \
+ if (elapsedTime > 1000) { /* or whatever threshold you like; I'm using 1 millisecond here */ \
+ DEBUGLOG(MUTEX_WAIT_TIME_DLEVEL, "Thread took %llu microseconds to acquire mutex %s \n", \
+ elapsedTime, #mutex); \
+ } \
+} else pthread_mutex_lock(mutex);
+
+#else
+
+# define DEBUGLOG(l, ...) {} /* disabled */
+# define PTHREAD_MUTEX_LOCK(m) pthread_mutex_lock(m)
+# define DEBUG_PRINTHEX(l,p,n) {}
+
+#endif
+
+
+/* ===== Buffer Pool ===== */
+
+typedef struct buffer_s {
+ void* start;
+ size_t size;
+} buffer_t;
+
+static const buffer_t g_nullBuffer = { NULL, 0 };
+
+typedef struct ZSTDMT_bufferPool_s {
+ unsigned totalBuffers;
+ unsigned nbBuffers;
+ buffer_t bTable[1]; /* variable size */
+} ZSTDMT_bufferPool;
+
+static ZSTDMT_bufferPool* ZSTDMT_createBufferPool(unsigned nbThreads)
+{
+ unsigned const maxNbBuffers = 2*nbThreads + 2;
+ ZSTDMT_bufferPool* const bufPool = (ZSTDMT_bufferPool*)calloc(1, sizeof(ZSTDMT_bufferPool) + (maxNbBuffers-1) * sizeof(buffer_t));
+ if (bufPool==NULL) return NULL;
+ bufPool->totalBuffers = maxNbBuffers;
+ bufPool->nbBuffers = 0;
+ return bufPool;
+}
+
+static void ZSTDMT_freeBufferPool(ZSTDMT_bufferPool* bufPool)
+{
+ unsigned u;
+ if (!bufPool) return; /* compatibility with free on NULL */
+ for (u=0; u<bufPool->totalBuffers; u++)
+ free(bufPool->bTable[u].start);
+ free(bufPool);
+}
+
+/* assumption : invocation from main thread only ! */
+static buffer_t ZSTDMT_getBuffer(ZSTDMT_bufferPool* pool, size_t bSize)
+{
+ if (pool->nbBuffers) { /* try to use an existing buffer */
+ buffer_t const buf = pool->bTable[--(pool->nbBuffers)];
+ size_t const availBufferSize = buf.size;
+ if ((availBufferSize >= bSize) & (availBufferSize <= 10*bSize)) /* large enough, but not too much */
+ return buf;
+ free(buf.start); /* size conditions not respected : scratch this buffer and create a new one */
+ }
+ /* create new buffer */
+ { buffer_t buffer;
+ void* const start = malloc(bSize);
+ if (start==NULL) bSize = 0;
+ buffer.start = start; /* note : start can be NULL if malloc fails ! */
+ buffer.size = bSize;
+ return buffer;
+ }
+}
+
+/* store buffer for later re-use, up to pool capacity */
+static void ZSTDMT_releaseBuffer(ZSTDMT_bufferPool* pool, buffer_t buf)
+{
+ if (buf.start == NULL) return; /* release on NULL */
+ if (pool->nbBuffers < pool->totalBuffers) {
+ pool->bTable[pool->nbBuffers++] = buf; /* store for later re-use */
+ return;
+ }
+ /* Reached bufferPool capacity (should not happen) */
+ free(buf.start);
+}
+
+
+/* ===== CCtx Pool ===== */
+
+typedef struct {
+ unsigned totalCCtx;
+ unsigned availCCtx;
+ ZSTD_CCtx* cctx[1]; /* variable size */
+} ZSTDMT_CCtxPool;
+
+/* assumption : CCtxPool invocation only from main thread */
+
+/* note : all CCtx borrowed from the pool should be released back to the pool _before_ freeing the pool */
+static void ZSTDMT_freeCCtxPool(ZSTDMT_CCtxPool* pool)
+{
+ unsigned u;
+ for (u=0; u<pool->totalCCtx; u++)
+ ZSTD_freeCCtx(pool->cctx[u]); /* note : compatible with free on NULL */
+ free(pool);
+}
+
+/* ZSTDMT_createCCtxPool() :
+ * implies nbThreads >= 1 , checked by caller ZSTDMT_createCCtx() */
+static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(unsigned nbThreads)
+{
+ ZSTDMT_CCtxPool* const cctxPool = (ZSTDMT_CCtxPool*) calloc(1, sizeof(ZSTDMT_CCtxPool) + (nbThreads-1)*sizeof(ZSTD_CCtx*));
+ if (!cctxPool) return NULL;
+ cctxPool->totalCCtx = nbThreads;
+ cctxPool->availCCtx = 1; /* at least one cctx for single-thread mode */
+ cctxPool->cctx[0] = ZSTD_createCCtx();
+ if (!cctxPool->cctx[0]) { ZSTDMT_freeCCtxPool(cctxPool); return NULL; }
+ DEBUGLOG(1, "cctxPool created, with %u threads", nbThreads);
+ return cctxPool;
+}
+
+static ZSTD_CCtx* ZSTDMT_getCCtx(ZSTDMT_CCtxPool* pool)
+{
+ if (pool->availCCtx) {
+ pool->availCCtx--;
+ return pool->cctx[pool->availCCtx];
+ }
+ return ZSTD_createCCtx(); /* note : can be NULL, when creation fails ! */
+}
+
+static void ZSTDMT_releaseCCtx(ZSTDMT_CCtxPool* pool, ZSTD_CCtx* cctx)
+{
+ if (cctx==NULL) return; /* compatibility with release on NULL */
+ if (pool->availCCtx < pool->totalCCtx)
+ pool->cctx[pool->availCCtx++] = cctx;
+ else
+ /* pool overflow : should not happen, since totalCCtx==nbThreads */
+ ZSTD_freeCCtx(cctx);
+}
+
+
+/* ===== Thread worker ===== */
+
+typedef struct {
+ buffer_t buffer;
+ size_t filled;
+} inBuff_t;
+
+typedef struct {
+ ZSTD_CCtx* cctx;
+ buffer_t src;
+ const void* srcStart;
+ size_t srcSize;
+ size_t dictSize;
+ buffer_t dstBuff;
+ size_t cSize;
+ size_t dstFlushed;
+ unsigned firstChunk;
+ unsigned lastChunk;
+ unsigned jobCompleted;
+ unsigned jobScanned;
+ pthread_mutex_t* jobCompleted_mutex;
+ pthread_cond_t* jobCompleted_cond;
+ ZSTD_parameters params;
+ ZSTD_CDict* cdict;
+ unsigned long long fullFrameSize;
+} ZSTDMT_jobDescription;
+
+/* ZSTDMT_compressChunk() : POOL_function type */
+void ZSTDMT_compressChunk(void* jobDescription)
+{
+ ZSTDMT_jobDescription* const job = (ZSTDMT_jobDescription*)jobDescription;
+ const void* const src = (const char*)job->srcStart + job->dictSize;
+ buffer_t const dstBuff = job->dstBuff;
+ DEBUGLOG(3, "job (first:%u) (last:%u) : dictSize %u, srcSize %u", job->firstChunk, job->lastChunk, (U32)job->dictSize, (U32)job->srcSize);
+ if (job->cdict) {
+ size_t const initError = ZSTD_compressBegin_usingCDict(job->cctx, job->cdict, job->fullFrameSize);
+ if (job->cdict) DEBUGLOG(3, "using CDict ");
+ if (ZSTD_isError(initError)) { job->cSize = initError; goto _endJob; }
+ } else {
+ size_t const initError = ZSTD_compressBegin_advanced(job->cctx, job->srcStart, job->dictSize, job->params, job->fullFrameSize);
+ if (ZSTD_isError(initError)) { job->cSize = initError; goto _endJob; }
+ ZSTD_setCCtxParameter(job->cctx, ZSTD_p_forceWindow, 1);
+ }
+ if (!job->firstChunk) { /* flush frame header */
+ size_t const hSize = ZSTD_compressContinue(job->cctx, dstBuff.start, dstBuff.size, src, 0);
+ if (ZSTD_isError(hSize)) { job->cSize = hSize; goto _endJob; }
+ ZSTD_invalidateRepCodes(job->cctx);
+ }
+
+ DEBUGLOG(4, "Compressing : ");
+ DEBUG_PRINTHEX(4, job->srcStart, 12);
+ job->cSize = (job->lastChunk) ? /* last chunk signal */
+ ZSTD_compressEnd (job->cctx, dstBuff.start, dstBuff.size, src, job->srcSize) :
+ ZSTD_compressContinue(job->cctx, dstBuff.start, dstBuff.size, src, job->srcSize);
+ DEBUGLOG(3, "compressed %u bytes into %u bytes (first:%u) (last:%u)", (unsigned)job->srcSize, (unsigned)job->cSize, job->firstChunk, job->lastChunk);
+
+_endJob:
+ PTHREAD_MUTEX_LOCK(job->jobCompleted_mutex);
+ job->jobCompleted = 1;
+ job->jobScanned = 0;
+ pthread_cond_signal(job->jobCompleted_cond);
+ pthread_mutex_unlock(job->jobCompleted_mutex);
+}
+
+
+/* ------------------------------------------ */
+/* ===== Multi-threaded compression ===== */
+/* ------------------------------------------ */
+
+struct ZSTDMT_CCtx_s {
+ POOL_ctx* factory;
+ ZSTDMT_bufferPool* buffPool;
+ ZSTDMT_CCtxPool* cctxPool;
+ pthread_mutex_t jobCompleted_mutex;
+ pthread_cond_t jobCompleted_cond;
+ size_t targetSectionSize;
+ size_t marginSize;
+ size_t inBuffSize;
+ size_t dictSize;
+ size_t targetDictSize;
+ inBuff_t inBuff;
+ ZSTD_parameters params;
+ XXH64_state_t xxhState;
+ unsigned nbThreads;
+ unsigned jobIDMask;
+ unsigned doneJobID;
+ unsigned nextJobID;
+ unsigned frameEnded;
+ unsigned allJobsCompleted;
+ unsigned overlapRLog;
+ unsigned long long frameContentSize;
+ size_t sectionSize;
+ ZSTD_CDict* cdict;
+ ZSTD_CStream* cstream;
+ ZSTDMT_jobDescription jobs[1]; /* variable size (must lies at the end) */
+};
+
+ZSTDMT_CCtx *ZSTDMT_createCCtx(unsigned nbThreads)
+{
+ ZSTDMT_CCtx* cctx;
+ U32 const minNbJobs = nbThreads + 2;
+ U32 const nbJobsLog2 = ZSTD_highbit32(minNbJobs) + 1;
+ U32 const nbJobs = 1 << nbJobsLog2;
+ DEBUGLOG(5, "nbThreads : %u ; minNbJobs : %u ; nbJobsLog2 : %u ; nbJobs : %u \n",
+ nbThreads, minNbJobs, nbJobsLog2, nbJobs);
+ if ((nbThreads < 1) | (nbThreads > ZSTDMT_NBTHREADS_MAX)) return NULL;
+ cctx = (ZSTDMT_CCtx*) calloc(1, sizeof(ZSTDMT_CCtx) + nbJobs*sizeof(ZSTDMT_jobDescription));
+ if (!cctx) return NULL;
+ cctx->nbThreads = nbThreads;
+ cctx->jobIDMask = nbJobs - 1;
+ cctx->allJobsCompleted = 1;
+ cctx->sectionSize = 0;
+ cctx->overlapRLog = 3;
+ cctx->factory = POOL_create(nbThreads, 1);
+ cctx->buffPool = ZSTDMT_createBufferPool(nbThreads);
+ cctx->cctxPool = ZSTDMT_createCCtxPool(nbThreads);
+ if (!cctx->factory | !cctx->buffPool | !cctx->cctxPool) { /* one object was not created */
+ ZSTDMT_freeCCtx(cctx);
+ return NULL;
+ }
+ if (nbThreads==1) {
+ cctx->cstream = ZSTD_createCStream();
+ if (!cctx->cstream) {
+ ZSTDMT_freeCCtx(cctx); return NULL;
+ } }
+ pthread_mutex_init(&cctx->jobCompleted_mutex, NULL); /* Todo : check init function return */
+ pthread_cond_init(&cctx->jobCompleted_cond, NULL);
+ DEBUGLOG(4, "mt_cctx created, for %u threads \n", nbThreads);
+ return cctx;
+}
+
+/* ZSTDMT_releaseAllJobResources() :
+ * Ensure all workers are killed first. */
+static void ZSTDMT_releaseAllJobResources(ZSTDMT_CCtx* mtctx)
+{
+ unsigned jobID;
+ for (jobID=0; jobID <= mtctx->jobIDMask; jobID++) {
+ ZSTDMT_releaseBuffer(mtctx->buffPool, mtctx->jobs[jobID].dstBuff);
+ mtctx->jobs[jobID].dstBuff = g_nullBuffer;
+ ZSTDMT_releaseBuffer(mtctx->buffPool, mtctx->jobs[jobID].src);
+ mtctx->jobs[jobID].src = g_nullBuffer;
+ ZSTDMT_releaseCCtx(mtctx->cctxPool, mtctx->jobs[jobID].cctx);
+ mtctx->jobs[jobID].cctx = NULL;
+ }
+ memset(mtctx->jobs, 0, (mtctx->jobIDMask+1)*sizeof(ZSTDMT_jobDescription));
+ ZSTDMT_releaseBuffer(mtctx->buffPool, mtctx->inBuff.buffer);
+ mtctx->inBuff.buffer = g_nullBuffer;
+ mtctx->allJobsCompleted = 1;
+}
+
+size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx)
+{
+ if (mtctx==NULL) return 0; /* compatible with free on NULL */
+ POOL_free(mtctx->factory);
+ if (!mtctx->allJobsCompleted) ZSTDMT_releaseAllJobResources(mtctx); /* stop workers first */
+ ZSTDMT_freeBufferPool(mtctx->buffPool); /* release job resources into pools first */
+ ZSTDMT_freeCCtxPool(mtctx->cctxPool);
+ ZSTD_freeCDict(mtctx->cdict);
+ ZSTD_freeCStream(mtctx->cstream);
+ pthread_mutex_destroy(&mtctx->jobCompleted_mutex);
+ pthread_cond_destroy(&mtctx->jobCompleted_cond);
+ free(mtctx);
+ return 0;
+}
+
+size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSDTMT_parameter parameter, unsigned value)
+{
+ switch(parameter)
+ {
+ case ZSTDMT_p_sectionSize :
+ mtctx->sectionSize = value;
+ return 0;
+ case ZSTDMT_p_overlapSectionLog :
+ DEBUGLOG(4, "ZSTDMT_p_overlapSectionLog : %u", value);
+ mtctx->overlapRLog = (value >= 9) ? 0 : 9 - value;
+ return 0;
+ default :
+ return ERROR(compressionParameter_unsupported);
+ }
+}
+
+
+/* ------------------------------------------ */
+/* ===== Multi-threaded compression ===== */
+/* ------------------------------------------ */
+
+size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ int compressionLevel)
+{
+ ZSTD_parameters params = ZSTD_getParams(compressionLevel, srcSize, 0);
+ size_t const chunkTargetSize = (size_t)1 << (params.cParams.windowLog + 2);
+ unsigned const nbChunksMax = (unsigned)(srcSize / chunkTargetSize) + (srcSize < chunkTargetSize) /* min 1 */;
+ unsigned nbChunks = MIN(nbChunksMax, mtctx->nbThreads);
+ size_t const proposedChunkSize = (srcSize + (nbChunks-1)) / nbChunks;
+ size_t const avgChunkSize = ((proposedChunkSize & 0x1FFFF) < 0xFFFF) ? proposedChunkSize + 0xFFFF : proposedChunkSize; /* avoid too small last block */
+ size_t remainingSrcSize = srcSize;
+ const char* const srcStart = (const char*)src;
+ size_t frameStartPos = 0;
+
+ DEBUGLOG(3, "windowLog : %2u => chunkTargetSize : %u bytes ", params.cParams.windowLog, (U32)chunkTargetSize);
+ DEBUGLOG(2, "nbChunks : %2u (chunkSize : %u bytes) ", nbChunks, (U32)avgChunkSize);
+ params.fParams.contentSizeFlag = 1;
+
+ if (nbChunks==1) { /* fallback to single-thread mode */
+ ZSTD_CCtx* const cctx = mtctx->cctxPool->cctx[0];
+ return ZSTD_compressCCtx(cctx, dst, dstCapacity, src, srcSize, compressionLevel);
+ }
+
+ { unsigned u;
+ for (u=0; u<nbChunks; u++) {
+ size_t const chunkSize = MIN(remainingSrcSize, avgChunkSize);
+ size_t const dstBufferCapacity = u ? ZSTD_compressBound(chunkSize) : dstCapacity;
+ buffer_t const dstAsBuffer = { dst, dstCapacity };
+ buffer_t const dstBuffer = u ? ZSTDMT_getBuffer(mtctx->buffPool, dstBufferCapacity) : dstAsBuffer;
+ ZSTD_CCtx* const cctx = ZSTDMT_getCCtx(mtctx->cctxPool);
+
+ if ((cctx==NULL) || (dstBuffer.start==NULL)) {
+ mtctx->jobs[u].cSize = ERROR(memory_allocation); /* job result */
+ mtctx->jobs[u].jobCompleted = 1;
+ nbChunks = u+1;
+ break; /* let's wait for previous jobs to complete, but don't start new ones */
+ }
+
+ mtctx->jobs[u].srcStart = srcStart + frameStartPos;
+ mtctx->jobs[u].srcSize = chunkSize;
+ mtctx->jobs[u].fullFrameSize = srcSize;
+ mtctx->jobs[u].params = params;
+ mtctx->jobs[u].dstBuff = dstBuffer;
+ mtctx->jobs[u].cctx = cctx;
+ mtctx->jobs[u].firstChunk = (u==0);
+ mtctx->jobs[u].lastChunk = (u==nbChunks-1);
+ mtctx->jobs[u].jobCompleted = 0;
+ mtctx->jobs[u].jobCompleted_mutex = &mtctx->jobCompleted_mutex;
+ mtctx->jobs[u].jobCompleted_cond = &mtctx->jobCompleted_cond;
+
+ DEBUGLOG(3, "posting job %u (%u bytes)", u, (U32)chunkSize);
+ DEBUG_PRINTHEX(3, mtctx->jobs[u].srcStart, 12);
+ POOL_add(mtctx->factory, ZSTDMT_compressChunk, &mtctx->jobs[u]);
+
+ frameStartPos += chunkSize;
+ remainingSrcSize -= chunkSize;
+ } }
+ /* note : since nbChunks <= nbThreads, all jobs should be running immediately in parallel */
+
+ { unsigned chunkID;
+ size_t error = 0, dstPos = 0;
+ for (chunkID=0; chunkID<nbChunks; chunkID++) {
+ DEBUGLOG(3, "waiting for chunk %u ", chunkID);
+ PTHREAD_MUTEX_LOCK(&mtctx->jobCompleted_mutex);
+ while (mtctx->jobs[chunkID].jobCompleted==0) {
+ DEBUGLOG(4, "waiting for jobCompleted signal from chunk %u", chunkID);
+ pthread_cond_wait(&mtctx->jobCompleted_cond, &mtctx->jobCompleted_mutex);
+ }
+ pthread_mutex_unlock(&mtctx->jobCompleted_mutex);
+ DEBUGLOG(3, "ready to write chunk %u ", chunkID);
+
+ ZSTDMT_releaseCCtx(mtctx->cctxPool, mtctx->jobs[chunkID].cctx);
+ mtctx->jobs[chunkID].cctx = NULL;
+ mtctx->jobs[chunkID].srcStart = NULL;
+ { size_t const cSize = mtctx->jobs[chunkID].cSize;
+ if (ZSTD_isError(cSize)) error = cSize;
+ if ((!error) && (dstPos + cSize > dstCapacity)) error = ERROR(dstSize_tooSmall);
+ if (chunkID) { /* note : chunk 0 is already written directly into dst */
+ if (!error) memcpy((char*)dst + dstPos, mtctx->jobs[chunkID].dstBuff.start, cSize);
+ ZSTDMT_releaseBuffer(mtctx->buffPool, mtctx->jobs[chunkID].dstBuff);
+ mtctx->jobs[chunkID].dstBuff = g_nullBuffer;
+ }
+ dstPos += cSize ;
+ }
+ }
+ if (!error) DEBUGLOG(3, "compressed size : %u ", (U32)dstPos);
+ return error ? error : dstPos;
+ }
+
+}
+
+
+/* ====================================== */
+/* ======= Streaming API ======= */
+/* ====================================== */
+
+static void ZSTDMT_waitForAllJobsCompleted(ZSTDMT_CCtx* zcs) {
+ while (zcs->doneJobID < zcs->nextJobID) {
+ unsigned const jobID = zcs->doneJobID & zcs->jobIDMask;
+ PTHREAD_MUTEX_LOCK(&zcs->jobCompleted_mutex);
+ while (zcs->jobs[jobID].jobCompleted==0) {
+ DEBUGLOG(4, "waiting for jobCompleted signal from chunk %u", zcs->doneJobID); /* we want to block when waiting for data to flush */
+ pthread_cond_wait(&zcs->jobCompleted_cond, &zcs->jobCompleted_mutex);
+ }
+ pthread_mutex_unlock(&zcs->jobCompleted_mutex);
+ zcs->doneJobID++;
+ }
+}
+
+
+static size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs,
+ const void* dict, size_t dictSize, unsigned updateDict,
+ ZSTD_parameters params, unsigned long long pledgedSrcSize)
+{
+ ZSTD_customMem const cmem = { NULL, NULL, NULL };
+ DEBUGLOG(3, "Started new compression, with windowLog : %u", params.cParams.windowLog);
+ if (zcs->nbThreads==1) return ZSTD_initCStream_advanced(zcs->cstream, dict, dictSize, params, pledgedSrcSize);
+ if (zcs->allJobsCompleted == 0) { /* previous job not correctly finished */
+ ZSTDMT_waitForAllJobsCompleted(zcs);
+ ZSTDMT_releaseAllJobResources(zcs);
+ zcs->allJobsCompleted = 1;
+ }
+ zcs->params = params;
+ if (updateDict) {
+ ZSTD_freeCDict(zcs->cdict); zcs->cdict = NULL;
+ if (dict && dictSize) {
+ zcs->cdict = ZSTD_createCDict_advanced(dict, dictSize, 0, params, cmem);
+ if (zcs->cdict == NULL) return ERROR(memory_allocation);
+ } }
+ zcs->frameContentSize = pledgedSrcSize;
+ zcs->targetDictSize = (zcs->overlapRLog>=9) ? 0 : (size_t)1 << (zcs->params.cParams.windowLog - zcs->overlapRLog);
+ DEBUGLOG(4, "overlapRLog : %u ", zcs->overlapRLog);
+ DEBUGLOG(3, "overlap Size : %u KB", (U32)(zcs->targetDictSize>>10));
+ zcs->targetSectionSize = zcs->sectionSize ? zcs->sectionSize : (size_t)1 << (zcs->params.cParams.windowLog + 2);
+ zcs->targetSectionSize = MAX(ZSTDMT_SECTION_SIZE_MIN, zcs->targetSectionSize);
+ zcs->targetSectionSize = MAX(zcs->targetDictSize, zcs->targetSectionSize);
+ DEBUGLOG(3, "Section Size : %u KB", (U32)(zcs->targetSectionSize>>10));
+ zcs->marginSize = zcs->targetSectionSize >> 2;
+ zcs->inBuffSize = zcs->targetDictSize + zcs->targetSectionSize + zcs->marginSize;
+ zcs->inBuff.buffer = ZSTDMT_getBuffer(zcs->buffPool, zcs->inBuffSize);
+ if (zcs->inBuff.buffer.start == NULL) return ERROR(memory_allocation);
+ zcs->inBuff.filled = 0;
+ zcs->dictSize = 0;
+ zcs->doneJobID = 0;
+ zcs->nextJobID = 0;
+ zcs->frameEnded = 0;
+ zcs->allJobsCompleted = 0;
+ if (params.fParams.checksumFlag) XXH64_reset(&zcs->xxhState, 0);
+ return 0;
+}
+
+size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* zcs,
+ const void* dict, size_t dictSize,
+ ZSTD_parameters params, unsigned long long pledgedSrcSize)
+{
+ return ZSTDMT_initCStream_internal(zcs, dict, dictSize, 1, params, pledgedSrcSize);
+}
+
+/* ZSTDMT_resetCStream() :
+ * pledgedSrcSize is optional and can be zero == unknown */
+size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* zcs, unsigned long long pledgedSrcSize)
+{
+ if (zcs->nbThreads==1) return ZSTD_resetCStream(zcs->cstream, pledgedSrcSize);
+ return ZSTDMT_initCStream_internal(zcs, NULL, 0, 0, zcs->params, pledgedSrcSize);
+}
+
+size_t ZSTDMT_initCStream(ZSTDMT_CCtx* zcs, int compressionLevel) {
+ ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, 0);
+ return ZSTDMT_initCStream_internal(zcs, NULL, 0, 1, params, 0);
+}
+
+
+static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* zcs, size_t srcSize, unsigned endFrame)
+{
+ size_t const dstBufferCapacity = ZSTD_compressBound(srcSize);
+ buffer_t const dstBuffer = ZSTDMT_getBuffer(zcs->buffPool, dstBufferCapacity);
+ ZSTD_CCtx* const cctx = ZSTDMT_getCCtx(zcs->cctxPool);
+ unsigned const jobID = zcs->nextJobID & zcs->jobIDMask;
+
+ if ((cctx==NULL) || (dstBuffer.start==NULL)) {
+ zcs->jobs[jobID].jobCompleted = 1;
+ zcs->nextJobID++;
+ ZSTDMT_waitForAllJobsCompleted(zcs);
+ ZSTDMT_releaseAllJobResources(zcs);
+ return ERROR(memory_allocation);
+ }
+
+ DEBUGLOG(4, "preparing job %u to compress %u bytes with %u preload ", zcs->nextJobID, (U32)srcSize, (U32)zcs->dictSize);
+ zcs->jobs[jobID].src = zcs->inBuff.buffer;
+ zcs->jobs[jobID].srcStart = zcs->inBuff.buffer.start;
+ zcs->jobs[jobID].srcSize = srcSize;
+ zcs->jobs[jobID].dictSize = zcs->dictSize; /* note : zcs->inBuff.filled is presumed >= srcSize + dictSize */
+ zcs->jobs[jobID].params = zcs->params;
+ if (zcs->nextJobID) zcs->jobs[jobID].params.fParams.checksumFlag = 0; /* do not calculate checksum within sections, just keep it in header for first section */
+ zcs->jobs[jobID].cdict = zcs->nextJobID==0 ? zcs->cdict : NULL;
+ zcs->jobs[jobID].fullFrameSize = zcs->frameContentSize;
+ zcs->jobs[jobID].dstBuff = dstBuffer;
+ zcs->jobs[jobID].cctx = cctx;
+ zcs->jobs[jobID].firstChunk = (zcs->nextJobID==0);
+ zcs->jobs[jobID].lastChunk = endFrame;
+ zcs->jobs[jobID].jobCompleted = 0;
+ zcs->jobs[jobID].dstFlushed = 0;
+ zcs->jobs[jobID].jobCompleted_mutex = &zcs->jobCompleted_mutex;
+ zcs->jobs[jobID].jobCompleted_cond = &zcs->jobCompleted_cond;
+
+ /* get a new buffer for next input */
+ if (!endFrame) {
+ size_t const newDictSize = MIN(srcSize + zcs->dictSize, zcs->targetDictSize);
+ zcs->inBuff.buffer = ZSTDMT_getBuffer(zcs->buffPool, zcs->inBuffSize);
+ if (zcs->inBuff.buffer.start == NULL) { /* not enough memory to allocate next input buffer */
+ zcs->jobs[jobID].jobCompleted = 1;
+ zcs->nextJobID++;
+ ZSTDMT_waitForAllJobsCompleted(zcs);
+ ZSTDMT_releaseAllJobResources(zcs);
+ return ERROR(memory_allocation);
+ }
+ DEBUGLOG(5, "inBuff filled to %u", (U32)zcs->inBuff.filled);
+ zcs->inBuff.filled -= srcSize + zcs->dictSize - newDictSize;
+ DEBUGLOG(5, "new job : filled to %u, with %u dict and %u src", (U32)zcs->inBuff.filled, (U32)newDictSize, (U32)(zcs->inBuff.filled - newDictSize));
+ memmove(zcs->inBuff.buffer.start, (const char*)zcs->jobs[jobID].srcStart + zcs->dictSize + srcSize - newDictSize, zcs->inBuff.filled);
+ DEBUGLOG(5, "new inBuff pre-filled");
+ zcs->dictSize = newDictSize;
+ } else {
+ zcs->inBuff.buffer = g_nullBuffer;
+ zcs->inBuff.filled = 0;
+ zcs->dictSize = 0;
+ zcs->frameEnded = 1;
+ if (zcs->nextJobID == 0)
+ zcs->params.fParams.checksumFlag = 0; /* single chunk : checksum is calculated directly within worker thread */
+ }
+
+ DEBUGLOG(3, "posting job %u : %u bytes (end:%u) (note : doneJob = %u=>%u)", zcs->nextJobID, (U32)zcs->jobs[jobID].srcSize, zcs->jobs[jobID].lastChunk, zcs->doneJobID, zcs->doneJobID & zcs->jobIDMask);
+ POOL_add(zcs->factory, ZSTDMT_compressChunk, &zcs->jobs[jobID]); /* this call is blocking when thread worker pool is exhausted */
+ zcs->nextJobID++;
+ return 0;
+}
+
+
+/* ZSTDMT_flushNextJob() :
+ * output : will be updated with amount of data flushed .
+ * blockToFlush : if >0, the function will block and wait if there is no data available to flush .
+ * @return : amount of data remaining within internal buffer, 1 if unknown but > 0, 0 if no more, or an error code */
+static size_t ZSTDMT_flushNextJob(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output, unsigned blockToFlush)
+{
+ unsigned const wJobID = zcs->doneJobID & zcs->jobIDMask;
+ if (zcs->doneJobID == zcs->nextJobID) return 0; /* all flushed ! */
+ PTHREAD_MUTEX_LOCK(&zcs->jobCompleted_mutex);
+ while (zcs->jobs[wJobID].jobCompleted==0) {
+ DEBUGLOG(5, "waiting for jobCompleted signal from job %u", zcs->doneJobID);
+ if (!blockToFlush) { pthread_mutex_unlock(&zcs->jobCompleted_mutex); return 0; } /* nothing ready to be flushed => skip */
+ pthread_cond_wait(&zcs->jobCompleted_cond, &zcs->jobCompleted_mutex); /* block when nothing available to flush */
+ }
+ pthread_mutex_unlock(&zcs->jobCompleted_mutex);
+ /* compression job completed : output can be flushed */
+ { ZSTDMT_jobDescription job = zcs->jobs[wJobID];
+ if (!job.jobScanned) {
+ if (ZSTD_isError(job.cSize)) {
+ DEBUGLOG(5, "compression error detected ");
+ ZSTDMT_waitForAllJobsCompleted(zcs);
+ ZSTDMT_releaseAllJobResources(zcs);
+ return job.cSize;
+ }
+ ZSTDMT_releaseCCtx(zcs->cctxPool, job.cctx);
+ zcs->jobs[wJobID].cctx = NULL;
+ DEBUGLOG(5, "zcs->params.fParams.checksumFlag : %u ", zcs->params.fParams.checksumFlag);
+ if (zcs->params.fParams.checksumFlag) {
+ XXH64_update(&zcs->xxhState, (const char*)job.srcStart + job.dictSize, job.srcSize);
+ if (zcs->frameEnded && (zcs->doneJobID+1 == zcs->nextJobID)) { /* write checksum at end of last section */
+ U32 const checksum = (U32)XXH64_digest(&zcs->xxhState);
+ DEBUGLOG(4, "writing checksum : %08X \n", checksum);
+ MEM_writeLE32((char*)job.dstBuff.start + job.cSize, checksum);
+ job.cSize += 4;
+ zcs->jobs[wJobID].cSize += 4;
+ } }
+ ZSTDMT_releaseBuffer(zcs->buffPool, job.src);
+ zcs->jobs[wJobID].srcStart = NULL;
+ zcs->jobs[wJobID].src = g_nullBuffer;
+ zcs->jobs[wJobID].jobScanned = 1;
+ }
+ { size_t const toWrite = MIN(job.cSize - job.dstFlushed, output->size - output->pos);
+ DEBUGLOG(4, "Flushing %u bytes from job %u ", (U32)toWrite, zcs->doneJobID);
+ memcpy((char*)output->dst + output->pos, (const char*)job.dstBuff.start + job.dstFlushed, toWrite);
+ output->pos += toWrite;
+ job.dstFlushed += toWrite;
+ }
+ if (job.dstFlushed == job.cSize) { /* output buffer fully flushed => move to next one */
+ ZSTDMT_releaseBuffer(zcs->buffPool, job.dstBuff);
+ zcs->jobs[wJobID].dstBuff = g_nullBuffer;
+ zcs->jobs[wJobID].jobCompleted = 0;
+ zcs->doneJobID++;
+ } else {
+ zcs->jobs[wJobID].dstFlushed = job.dstFlushed;
+ }
+ /* return value : how many bytes left in buffer ; fake it to 1 if unknown but >0 */
+ if (job.cSize > job.dstFlushed) return (job.cSize - job.dstFlushed);
+ if (zcs->doneJobID < zcs->nextJobID) return 1; /* still some buffer to flush */
+ zcs->allJobsCompleted = zcs->frameEnded; /* frame completed and entirely flushed */
+ return 0; /* everything flushed */
+} }
+
+
+size_t ZSTDMT_compressStream(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input)
+{
+ size_t const newJobThreshold = zcs->dictSize + zcs->targetSectionSize + zcs->marginSize;
+ if (zcs->frameEnded) return ERROR(stage_wrong); /* current frame being ended. Only flush is allowed. Restart with init */
+ if (zcs->nbThreads==1) return ZSTD_compressStream(zcs->cstream, output, input);
+
+ /* fill input buffer */
+ { size_t const toLoad = MIN(input->size - input->pos, zcs->inBuffSize - zcs->inBuff.filled);
+ memcpy((char*)zcs->inBuff.buffer.start + zcs->inBuff.filled, input->src, toLoad);
+ input->pos += toLoad;
+ zcs->inBuff.filled += toLoad;
+ }
+
+ if ( (zcs->inBuff.filled >= newJobThreshold) /* filled enough : let's compress */
+ && (zcs->nextJobID <= zcs->doneJobID + zcs->jobIDMask) ) { /* avoid overwriting job round buffer */
+ CHECK_F( ZSTDMT_createCompressionJob(zcs, zcs->targetSectionSize, 0) );
+ }
+
+ /* check for data to flush */
+ CHECK_F( ZSTDMT_flushNextJob(zcs, output, (zcs->inBuff.filled == zcs->inBuffSize)) ); /* block if it wasn't possible to create new job due to saturation */
+
+ /* recommended next input size : fill current input buffer */
+ return zcs->inBuffSize - zcs->inBuff.filled; /* note : could be zero when input buffer is fully filled and no more availability to create new job */
+}
+
+
+static size_t ZSTDMT_flushStream_internal(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output, unsigned endFrame)
+{
+ size_t const srcSize = zcs->inBuff.filled - zcs->dictSize;
+
+ if (srcSize) DEBUGLOG(4, "flushing : %u bytes left to compress", (U32)srcSize);
+ if ( ((srcSize > 0) || (endFrame && !zcs->frameEnded))
+ && (zcs->nextJobID <= zcs->doneJobID + zcs->jobIDMask) ) {
+ CHECK_F( ZSTDMT_createCompressionJob(zcs, srcSize, endFrame) );
+ }
+
+ /* check if there is any data available to flush */
+ DEBUGLOG(5, "zcs->doneJobID : %u ; zcs->nextJobID : %u ", zcs->doneJobID, zcs->nextJobID);
+ return ZSTDMT_flushNextJob(zcs, output, 1);
+}
+
+
+size_t ZSTDMT_flushStream(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output)
+{
+ if (zcs->nbThreads==1) return ZSTD_flushStream(zcs->cstream, output);
+ return ZSTDMT_flushStream_internal(zcs, output, 0);
+}
+
+size_t ZSTDMT_endStream(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output)
+{
+ if (zcs->nbThreads==1) return ZSTD_endStream(zcs->cstream, output);
+ return ZSTDMT_flushStream_internal(zcs, output, 1);
+}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/contrib/python-zstandard/zstd/compress/zstdmt_compress.h Tue Feb 28 11:13:25 2017 -0800
@@ -0,0 +1,78 @@
+/**
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+ #ifndef ZSTDMT_COMPRESS_H
+ #define ZSTDMT_COMPRESS_H
+
+ #if defined (__cplusplus)
+ extern "C" {
+ #endif
+
+
+/* Note : All prototypes defined in this file shall be considered experimental.
+ * There is no guarantee of API continuity (yet) on any of these prototypes */
+
+/* === Dependencies === */
+#include <stddef.h> /* size_t */
+#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters */
+#include "zstd.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTDLIB_API */
+
+
+/* === Simple one-pass functions === */
+
+typedef struct ZSTDMT_CCtx_s ZSTDMT_CCtx;
+ZSTDLIB_API ZSTDMT_CCtx* ZSTDMT_createCCtx(unsigned nbThreads);
+ZSTDLIB_API size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* cctx);
+
+ZSTDLIB_API size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ int compressionLevel);
+
+
+/* === Streaming functions === */
+
+ZSTDLIB_API size_t ZSTDMT_initCStream(ZSTDMT_CCtx* mtctx, int compressionLevel);
+ZSTDLIB_API size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* mtctx, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be zero == unknown */
+
+ZSTDLIB_API size_t ZSTDMT_compressStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
+
+ZSTDLIB_API size_t ZSTDMT_flushStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output); /**< @return : 0 == all flushed; >0 : still some data to be flushed; or an error code (ZSTD_isError()) */
+ZSTDLIB_API size_t ZSTDMT_endStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output); /**< @return : 0 == all flushed; >0 : still some data to be flushed; or an error code (ZSTD_isError()) */
+
+
+/* === Advanced functions and parameters === */
+
+#ifndef ZSTDMT_SECTION_SIZE_MIN
+# define ZSTDMT_SECTION_SIZE_MIN (1U << 20) /* 1 MB - Minimum size of each compression job */
+#endif
+
+ZSTDLIB_API size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx, const void* dict, size_t dictSize, /**< dict can be released after init, a local copy is preserved within zcs */
+ ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be zero == unknown */
+
+/* ZSDTMT_parameter :
+ * List of parameters that can be set using ZSTDMT_setMTCtxParameter() */
+typedef enum {
+ ZSTDMT_p_sectionSize, /* size of input "section". Each section is compressed in parallel. 0 means default, which is dynamically determined within compression functions */
+ ZSTDMT_p_overlapSectionLog /* Log of overlapped section; 0 == no overlap, 6(default) == use 1/8th of window, >=9 == use full window */
+} ZSDTMT_parameter;
+
+/* ZSTDMT_setMTCtxParameter() :
+ * allow setting individual parameters, one at a time, among a list of enums defined in ZSTDMT_parameter.
+ * The function must be called typically after ZSTD_createCCtx().
+ * Parameters not explicitly reset by ZSTDMT_init*() remain the same in consecutive compression sessions.
+ * @return : 0, or an error code (which can be tested using ZSTD_isError()) */
+ZSTDLIB_API size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSDTMT_parameter parameter, unsigned value);
+
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* ZSTDMT_COMPRESS_H */
--- a/contrib/python-zstandard/zstd/decompress/zstd_decompress.c Sat Feb 25 12:48:50 2017 +0900
+++ b/contrib/python-zstandard/zstd/decompress/zstd_decompress.c Tue Feb 28 11:13:25 2017 -0800
@@ -1444,7 +1444,7 @@
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, dict, dictSize);
#endif
- ZSTD_decompressBegin_usingDict(dctx, dict, dictSize);
+ CHECK_F(ZSTD_decompressBegin_usingDict(dctx, dict, dictSize));
ZSTD_checkContinuity(dctx, dst);
return ZSTD_decompressFrame(dctx, dst, dstCapacity, src, srcSize);
}
@@ -1671,9 +1671,9 @@
}
if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted);
- dctx->rep[0] = MEM_readLE32(dictPtr+0); if (dctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted);
- dctx->rep[1] = MEM_readLE32(dictPtr+4); if (dctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted);
- dctx->rep[2] = MEM_readLE32(dictPtr+8); if (dctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted);
+ dctx->rep[0] = MEM_readLE32(dictPtr+0); if (dctx->rep[0] == 0 || dctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted);
+ dctx->rep[1] = MEM_readLE32(dictPtr+4); if (dctx->rep[1] == 0 || dctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted);
+ dctx->rep[2] = MEM_readLE32(dictPtr+8); if (dctx->rep[2] == 0 || dctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted);
dictPtr += 12;
dctx->litEntropy = dctx->fseEntropy = 1;
@@ -1713,39 +1713,44 @@
/* ====== ZSTD_DDict ====== */
struct ZSTD_DDict_s {
- void* dict;
+ void* dictBuffer;
+ const void* dictContent;
size_t dictSize;
ZSTD_DCtx* refContext;
}; /* typedef'd to ZSTD_DDict within "zstd.h" */
-ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_customMem customMem)
+ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, unsigned byReference, ZSTD_customMem customMem)
{
if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem;
if (!customMem.customAlloc || !customMem.customFree) return NULL;
{ ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_malloc(sizeof(ZSTD_DDict), customMem);
- void* const dictContent = ZSTD_malloc(dictSize, customMem);
ZSTD_DCtx* const dctx = ZSTD_createDCtx_advanced(customMem);
- if (!dictContent || !ddict || !dctx) {
- ZSTD_free(dictContent, customMem);
+ if (!ddict || !dctx) {
ZSTD_free(ddict, customMem);
ZSTD_free(dctx, customMem);
return NULL;
}
- if (dictSize) {
- memcpy(dictContent, dict, dictSize);
+ if ((byReference) || (!dict) || (!dictSize)) {
+ ddict->dictBuffer = NULL;
+ ddict->dictContent = dict;
+ } else {
+ void* const internalBuffer = ZSTD_malloc(dictSize, customMem);
+ if (!internalBuffer) { ZSTD_free(dctx, customMem); ZSTD_free(ddict, customMem); return NULL; }
+ memcpy(internalBuffer, dict, dictSize);
+ ddict->dictBuffer = internalBuffer;
+ ddict->dictContent = internalBuffer;
}
- { size_t const errorCode = ZSTD_decompressBegin_usingDict(dctx, dictContent, dictSize);
+ { size_t const errorCode = ZSTD_decompressBegin_usingDict(dctx, ddict->dictContent, dictSize);
if (ZSTD_isError(errorCode)) {
- ZSTD_free(dictContent, customMem);
+ ZSTD_free(ddict->dictBuffer, customMem);
ZSTD_free(ddict, customMem);
ZSTD_free(dctx, customMem);
return NULL;
} }
- ddict->dict = dictContent;
ddict->dictSize = dictSize;
ddict->refContext = dctx;
return ddict;
@@ -1758,15 +1763,27 @@
ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize)
{
ZSTD_customMem const allocator = { NULL, NULL, NULL };
- return ZSTD_createDDict_advanced(dict, dictSize, allocator);
+ return ZSTD_createDDict_advanced(dict, dictSize, 0, allocator);
}
+
+/*! ZSTD_createDDict_byReference() :
+ * Create a digested dictionary, ready to start decompression operation without startup delay.
+ * Dictionary content is simply referenced, and therefore stays in dictBuffer.
+ * It is important that dictBuffer outlives DDict, it must remain read accessible throughout the lifetime of DDict */
+ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize)
+{
+ ZSTD_customMem const allocator = { NULL, NULL, NULL };
+ return ZSTD_createDDict_advanced(dictBuffer, dictSize, 1, allocator);
+}
+
+
size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
{
if (ddict==NULL) return 0; /* support free on NULL */
{ ZSTD_customMem const cMem = ddict->refContext->customMem;
ZSTD_freeDCtx(ddict->refContext);
- ZSTD_free(ddict->dict, cMem);
+ ZSTD_free(ddict->dictBuffer, cMem);
ZSTD_free(ddict, cMem);
return 0;
}
@@ -1775,7 +1792,7 @@
size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict)
{
if (ddict==NULL) return 0; /* support sizeof on NULL */
- return sizeof(*ddict) + sizeof(ddict->refContext) + ddict->dictSize;
+ return sizeof(*ddict) + ZSTD_sizeof_DCtx(ddict->refContext) + (ddict->dictBuffer ? ddict->dictSize : 0) ;
}
/*! ZSTD_getDictID_fromDict() :
@@ -1796,7 +1813,7 @@
unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict)
{
if (ddict==NULL) return 0;
- return ZSTD_getDictID_fromDict(ddict->dict, ddict->dictSize);
+ return ZSTD_getDictID_fromDict(ddict->dictContent, ddict->dictSize);
}
/*! ZSTD_getDictID_fromFrame() :
@@ -1827,7 +1844,7 @@
const ZSTD_DDict* ddict)
{
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
- if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, ddict->dict, ddict->dictSize);
+ if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, ddict->dictContent, ddict->dictSize);
#endif
ZSTD_refDCtx(dctx, ddict->refContext);
ZSTD_checkContinuity(dctx, dst);
@@ -1919,7 +1936,7 @@
zds->stage = zdss_loadHeader;
zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
ZSTD_freeDDict(zds->ddictLocal);
- if (dict) {
+ if (dict && dictSize >= 8) {
zds->ddictLocal = ZSTD_createDDict(dict, dictSize);
if (zds->ddictLocal == NULL) return ERROR(memory_allocation);
} else zds->ddictLocal = NULL;
@@ -1956,7 +1973,7 @@
switch(paramType)
{
default : return ERROR(parameter_unknown);
- case ZSTDdsp_maxWindowSize : zds->maxWindowSize = paramValue ? paramValue : (U32)(-1); break;
+ case DStream_p_maxWindowSize : zds->maxWindowSize = paramValue ? paramValue : (U32)(-1); break;
}
return 0;
}
@@ -2007,7 +2024,7 @@
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
{ U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart);
if (legacyVersion) {
- const void* const dict = zds->ddict ? zds->ddict->dict : NULL;
+ const void* const dict = zds->ddict ? zds->ddict->dictContent : NULL;
size_t const dictSize = zds->ddict ? zds->ddict->dictSize : 0;
CHECK_F(ZSTD_initLegacyStream(&zds->legacyContext, zds->previousLegacyVersion, legacyVersion,
dict, dictSize));
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/contrib/python-zstandard/zstd/dictBuilder/cover.c Tue Feb 28 11:13:25 2017 -0800
@@ -0,0 +1,1021 @@
+/**
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+/*-*************************************
+* Dependencies
+***************************************/
+#include <stdio.h> /* fprintf */
+#include <stdlib.h> /* malloc, free, qsort */
+#include <string.h> /* memset */
+#include <time.h> /* clock */
+
+#include "mem.h" /* read */
+#include "pool.h"
+#include "threading.h"
+#include "zstd_internal.h" /* includes zstd.h */
+#ifndef ZDICT_STATIC_LINKING_ONLY
+#define ZDICT_STATIC_LINKING_ONLY
+#endif
+#include "zdict.h"
+
+/*-*************************************
+* Constants
+***************************************/
+#define COVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((U32)-1) : ((U32)1 GB))
+
+/*-*************************************
+* Console display
+***************************************/
+static int g_displayLevel = 2;
+#define DISPLAY(...) \
+ { \
+ fprintf(stderr, __VA_ARGS__); \
+ fflush(stderr); \
+ }
+#define LOCALDISPLAYLEVEL(displayLevel, l, ...) \
+ if (displayLevel >= l) { \
+ DISPLAY(__VA_ARGS__); \
+ } /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */
+#define DISPLAYLEVEL(l, ...) LOCALDISPLAYLEVEL(g_displayLevel, l, __VA_ARGS__)
+
+#define LOCALDISPLAYUPDATE(displayLevel, l, ...) \
+ if (displayLevel >= l) { \
+ if ((clock() - g_time > refreshRate) || (displayLevel >= 4)) { \
+ g_time = clock(); \
+ DISPLAY(__VA_ARGS__); \
+ if (displayLevel >= 4) \
+ fflush(stdout); \
+ } \
+ }
+#define DISPLAYUPDATE(l, ...) LOCALDISPLAYUPDATE(g_displayLevel, l, __VA_ARGS__)
+static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100;
+static clock_t g_time = 0;
+
+/*-*************************************
+* Hash table
+***************************************
+* A small specialized hash map for storing activeDmers.
+* The map does not resize, so if it becomes full it will loop forever.
+* Thus, the map must be large enough to store every value.
+* The map implements linear probing and keeps its load less than 0.5.
+*/
+
+#define MAP_EMPTY_VALUE ((U32)-1)
+typedef struct COVER_map_pair_t_s {
+ U32 key;
+ U32 value;
+} COVER_map_pair_t;
+
+typedef struct COVER_map_s {
+ COVER_map_pair_t *data;
+ U32 sizeLog;
+ U32 size;
+ U32 sizeMask;
+} COVER_map_t;
+
+/**
+ * Clear the map.
+ */
+static void COVER_map_clear(COVER_map_t *map) {
+ memset(map->data, MAP_EMPTY_VALUE, map->size * sizeof(COVER_map_pair_t));
+}
+
+/**
+ * Initializes a map of the given size.
+ * Returns 1 on success and 0 on failure.
+ * The map must be destroyed with COVER_map_destroy().
+ * The map is only guaranteed to be large enough to hold size elements.
+ */
+static int COVER_map_init(COVER_map_t *map, U32 size) {
+ map->sizeLog = ZSTD_highbit32(size) + 2;
+ map->size = (U32)1 << map->sizeLog;
+ map->sizeMask = map->size - 1;
+ map->data = (COVER_map_pair_t *)malloc(map->size * sizeof(COVER_map_pair_t));
+ if (!map->data) {
+ map->sizeLog = 0;
+ map->size = 0;
+ return 0;
+ }
+ COVER_map_clear(map);
+ return 1;
+}
+
+/**
+ * Internal hash function
+ */
+static const U32 prime4bytes = 2654435761U;
+static U32 COVER_map_hash(COVER_map_t *map, U32 key) {
+ return (key * prime4bytes) >> (32 - map->sizeLog);
+}
+
+/**
+ * Helper function that returns the index that a key should be placed into.
+ */
+static U32 COVER_map_index(COVER_map_t *map, U32 key) {
+ const U32 hash = COVER_map_hash(map, key);
+ U32 i;
+ for (i = hash;; i = (i + 1) & map->sizeMask) {
+ COVER_map_pair_t *pos = &map->data[i];
+ if (pos->value == MAP_EMPTY_VALUE) {
+ return i;
+ }
+ if (pos->key == key) {
+ return i;
+ }
+ }
+}
+
+/**
+ * Returns the pointer to the value for key.
+ * If key is not in the map, it is inserted and the value is set to 0.
+ * The map must not be full.
+ */
+static U32 *COVER_map_at(COVER_map_t *map, U32 key) {
+ COVER_map_pair_t *pos = &map->data[COVER_map_index(map, key)];
+ if (pos->value == MAP_EMPTY_VALUE) {
+ pos->key = key;
+ pos->value = 0;
+ }
+ return &pos->value;
+}
+
+/**
+ * Deletes key from the map if present.
+ */
+static void COVER_map_remove(COVER_map_t *map, U32 key) {
+ U32 i = COVER_map_index(map, key);
+ COVER_map_pair_t *del = &map->data[i];
+ U32 shift = 1;
+ if (del->value == MAP_EMPTY_VALUE) {
+ return;
+ }
+ for (i = (i + 1) & map->sizeMask;; i = (i + 1) & map->sizeMask) {
+ COVER_map_pair_t *const pos = &map->data[i];
+ /* If the position is empty we are done */
+ if (pos->value == MAP_EMPTY_VALUE) {
+ del->value = MAP_EMPTY_VALUE;
+ return;
+ }
+ /* If pos can be moved to del do so */
+ if (((i - COVER_map_hash(map, pos->key)) & map->sizeMask) >= shift) {
+ del->key = pos->key;
+ del->value = pos->value;
+ del = pos;
+ shift = 1;
+ } else {
+ ++shift;
+ }
+ }
+}
+
+/**
+ * Destroyes a map that is inited with COVER_map_init().
+ */
+static void COVER_map_destroy(COVER_map_t *map) {
+ if (map->data) {
+ free(map->data);
+ }
+ map->data = NULL;
+ map->size = 0;
+}
+
+/*-*************************************
+* Context
+***************************************/
+
+typedef struct {
+ const BYTE *samples;
+ size_t *offsets;
+ const size_t *samplesSizes;
+ size_t nbSamples;
+ U32 *suffix;
+ size_t suffixSize;
+ U32 *freqs;
+ U32 *dmerAt;
+ unsigned d;
+} COVER_ctx_t;
+
+/* We need a global context for qsort... */
+static COVER_ctx_t *g_ctx = NULL;
+
+/*-*************************************
+* Helper functions
+***************************************/
+
+/**
+ * Returns the sum of the sample sizes.
+ */
+static size_t COVER_sum(const size_t *samplesSizes, unsigned nbSamples) {
+ size_t sum = 0;
+ size_t i;
+ for (i = 0; i < nbSamples; ++i) {
+ sum += samplesSizes[i];
+ }
+ return sum;
+}
+
+/**
+ * Returns -1 if the dmer at lp is less than the dmer at rp.
+ * Return 0 if the dmers at lp and rp are equal.
+ * Returns 1 if the dmer at lp is greater than the dmer at rp.
+ */
+static int COVER_cmp(COVER_ctx_t *ctx, const void *lp, const void *rp) {
+ const U32 lhs = *(const U32 *)lp;
+ const U32 rhs = *(const U32 *)rp;
+ return memcmp(ctx->samples + lhs, ctx->samples + rhs, ctx->d);
+}
+
+/**
+ * Same as COVER_cmp() except ties are broken by pointer value
+ * NOTE: g_ctx must be set to call this function. A global is required because
+ * qsort doesn't take an opaque pointer.
+ */
+static int COVER_strict_cmp(const void *lp, const void *rp) {
+ int result = COVER_cmp(g_ctx, lp, rp);
+ if (result == 0) {
+ result = lp < rp ? -1 : 1;
+ }
+ return result;
+}
+
+/**
+ * Returns the first pointer in [first, last) whose element does not compare
+ * less than value. If no such element exists it returns last.
+ */
+static const size_t *COVER_lower_bound(const size_t *first, const size_t *last,
+ size_t value) {
+ size_t count = last - first;
+ while (count != 0) {
+ size_t step = count / 2;
+ const size_t *ptr = first;
+ ptr += step;
+ if (*ptr < value) {
+ first = ++ptr;
+ count -= step + 1;
+ } else {
+ count = step;
+ }
+ }
+ return first;
+}
+
+/**
+ * Generic groupBy function.
+ * Groups an array sorted by cmp into groups with equivalent values.
+ * Calls grp for each group.
+ */
+static void
+COVER_groupBy(const void *data, size_t count, size_t size, COVER_ctx_t *ctx,
+ int (*cmp)(COVER_ctx_t *, const void *, const void *),
+ void (*grp)(COVER_ctx_t *, const void *, const void *)) {
+ const BYTE *ptr = (const BYTE *)data;
+ size_t num = 0;
+ while (num < count) {
+ const BYTE *grpEnd = ptr + size;
+ ++num;
+ while (num < count && cmp(ctx, ptr, grpEnd) == 0) {
+ grpEnd += size;
+ ++num;
+ }
+ grp(ctx, ptr, grpEnd);
+ ptr = grpEnd;
+ }
+}
+
+/*-*************************************
+* Cover functions
+***************************************/
+
+/**
+ * Called on each group of positions with the same dmer.
+ * Counts the frequency of each dmer and saves it in the suffix array.
+ * Fills `ctx->dmerAt`.
+ */
+static void COVER_group(COVER_ctx_t *ctx, const void *group,
+ const void *groupEnd) {
+ /* The group consists of all the positions with the same first d bytes. */
+ const U32 *grpPtr = (const U32 *)group;
+ const U32 *grpEnd = (const U32 *)groupEnd;
+ /* The dmerId is how we will reference this dmer.
+ * This allows us to map the whole dmer space to a much smaller space, the
+ * size of the suffix array.
+ */
+ const U32 dmerId = (U32)(grpPtr - ctx->suffix);
+ /* Count the number of samples this dmer shows up in */
+ U32 freq = 0;
+ /* Details */
+ const size_t *curOffsetPtr = ctx->offsets;
+ const size_t *offsetsEnd = ctx->offsets + ctx->nbSamples;
+ /* Once *grpPtr >= curSampleEnd this occurrence of the dmer is in a
+ * different sample than the last.
+ */
+ size_t curSampleEnd = ctx->offsets[0];
+ for (; grpPtr != grpEnd; ++grpPtr) {
+ /* Save the dmerId for this position so we can get back to it. */
+ ctx->dmerAt[*grpPtr] = dmerId;
+ /* Dictionaries only help for the first reference to the dmer.
+ * After that zstd can reference the match from the previous reference.
+ * So only count each dmer once for each sample it is in.
+ */
+ if (*grpPtr < curSampleEnd) {
+ continue;
+ }
+ freq += 1;
+ /* Binary search to find the end of the sample *grpPtr is in.
+ * In the common case that grpPtr + 1 == grpEnd we can skip the binary
+ * search because the loop is over.
+ */
+ if (grpPtr + 1 != grpEnd) {
+ const size_t *sampleEndPtr =
+ COVER_lower_bound(curOffsetPtr, offsetsEnd, *grpPtr);
+ curSampleEnd = *sampleEndPtr;
+ curOffsetPtr = sampleEndPtr + 1;
+ }
+ }
+ /* At this point we are never going to look at this segment of the suffix
+ * array again. We take advantage of this fact to save memory.
+ * We store the frequency of the dmer in the first position of the group,
+ * which is dmerId.
+ */
+ ctx->suffix[dmerId] = freq;
+}
+
+/**
+ * A segment is a range in the source as well as the score of the segment.
+ */
+typedef struct {
+ U32 begin;
+ U32 end;
+ double score;
+} COVER_segment_t;
+
+/**
+ * Selects the best segment in an epoch.
+ * Segments of are scored according to the function:
+ *
+ * Let F(d) be the frequency of dmer d.
+ * Let S_i be the dmer at position i of segment S which has length k.
+ *
+ * Score(S) = F(S_1) + F(S_2) + ... + F(S_{k-d+1})
+ *
+ * Once the dmer d is in the dictionay we set F(d) = 0.
+ */
+static COVER_segment_t COVER_selectSegment(const COVER_ctx_t *ctx, U32 *freqs,
+ COVER_map_t *activeDmers, U32 begin,
+ U32 end, COVER_params_t parameters) {
+ /* Constants */
+ const U32 k = parameters.k;
+ const U32 d = parameters.d;
+ const U32 dmersInK = k - d + 1;
+ /* Try each segment (activeSegment) and save the best (bestSegment) */
+ COVER_segment_t bestSegment = {0, 0, 0};
+ COVER_segment_t activeSegment;
+ /* Reset the activeDmers in the segment */
+ COVER_map_clear(activeDmers);
+ /* The activeSegment starts at the beginning of the epoch. */
+ activeSegment.begin = begin;
+ activeSegment.end = begin;
+ activeSegment.score = 0;
+ /* Slide the activeSegment through the whole epoch.
+ * Save the best segment in bestSegment.
+ */
+ while (activeSegment.end < end) {
+ /* The dmerId for the dmer at the next position */
+ U32 newDmer = ctx->dmerAt[activeSegment.end];
+ /* The entry in activeDmers for this dmerId */
+ U32 *newDmerOcc = COVER_map_at(activeDmers, newDmer);
+ /* If the dmer isn't already present in the segment add its score. */
+ if (*newDmerOcc == 0) {
+ /* The paper suggest using the L-0.5 norm, but experiments show that it
+ * doesn't help.
+ */
+ activeSegment.score += freqs[newDmer];
+ }
+ /* Add the dmer to the segment */
+ activeSegment.end += 1;
+ *newDmerOcc += 1;
+
+ /* If the window is now too large, drop the first position */
+ if (activeSegment.end - activeSegment.begin == dmersInK + 1) {
+ U32 delDmer = ctx->dmerAt[activeSegment.begin];
+ U32 *delDmerOcc = COVER_map_at(activeDmers, delDmer);
+ activeSegment.begin += 1;
+ *delDmerOcc -= 1;
+ /* If this is the last occurence of the dmer, subtract its score */
+ if (*delDmerOcc == 0) {
+ COVER_map_remove(activeDmers, delDmer);
+ activeSegment.score -= freqs[delDmer];
+ }
+ }
+
+ /* If this segment is the best so far save it */
+ if (activeSegment.score > bestSegment.score) {
+ bestSegment = activeSegment;
+ }
+ }
+ {
+ /* Trim off the zero frequency head and tail from the segment. */
+ U32 newBegin = bestSegment.end;
+ U32 newEnd = bestSegment.begin;
+ U32 pos;
+ for (pos = bestSegment.begin; pos != bestSegment.end; ++pos) {
+ U32 freq = freqs[ctx->dmerAt[pos]];
+ if (freq != 0) {
+ newBegin = MIN(newBegin, pos);
+ newEnd = pos + 1;
+ }
+ }
+ bestSegment.begin = newBegin;
+ bestSegment.end = newEnd;
+ }
+ {
+ /* Zero out the frequency of each dmer covered by the chosen segment. */
+ U32 pos;
+ for (pos = bestSegment.begin; pos != bestSegment.end; ++pos) {
+ freqs[ctx->dmerAt[pos]] = 0;
+ }
+ }
+ return bestSegment;
+}
+
+/**
+ * Check the validity of the parameters.
+ * Returns non-zero if the parameters are valid and 0 otherwise.
+ */
+static int COVER_checkParameters(COVER_params_t parameters) {
+ /* k and d are required parameters */
+ if (parameters.d == 0 || parameters.k == 0) {
+ return 0;
+ }
+ /* d <= k */
+ if (parameters.d > parameters.k) {
+ return 0;
+ }
+ return 1;
+}
+
+/**
+ * Clean up a context initialized with `COVER_ctx_init()`.
+ */
+static void COVER_ctx_destroy(COVER_ctx_t *ctx) {
+ if (!ctx) {
+ return;
+ }
+ if (ctx->suffix) {
+ free(ctx->suffix);
+ ctx->suffix = NULL;
+ }
+ if (ctx->freqs) {
+ free(ctx->freqs);
+ ctx->freqs = NULL;
+ }
+ if (ctx->dmerAt) {
+ free(ctx->dmerAt);
+ ctx->dmerAt = NULL;
+ }
+ if (ctx->offsets) {
+ free(ctx->offsets);
+ ctx->offsets = NULL;
+ }
+}
+
+/**
+ * Prepare a context for dictionary building.
+ * The context is only dependent on the parameter `d` and can used multiple
+ * times.
+ * Returns 1 on success or zero on error.
+ * The context must be destroyed with `COVER_ctx_destroy()`.
+ */
+static int COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer,
+ const size_t *samplesSizes, unsigned nbSamples,
+ unsigned d) {
+ const BYTE *const samples = (const BYTE *)samplesBuffer;
+ const size_t totalSamplesSize = COVER_sum(samplesSizes, nbSamples);
+ /* Checks */
+ if (totalSamplesSize < d ||
+ totalSamplesSize >= (size_t)COVER_MAX_SAMPLES_SIZE) {
+ DISPLAYLEVEL(1, "Total samples size is too large, maximum size is %u MB\n",
+ (COVER_MAX_SAMPLES_SIZE >> 20));
+ return 0;
+ }
+ /* Zero the context */
+ memset(ctx, 0, sizeof(*ctx));
+ DISPLAYLEVEL(2, "Training on %u samples of total size %u\n", nbSamples,
+ (U32)totalSamplesSize);
+ ctx->samples = samples;
+ ctx->samplesSizes = samplesSizes;
+ ctx->nbSamples = nbSamples;
+ /* Partial suffix array */
+ ctx->suffixSize = totalSamplesSize - d + 1;
+ ctx->suffix = (U32 *)malloc(ctx->suffixSize * sizeof(U32));
+ /* Maps index to the dmerID */
+ ctx->dmerAt = (U32 *)malloc(ctx->suffixSize * sizeof(U32));
+ /* The offsets of each file */
+ ctx->offsets = (size_t *)malloc((nbSamples + 1) * sizeof(size_t));
+ if (!ctx->suffix || !ctx->dmerAt || !ctx->offsets) {
+ DISPLAYLEVEL(1, "Failed to allocate scratch buffers\n");
+ COVER_ctx_destroy(ctx);
+ return 0;
+ }
+ ctx->freqs = NULL;
+ ctx->d = d;
+
+ /* Fill offsets from the samlesSizes */
+ {
+ U32 i;
+ ctx->offsets[0] = 0;
+ for (i = 1; i <= nbSamples; ++i) {
+ ctx->offsets[i] = ctx->offsets[i - 1] + samplesSizes[i - 1];
+ }
+ }
+ DISPLAYLEVEL(2, "Constructing partial suffix array\n");
+ {
+ /* suffix is a partial suffix array.
+ * It only sorts suffixes by their first parameters.d bytes.
+ * The sort is stable, so each dmer group is sorted by position in input.
+ */
+ U32 i;
+ for (i = 0; i < ctx->suffixSize; ++i) {
+ ctx->suffix[i] = i;
+ }
+ /* qsort doesn't take an opaque pointer, so pass as a global */
+ g_ctx = ctx;
+ qsort(ctx->suffix, ctx->suffixSize, sizeof(U32), &COVER_strict_cmp);
+ }
+ DISPLAYLEVEL(2, "Computing frequencies\n");
+ /* For each dmer group (group of positions with the same first d bytes):
+ * 1. For each position we set dmerAt[position] = dmerID. The dmerID is
+ * (groupBeginPtr - suffix). This allows us to go from position to
+ * dmerID so we can look up values in freq.
+ * 2. We calculate how many samples the dmer occurs in and save it in
+ * freqs[dmerId].
+ */
+ COVER_groupBy(ctx->suffix, ctx->suffixSize, sizeof(U32), ctx, &COVER_cmp,
+ &COVER_group);
+ ctx->freqs = ctx->suffix;
+ ctx->suffix = NULL;
+ return 1;
+}
+
+/**
+ * Given the prepared context build the dictionary.
+ */
+static size_t COVER_buildDictionary(const COVER_ctx_t *ctx, U32 *freqs,
+ COVER_map_t *activeDmers, void *dictBuffer,
+ size_t dictBufferCapacity,
+ COVER_params_t parameters) {
+ BYTE *const dict = (BYTE *)dictBuffer;
+ size_t tail = dictBufferCapacity;
+ /* Divide the data up into epochs of equal size.
+ * We will select at least one segment from each epoch.
+ */
+ const U32 epochs = (U32)(dictBufferCapacity / parameters.k);
+ const U32 epochSize = (U32)(ctx->suffixSize / epochs);
+ size_t epoch;
+ DISPLAYLEVEL(2, "Breaking content into %u epochs of size %u\n", epochs,
+ epochSize);
+ /* Loop through the epochs until there are no more segments or the dictionary
+ * is full.
+ */
+ for (epoch = 0; tail > 0; epoch = (epoch + 1) % epochs) {
+ const U32 epochBegin = (U32)(epoch * epochSize);
+ const U32 epochEnd = epochBegin + epochSize;
+ size_t segmentSize;
+ /* Select a segment */
+ COVER_segment_t segment = COVER_selectSegment(
+ ctx, freqs, activeDmers, epochBegin, epochEnd, parameters);
+ /* Trim the segment if necessary and if it is empty then we are done */
+ segmentSize = MIN(segment.end - segment.begin + parameters.d - 1, tail);
+ if (segmentSize == 0) {
+ break;
+ }
+ /* We fill the dictionary from the back to allow the best segments to be
+ * referenced with the smallest offsets.
+ */
+ tail -= segmentSize;
+ memcpy(dict + tail, ctx->samples + segment.begin, segmentSize);
+ DISPLAYUPDATE(
+ 2, "\r%u%% ",
+ (U32)(((dictBufferCapacity - tail) * 100) / dictBufferCapacity));
+ }
+ DISPLAYLEVEL(2, "\r%79s\r", "");
+ return tail;
+}
+
+/**
+ * Translate from COVER_params_t to ZDICT_params_t required for finalizing the
+ * dictionary.
+ */
+static ZDICT_params_t COVER_translateParams(COVER_params_t parameters) {
+ ZDICT_params_t zdictParams;
+ memset(&zdictParams, 0, sizeof(zdictParams));
+ zdictParams.notificationLevel = 1;
+ zdictParams.dictID = parameters.dictID;
+ zdictParams.compressionLevel = parameters.compressionLevel;
+ return zdictParams;
+}
+
+/**
+ * Constructs a dictionary using a heuristic based on the following paper:
+ *
+ * Liao, Petri, Moffat, Wirth
+ * Effective Construction of Relative Lempel-Ziv Dictionaries
+ * Published in WWW 2016.
+ */
+ZDICTLIB_API size_t COVER_trainFromBuffer(
+ void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer,
+ const size_t *samplesSizes, unsigned nbSamples, COVER_params_t parameters) {
+ BYTE *const dict = (BYTE *)dictBuffer;
+ COVER_ctx_t ctx;
+ COVER_map_t activeDmers;
+ /* Checks */
+ if (!COVER_checkParameters(parameters)) {
+ DISPLAYLEVEL(1, "Cover parameters incorrect\n");
+ return ERROR(GENERIC);
+ }
+ if (nbSamples == 0) {
+ DISPLAYLEVEL(1, "Cover must have at least one input file\n");
+ return ERROR(GENERIC);
+ }
+ if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) {
+ DISPLAYLEVEL(1, "dictBufferCapacity must be at least %u\n",
+ ZDICT_DICTSIZE_MIN);
+ return ERROR(dstSize_tooSmall);
+ }
+ /* Initialize global data */
+ g_displayLevel = parameters.notificationLevel;
+ /* Initialize context and activeDmers */
+ if (!COVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples,
+ parameters.d)) {
+ return ERROR(GENERIC);
+ }
+ if (!COVER_map_init(&activeDmers, parameters.k - parameters.d + 1)) {
+ DISPLAYLEVEL(1, "Failed to allocate dmer map: out of memory\n");
+ COVER_ctx_destroy(&ctx);
+ return ERROR(GENERIC);
+ }
+
+ DISPLAYLEVEL(2, "Building dictionary\n");
+ {
+ const size_t tail =
+ COVER_buildDictionary(&ctx, ctx.freqs, &activeDmers, dictBuffer,
+ dictBufferCapacity, parameters);
+ ZDICT_params_t zdictParams = COVER_translateParams(parameters);
+ const size_t dictionarySize = ZDICT_finalizeDictionary(
+ dict, dictBufferCapacity, dict + tail, dictBufferCapacity - tail,
+ samplesBuffer, samplesSizes, nbSamples, zdictParams);
+ if (!ZSTD_isError(dictionarySize)) {
+ DISPLAYLEVEL(2, "Constructed dictionary of size %u\n",
+ (U32)dictionarySize);
+ }
+ COVER_ctx_destroy(&ctx);
+ COVER_map_destroy(&activeDmers);
+ return dictionarySize;
+ }
+}
+
+/**
+ * COVER_best_t is used for two purposes:
+ * 1. Synchronizing threads.
+ * 2. Saving the best parameters and dictionary.
+ *
+ * All of the methods except COVER_best_init() are thread safe if zstd is
+ * compiled with multithreaded support.
+ */
+typedef struct COVER_best_s {
+ pthread_mutex_t mutex;
+ pthread_cond_t cond;
+ size_t liveJobs;
+ void *dict;
+ size_t dictSize;
+ COVER_params_t parameters;
+ size_t compressedSize;
+} COVER_best_t;
+
+/**
+ * Initialize the `COVER_best_t`.
+ */
+static void COVER_best_init(COVER_best_t *best) {
+ if (!best) {
+ return;
+ }
+ pthread_mutex_init(&best->mutex, NULL);
+ pthread_cond_init(&best->cond, NULL);
+ best->liveJobs = 0;
+ best->dict = NULL;
+ best->dictSize = 0;
+ best->compressedSize = (size_t)-1;
+ memset(&best->parameters, 0, sizeof(best->parameters));
+}
+
+/**
+ * Wait until liveJobs == 0.
+ */
+static void COVER_best_wait(COVER_best_t *best) {
+ if (!best) {
+ return;
+ }
+ pthread_mutex_lock(&best->mutex);
+ while (best->liveJobs != 0) {
+ pthread_cond_wait(&best->cond, &best->mutex);
+ }
+ pthread_mutex_unlock(&best->mutex);
+}
+
+/**
+ * Call COVER_best_wait() and then destroy the COVER_best_t.
+ */
+static void COVER_best_destroy(COVER_best_t *best) {
+ if (!best) {
+ return;
+ }
+ COVER_best_wait(best);
+ if (best->dict) {
+ free(best->dict);
+ }
+ pthread_mutex_destroy(&best->mutex);
+ pthread_cond_destroy(&best->cond);
+}
+
+/**
+ * Called when a thread is about to be launched.
+ * Increments liveJobs.
+ */
+static void COVER_best_start(COVER_best_t *best) {
+ if (!best) {
+ return;
+ }
+ pthread_mutex_lock(&best->mutex);
+ ++best->liveJobs;
+ pthread_mutex_unlock(&best->mutex);
+}
+
+/**
+ * Called when a thread finishes executing, both on error or success.
+ * Decrements liveJobs and signals any waiting threads if liveJobs == 0.
+ * If this dictionary is the best so far save it and its parameters.
+ */
+static void COVER_best_finish(COVER_best_t *best, size_t compressedSize,
+ COVER_params_t parameters, void *dict,
+ size_t dictSize) {
+ if (!best) {
+ return;
+ }
+ {
+ size_t liveJobs;
+ pthread_mutex_lock(&best->mutex);
+ --best->liveJobs;
+ liveJobs = best->liveJobs;
+ /* If the new dictionary is better */
+ if (compressedSize < best->compressedSize) {
+ /* Allocate space if necessary */
+ if (!best->dict || best->dictSize < dictSize) {
+ if (best->dict) {
+ free(best->dict);
+ }
+ best->dict = malloc(dictSize);
+ if (!best->dict) {
+ best->compressedSize = ERROR(GENERIC);
+ best->dictSize = 0;
+ return;
+ }
+ }
+ /* Save the dictionary, parameters, and size */
+ memcpy(best->dict, dict, dictSize);
+ best->dictSize = dictSize;
+ best->parameters = parameters;
+ best->compressedSize = compressedSize;
+ }
+ pthread_mutex_unlock(&best->mutex);
+ if (liveJobs == 0) {
+ pthread_cond_broadcast(&best->cond);
+ }
+ }
+}
+
+/**
+ * Parameters for COVER_tryParameters().
+ */
+typedef struct COVER_tryParameters_data_s {
+ const COVER_ctx_t *ctx;
+ COVER_best_t *best;
+ size_t dictBufferCapacity;
+ COVER_params_t parameters;
+} COVER_tryParameters_data_t;
+
+/**
+ * Tries a set of parameters and upates the COVER_best_t with the results.
+ * This function is thread safe if zstd is compiled with multithreaded support.
+ * It takes its parameters as an *OWNING* opaque pointer to support threading.
+ */
+static void COVER_tryParameters(void *opaque) {
+ /* Save parameters as local variables */
+ COVER_tryParameters_data_t *const data = (COVER_tryParameters_data_t *)opaque;
+ const COVER_ctx_t *const ctx = data->ctx;
+ const COVER_params_t parameters = data->parameters;
+ size_t dictBufferCapacity = data->dictBufferCapacity;
+ size_t totalCompressedSize = ERROR(GENERIC);
+ /* Allocate space for hash table, dict, and freqs */
+ COVER_map_t activeDmers;
+ BYTE *const dict = (BYTE * const)malloc(dictBufferCapacity);
+ U32 *freqs = (U32 *)malloc(ctx->suffixSize * sizeof(U32));
+ if (!COVER_map_init(&activeDmers, parameters.k - parameters.d + 1)) {
+ DISPLAYLEVEL(1, "Failed to allocate dmer map: out of memory\n");
+ goto _cleanup;
+ }
+ if (!dict || !freqs) {
+ DISPLAYLEVEL(1, "Failed to allocate buffers: out of memory\n");
+ goto _cleanup;
+ }
+ /* Copy the frequencies because we need to modify them */
+ memcpy(freqs, ctx->freqs, ctx->suffixSize * sizeof(U32));
+ /* Build the dictionary */
+ {
+ const size_t tail = COVER_buildDictionary(ctx, freqs, &activeDmers, dict,
+ dictBufferCapacity, parameters);
+ const ZDICT_params_t zdictParams = COVER_translateParams(parameters);
+ dictBufferCapacity = ZDICT_finalizeDictionary(
+ dict, dictBufferCapacity, dict + tail, dictBufferCapacity - tail,
+ ctx->samples, ctx->samplesSizes, (unsigned)ctx->nbSamples, zdictParams);
+ if (ZDICT_isError(dictBufferCapacity)) {
+ DISPLAYLEVEL(1, "Failed to finalize dictionary\n");
+ goto _cleanup;
+ }
+ }
+ /* Check total compressed size */
+ {
+ /* Pointers */
+ ZSTD_CCtx *cctx;
+ ZSTD_CDict *cdict;
+ void *dst;
+ /* Local variables */
+ size_t dstCapacity;
+ size_t i;
+ /* Allocate dst with enough space to compress the maximum sized sample */
+ {
+ size_t maxSampleSize = 0;
+ for (i = 0; i < ctx->nbSamples; ++i) {
+ maxSampleSize = MAX(ctx->samplesSizes[i], maxSampleSize);
+ }
+ dstCapacity = ZSTD_compressBound(maxSampleSize);
+ dst = malloc(dstCapacity);
+ }
+ /* Create the cctx and cdict */
+ cctx = ZSTD_createCCtx();
+ cdict =
+ ZSTD_createCDict(dict, dictBufferCapacity, parameters.compressionLevel);
+ if (!dst || !cctx || !cdict) {
+ goto _compressCleanup;
+ }
+ /* Compress each sample and sum their sizes (or error) */
+ totalCompressedSize = 0;
+ for (i = 0; i < ctx->nbSamples; ++i) {
+ const size_t size = ZSTD_compress_usingCDict(
+ cctx, dst, dstCapacity, ctx->samples + ctx->offsets[i],
+ ctx->samplesSizes[i], cdict);
+ if (ZSTD_isError(size)) {
+ totalCompressedSize = ERROR(GENERIC);
+ goto _compressCleanup;
+ }
+ totalCompressedSize += size;
+ }
+ _compressCleanup:
+ ZSTD_freeCCtx(cctx);
+ ZSTD_freeCDict(cdict);
+ if (dst) {
+ free(dst);
+ }
+ }
+
+_cleanup:
+ COVER_best_finish(data->best, totalCompressedSize, parameters, dict,
+ dictBufferCapacity);
+ free(data);
+ COVER_map_destroy(&activeDmers);
+ if (dict) {
+ free(dict);
+ }
+ if (freqs) {
+ free(freqs);
+ }
+}
+
+ZDICTLIB_API size_t COVER_optimizeTrainFromBuffer(void *dictBuffer,
+ size_t dictBufferCapacity,
+ const void *samplesBuffer,
+ const size_t *samplesSizes,
+ unsigned nbSamples,
+ COVER_params_t *parameters) {
+ /* constants */
+ const unsigned nbThreads = parameters->nbThreads;
+ const unsigned kMinD = parameters->d == 0 ? 6 : parameters->d;
+ const unsigned kMaxD = parameters->d == 0 ? 16 : parameters->d;
+ const unsigned kMinK = parameters->k == 0 ? kMaxD : parameters->k;
+ const unsigned kMaxK = parameters->k == 0 ? 2048 : parameters->k;
+ const unsigned kSteps = parameters->steps == 0 ? 32 : parameters->steps;
+ const unsigned kStepSize = MAX((kMaxK - kMinK) / kSteps, 1);
+ const unsigned kIterations =
+ (1 + (kMaxD - kMinD) / 2) * (1 + (kMaxK - kMinK) / kStepSize);
+ /* Local variables */
+ const int displayLevel = parameters->notificationLevel;
+ unsigned iteration = 1;
+ unsigned d;
+ unsigned k;
+ COVER_best_t best;
+ POOL_ctx *pool = NULL;
+ /* Checks */
+ if (kMinK < kMaxD || kMaxK < kMinK) {
+ LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect parameters\n");
+ return ERROR(GENERIC);
+ }
+ if (nbSamples == 0) {
+ DISPLAYLEVEL(1, "Cover must have at least one input file\n");
+ return ERROR(GENERIC);
+ }
+ if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) {
+ DISPLAYLEVEL(1, "dictBufferCapacity must be at least %u\n",
+ ZDICT_DICTSIZE_MIN);
+ return ERROR(dstSize_tooSmall);
+ }
+ if (nbThreads > 1) {
+ pool = POOL_create(nbThreads, 1);
+ if (!pool) {
+ return ERROR(memory_allocation);
+ }
+ }
+ /* Initialization */
+ COVER_best_init(&best);
+ /* Turn down global display level to clean up display at level 2 and below */
+ g_displayLevel = parameters->notificationLevel - 1;
+ /* Loop through d first because each new value needs a new context */
+ LOCALDISPLAYLEVEL(displayLevel, 2, "Trying %u different sets of parameters\n",
+ kIterations);
+ for (d = kMinD; d <= kMaxD; d += 2) {
+ /* Initialize the context for this value of d */
+ COVER_ctx_t ctx;
+ LOCALDISPLAYLEVEL(displayLevel, 3, "d=%u\n", d);
+ if (!COVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, d)) {
+ LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to initialize context\n");
+ COVER_best_destroy(&best);
+ return ERROR(GENERIC);
+ }
+ /* Loop through k reusing the same context */
+ for (k = kMinK; k <= kMaxK; k += kStepSize) {
+ /* Prepare the arguments */
+ COVER_tryParameters_data_t *data = (COVER_tryParameters_data_t *)malloc(
+ sizeof(COVER_tryParameters_data_t));
+ LOCALDISPLAYLEVEL(displayLevel, 3, "k=%u\n", k);
+ if (!data) {
+ LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to allocate parameters\n");
+ COVER_best_destroy(&best);
+ COVER_ctx_destroy(&ctx);
+ return ERROR(GENERIC);
+ }
+ data->ctx = &ctx;
+ data->best = &best;
+ data->dictBufferCapacity = dictBufferCapacity;
+ data->parameters = *parameters;
+ data->parameters.k = k;
+ data->parameters.d = d;
+ data->parameters.steps = kSteps;
+ /* Check the parameters */
+ if (!COVER_checkParameters(data->parameters)) {
+ DISPLAYLEVEL(1, "Cover parameters incorrect\n");
+ continue;
+ }
+ /* Call the function and pass ownership of data to it */
+ COVER_best_start(&best);
+ if (pool) {
+ POOL_add(pool, &COVER_tryParameters, data);
+ } else {
+ COVER_tryParameters(data);
+ }
+ /* Print status */
+ LOCALDISPLAYUPDATE(displayLevel, 2, "\r%u%% ",
+ (U32)((iteration * 100) / kIterations));
+ ++iteration;
+ }
+ COVER_best_wait(&best);
+ COVER_ctx_destroy(&ctx);
+ }
+ LOCALDISPLAYLEVEL(displayLevel, 2, "\r%79s\r", "");
+ /* Fill the output buffer and parameters with output of the best parameters */
+ {
+ const size_t dictSize = best.dictSize;
+ if (ZSTD_isError(best.compressedSize)) {
+ COVER_best_destroy(&best);
+ return best.compressedSize;
+ }
+ *parameters = best.parameters;
+ memcpy(dictBuffer, best.dict, dictSize);
+ COVER_best_destroy(&best);
+ POOL_free(pool);
+ return dictSize;
+ }
+}
--- a/contrib/python-zstandard/zstd/dictBuilder/zdict.c Sat Feb 25 12:48:50 2017 +0900
+++ b/contrib/python-zstandard/zstd/dictBuilder/zdict.c Tue Feb 28 11:13:25 2017 -0800
@@ -36,12 +36,11 @@
#include <time.h> /* clock */
#include "mem.h" /* read */
-#include "error_private.h"
#include "fse.h" /* FSE_normalizeCount, FSE_writeNCount */
#define HUF_STATIC_LINKING_ONLY
-#include "huf.h"
+#include "huf.h" /* HUF_buildCTable, HUF_writeCTable */
#include "zstd_internal.h" /* includes zstd.h */
-#include "xxhash.h"
+#include "xxhash.h" /* XXH64 */
#include "divsufsort.h"
#ifndef ZDICT_STATIC_LINKING_ONLY
# define ZDICT_STATIC_LINKING_ONLY
@@ -61,7 +60,7 @@
#define NOISELENGTH 32
#define MINRATIO 4
-static const int g_compressionLevel_default = 5;
+static const int g_compressionLevel_default = 6;
static const U32 g_selectivity_default = 9;
static const size_t g_provision_entropySize = 200;
static const size_t g_min_fast_dictContent = 192;
@@ -307,13 +306,13 @@
} while (length >=MINMATCHLENGTH);
/* look backward */
- length = MINMATCHLENGTH;
- while ((length >= MINMATCHLENGTH) & (start > 0)) {
- length = ZDICT_count(b + pos, b + suffix[start - 1]);
- if (length >= LLIMIT) length = LLIMIT - 1;
- lengthList[length]++;
- if (length >= MINMATCHLENGTH) start--;
- }
+ length = MINMATCHLENGTH;
+ while ((length >= MINMATCHLENGTH) & (start > 0)) {
+ length = ZDICT_count(b + pos, b + suffix[start - 1]);
+ if (length >= LLIMIT) length = LLIMIT - 1;
+ lengthList[length]++;
+ if (length >= MINMATCHLENGTH) start--;
+ }
/* largest useful length */
memset(cumulLength, 0, sizeof(cumulLength));
@@ -570,7 +569,7 @@
if (ZSTD_isError(errorCode)) { DISPLAYLEVEL(1, "warning : ZSTD_copyCCtx failed \n"); return; }
}
cSize = ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_ABSOLUTEMAX, src, srcSize);
- if (ZSTD_isError(cSize)) { DISPLAYLEVEL(1, "warning : could not compress sample size %u \n", (U32)srcSize); return; }
+ if (ZSTD_isError(cSize)) { DISPLAYLEVEL(3, "warning : could not compress sample size %u \n", (U32)srcSize); return; }
if (cSize) { /* if == 0; block is not compressible */
const seqStore_t* seqStorePtr = ZSTD_getSeqStore(esr.zc);
@@ -825,6 +824,55 @@
}
+
+size_t ZDICT_finalizeDictionary(void* dictBuffer, size_t dictBufferCapacity,
+ const void* customDictContent, size_t dictContentSize,
+ const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
+ ZDICT_params_t params)
+{
+ size_t hSize;
+#define HBUFFSIZE 256
+ BYTE header[HBUFFSIZE];
+ int const compressionLevel = (params.compressionLevel <= 0) ? g_compressionLevel_default : params.compressionLevel;
+ U32 const notificationLevel = params.notificationLevel;
+
+ /* check conditions */
+ if (dictBufferCapacity < dictContentSize) return ERROR(dstSize_tooSmall);
+ if (dictContentSize < ZDICT_CONTENTSIZE_MIN) return ERROR(srcSize_wrong);
+ if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) return ERROR(dstSize_tooSmall);
+
+ /* dictionary header */
+ MEM_writeLE32(header, ZSTD_DICT_MAGIC);
+ { U64 const randomID = XXH64(customDictContent, dictContentSize, 0);
+ U32 const compliantID = (randomID % ((1U<<31)-32768)) + 32768;
+ U32 const dictID = params.dictID ? params.dictID : compliantID;
+ MEM_writeLE32(header+4, dictID);
+ }
+ hSize = 8;
+
+ /* entropy tables */
+ DISPLAYLEVEL(2, "\r%70s\r", ""); /* clean display line */
+ DISPLAYLEVEL(2, "statistics ... \n");
+ { size_t const eSize = ZDICT_analyzeEntropy(header+hSize, HBUFFSIZE-hSize,
+ compressionLevel,
+ samplesBuffer, samplesSizes, nbSamples,
+ customDictContent, dictContentSize,
+ notificationLevel);
+ if (ZDICT_isError(eSize)) return eSize;
+ hSize += eSize;
+ }
+
+ /* copy elements in final buffer ; note : src and dst buffer can overlap */
+ if (hSize + dictContentSize > dictBufferCapacity) dictContentSize = dictBufferCapacity - hSize;
+ { size_t const dictSize = hSize + dictContentSize;
+ char* dictEnd = (char*)dictBuffer + dictSize;
+ memmove(dictEnd - dictContentSize, customDictContent, dictContentSize);
+ memcpy(dictBuffer, header, hSize);
+ return dictSize;
+ }
+}
+
+
size_t ZDICT_addEntropyTablesFromBuffer_advanced(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity,
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
ZDICT_params_t params)
--- a/contrib/python-zstandard/zstd/dictBuilder/zdict.h Sat Feb 25 12:48:50 2017 +0900
+++ b/contrib/python-zstandard/zstd/dictBuilder/zdict.h Tue Feb 28 11:13:25 2017 -0800
@@ -19,15 +19,18 @@
#include <stddef.h> /* size_t */
-/*====== Export for Windows ======*/
-/*!
-* ZSTD_DLL_EXPORT :
-* Enable exporting of functions when building a Windows DLL
-*/
-#if defined(_WIN32) && defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
-# define ZDICTLIB_API __declspec(dllexport)
+/* ===== ZDICTLIB_API : control library symbols visibility ===== */
+#if defined(__GNUC__) && (__GNUC__ >= 4)
+# define ZDICTLIB_VISIBILITY __attribute__ ((visibility ("default")))
#else
-# define ZDICTLIB_API
+# define ZDICTLIB_VISIBILITY
+#endif
+#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
+# define ZDICTLIB_API __declspec(dllexport) ZDICTLIB_VISIBILITY
+#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
+# define ZDICTLIB_API __declspec(dllimport) ZDICTLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
+#else
+# define ZDICTLIB_API ZDICTLIB_VISIBILITY
#endif
@@ -79,27 +82,114 @@
or an error code, which can be tested by ZDICT_isError().
note : ZDICT_trainFromBuffer_advanced() will send notifications into stderr if instructed to, using notificationLevel>0.
*/
-size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dictBufferCapacity,
+ZDICTLIB_API size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dictBufferCapacity,
+ const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
+ ZDICT_params_t parameters);
+
+/*! COVER_params_t :
+ For all values 0 means default.
+ kMin and d are the only required parameters.
+*/
+typedef struct {
+ unsigned k; /* Segment size : constraint: 0 < k : Reasonable range [16, 2048+] */
+ unsigned d; /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */
+ unsigned steps; /* Number of steps : Only used for optimization : 0 means default (32) : Higher means more parameters checked */
+
+ unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */
+ unsigned notificationLevel; /* Write to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */
+ unsigned dictID; /* 0 means auto mode (32-bits random value); other : force dictID value */
+ int compressionLevel; /* 0 means default; target a specific zstd compression level */
+} COVER_params_t;
+
+
+/*! COVER_trainFromBuffer() :
+ Train a dictionary from an array of samples using the COVER algorithm.
+ Samples must be stored concatenated in a single flat buffer `samplesBuffer`,
+ supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order.
+ The resulting dictionary will be saved into `dictBuffer`.
+ @return : size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
+ or an error code, which can be tested with ZDICT_isError().
+ Note : COVER_trainFromBuffer() requires about 9 bytes of memory for each input byte.
+ Tips : In general, a reasonable dictionary has a size of ~ 100 KB.
+ It's obviously possible to target smaller or larger ones, just by specifying different `dictBufferCapacity`.
+ In general, it's recommended to provide a few thousands samples, but this can vary a lot.
+ It's recommended that total size of all samples be about ~x100 times the target size of dictionary.
+*/
+ZDICTLIB_API size_t COVER_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity,
+ const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
+ COVER_params_t parameters);
+
+/*! COVER_optimizeTrainFromBuffer() :
+ The same requirements as above hold for all the parameters except `parameters`.
+ This function tries many parameter combinations and picks the best parameters.
+ `*parameters` is filled with the best parameters found, and the dictionary
+ constructed with those parameters is stored in `dictBuffer`.
+
+ All of the parameters d, k, steps are optional.
+ If d is non-zero then we don't check multiple values of d, otherwise we check d = {6, 8, 10, 12, 14, 16}.
+ if steps is zero it defaults to its default value.
+ If k is non-zero then we don't check multiple values of k, otherwise we check steps values in [16, 2048].
+
+ @return : size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
+ or an error code, which can be tested with ZDICT_isError().
+ On success `*parameters` contains the parameters selected.
+ Note : COVER_optimizeTrainFromBuffer() requires about 8 bytes of memory for each input byte and additionally another 5 bytes of memory for each byte of memory for each thread.
+*/
+ZDICTLIB_API size_t COVER_optimizeTrainFromBuffer(void* dictBuffer, size_t dictBufferCapacity,
+ const void* samplesBuffer, const size_t *samplesSizes, unsigned nbSamples,
+ COVER_params_t *parameters);
+
+/*! ZDICT_finalizeDictionary() :
+
+ Given a custom content as a basis for dictionary, and a set of samples,
+ finalize dictionary by adding headers and statistics.
+
+ Samples must be stored concatenated in a flat buffer `samplesBuffer`,
+ supplied with an array of sizes `samplesSizes`, providing the size of each sample in order.
+
+ dictContentSize must be > ZDICT_CONTENTSIZE_MIN bytes.
+ maxDictSize must be >= dictContentSize, and must be > ZDICT_DICTSIZE_MIN bytes.
+
+ @return : size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`),
+ or an error code, which can be tested by ZDICT_isError().
+ note : ZDICT_finalizeDictionary() will push notifications into stderr if instructed to, using notificationLevel>0.
+ note 2 : dictBuffer and customDictContent can overlap
+*/
+#define ZDICT_CONTENTSIZE_MIN 256
+#define ZDICT_DICTSIZE_MIN 512
+ZDICTLIB_API size_t ZDICT_finalizeDictionary(void* dictBuffer, size_t dictBufferCapacity,
+ const void* customDictContent, size_t dictContentSize,
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
ZDICT_params_t parameters);
-/*! ZDICT_addEntropyTablesFromBuffer() :
-
- Given a content-only dictionary (built using any 3rd party algorithm),
- add entropy tables computed from an array of samples.
- Samples must be stored concatenated in a flat buffer `samplesBuffer`,
- supplied with an array of sizes `samplesSizes`, providing the size of each sample in order.
- The input dictionary content must be stored *at the end* of `dictBuffer`.
- Its size is `dictContentSize`.
- The resulting dictionary with added entropy tables will be *written back to `dictBuffer`*,
- starting from its beginning.
- @return : size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`).
-*/
+/* Deprecation warnings */
+/* It is generally possible to disable deprecation warnings from compiler,
+ for example with -Wno-deprecated-declarations for gcc
+ or _CRT_SECURE_NO_WARNINGS in Visual.
+ Otherwise, it's also possible to manually define ZDICT_DISABLE_DEPRECATE_WARNINGS */
+#ifdef ZDICT_DISABLE_DEPRECATE_WARNINGS
+# define ZDICT_DEPRECATED(message) ZDICTLIB_API /* disable deprecation warnings */
+#else
+# define ZDICT_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
+# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
+# define ZDICT_DEPRECATED(message) ZDICTLIB_API [[deprecated(message)]]
+# elif (ZDICT_GCC_VERSION >= 405) || defined(__clang__)
+# define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated(message)))
+# elif (ZDICT_GCC_VERSION >= 301)
+# define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated))
+# elif defined(_MSC_VER)
+# define ZDICT_DEPRECATED(message) ZDICTLIB_API __declspec(deprecated(message))
+# else
+# pragma message("WARNING: You need to implement ZDICT_DEPRECATED for this compiler")
+# define ZDICT_DEPRECATED(message) ZDICTLIB_API
+# endif
+#endif /* ZDICT_DISABLE_DEPRECATE_WARNINGS */
+
+ZDICT_DEPRECATED("use ZDICT_finalizeDictionary() instead")
size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity,
- const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples);
-
+ const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples);
#endif /* ZDICT_STATIC_LINKING_ONLY */
--- a/contrib/python-zstandard/zstd/zstd.h Sat Feb 25 12:48:50 2017 +0900
+++ b/contrib/python-zstandard/zstd/zstd.h Tue Feb 28 11:13:25 2017 -0800
@@ -20,13 +20,16 @@
/* ===== ZSTDLIB_API : control library symbols visibility ===== */
#if defined(__GNUC__) && (__GNUC__ >= 4)
-# define ZSTDLIB_API __attribute__ ((visibility ("default")))
-#elif defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
-# define ZSTDLIB_API __declspec(dllexport)
+# define ZSTDLIB_VISIBILITY __attribute__ ((visibility ("default")))
+#else
+# define ZSTDLIB_VISIBILITY
+#endif
+#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
+# define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBILITY
#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
-# define ZSTDLIB_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
+# define ZSTDLIB_API __declspec(dllimport) ZSTDLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
#else
-# define ZSTDLIB_API
+# define ZSTDLIB_API ZSTDLIB_VISIBILITY
#endif
@@ -53,7 +56,7 @@
/*------ Version ------*/
#define ZSTD_VERSION_MAJOR 1
#define ZSTD_VERSION_MINOR 1
-#define ZSTD_VERSION_RELEASE 2
+#define ZSTD_VERSION_RELEASE 3
#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
#define ZSTD_QUOTE(str) #str
@@ -170,8 +173,8 @@
* When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
* ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
* ZSTD_CDict can be created once and used by multiple threads concurrently, as its usage is read-only.
-* `dict` can be released after ZSTD_CDict creation. */
-ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel);
+* `dictBuffer` can be released after ZSTD_CDict creation, as its content is copied within CDict */
+ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, int compressionLevel);
/*! ZSTD_freeCDict() :
* Function frees memory allocated by ZSTD_createCDict(). */
@@ -191,8 +194,8 @@
/*! ZSTD_createDDict() :
* Create a digested dictionary, ready to start decompression operation without startup delay.
-* `dict` can be released after creation. */
-ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize);
+* dictBuffer can be released after DDict creation, as its content is copied inside DDict */
+ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize);
/*! ZSTD_freeDDict() :
* Function frees memory allocated with ZSTD_createDDict() */
@@ -325,7 +328,7 @@
* ***************************************************************************************/
/* --- Constants ---*/
-#define ZSTD_MAGICNUMBER 0xFD2FB528 /* v0.8 */
+#define ZSTD_MAGICNUMBER 0xFD2FB528 /* >= v0.8.0 */
#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U
#define ZSTD_WINDOWLOG_MAX_32 25
@@ -345,8 +348,9 @@
#define ZSTD_TARGETLENGTH_MAX 999
#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* for static allocation */
+#define ZSTD_FRAMEHEADERSIZE_MIN 6
static const size_t ZSTD_frameHeaderSize_prefix = 5;
-static const size_t ZSTD_frameHeaderSize_min = 6;
+static const size_t ZSTD_frameHeaderSize_min = ZSTD_FRAMEHEADERSIZE_MIN;
static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX;
static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable frame length */
@@ -365,9 +369,9 @@
} ZSTD_compressionParameters;
typedef struct {
- unsigned contentSizeFlag; /**< 1: content size will be in frame header (if known). */
- unsigned checksumFlag; /**< 1: will generate a 22-bits checksum at end of frame, to be used for error detection by decompressor */
- unsigned noDictIDFlag; /**< 1: no dict ID will be saved into frame header (if dictionary compression) */
+ unsigned contentSizeFlag; /**< 1: content size will be in frame header (when known) */
+ unsigned checksumFlag; /**< 1: generate a 32-bits checksum at end of frame, for error detection */
+ unsigned noDictIDFlag; /**< 1: no dictID will be saved into frame header (if dictionary compression) */
} ZSTD_frameParameters;
typedef struct {
@@ -397,9 +401,23 @@
* Gives the amount of memory used by a given ZSTD_CCtx */
ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
+typedef enum {
+ ZSTD_p_forceWindow /* Force back-references to remain < windowSize, even when referencing Dictionary content (default:0)*/
+} ZSTD_CCtxParameter;
+/*! ZSTD_setCCtxParameter() :
+ * Set advanced parameters, selected through enum ZSTD_CCtxParameter
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()) */
+ZSTDLIB_API size_t ZSTD_setCCtxParameter(ZSTD_CCtx* cctx, ZSTD_CCtxParameter param, unsigned value);
+
+/*! ZSTD_createCDict_byReference() :
+ * Create a digested dictionary for compression
+ * Dictionary content is simply referenced, and therefore stays in dictBuffer.
+ * It is important that dictBuffer outlives CDict, it must remain read accessible throughout the lifetime of CDict */
+ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
+
/*! ZSTD_createCDict_advanced() :
* Create a ZSTD_CDict using external alloc and free, and customized compression parameters */
-ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize,
+ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, unsigned byReference,
ZSTD_parameters params, ZSTD_customMem customMem);
/*! ZSTD_sizeof_CDict() :
@@ -455,6 +473,15 @@
* Gives the amount of memory used by a given ZSTD_DCtx */
ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
+/*! ZSTD_createDDict_byReference() :
+ * Create a digested dictionary, ready to start decompression operation without startup delay.
+ * Dictionary content is simply referenced, and therefore stays in dictBuffer.
+ * It is important that dictBuffer outlives DDict, it must remain read accessible throughout the lifetime of DDict */
+ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize);
+
+ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
+ unsigned byReference, ZSTD_customMem customMem);
+
/*! ZSTD_sizeof_DDict() :
* Gives the amount of memory used by a given ZSTD_DDict */
ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
@@ -463,13 +490,13 @@
* Provides the dictID stored within dictionary.
* if @return == 0, the dictionary is not conformant with Zstandard specification.
* It can still be loaded, but as a content-only dictionary. */
-unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize);
+ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize);
/*! ZSTD_getDictID_fromDDict() :
* Provides the dictID of the dictionary loaded into `ddict`.
* If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
* Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
-unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
+ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
/*! ZSTD_getDictID_fromFrame() :
* Provides the dictID required to decompressed the frame stored within `src`.
@@ -481,7 +508,7 @@
* - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
* - This is not a Zstandard frame.
* When identifying the exact failure cause, it's possible to used ZSTD_getFrameParams(), which will provide a more precise error code. */
-unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
+ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
/********************************************************************
@@ -491,7 +518,7 @@
/*===== Advanced Streaming compression functions =====*/
ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct */
-ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
+ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< note: a dict will not be used if dict == NULL or dictSize < 8 */
ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be zero == unknown */
ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /**< note : cdict will just be referenced, and must outlive compression session */
@@ -500,9 +527,9 @@
/*===== Advanced Streaming decompression functions =====*/
-typedef enum { ZSTDdsp_maxWindowSize } ZSTD_DStreamParameter_e;
+typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e;
ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
-ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
+ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); /**< note: a dict will not be used if dict == NULL or dictSize < 8 */
ZSTDLIB_API size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);
ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); /**< note : ddict will just be referenced, and must outlive decompression session */
ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); /**< re-use decompression parameters from previous init; saves dictionary loading */
@@ -542,10 +569,10 @@
In which case, it will "discard" the relevant memory section from its history.
Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum.
- It's possible to use a NULL,0 src content, in which case, it will write a final empty block to end the frame,
- Without last block mark, frames will be considered unfinished (broken) by decoders.
+ It's possible to use srcSize==0, in which case, it will write a final empty block to end the frame.
+ Without last block mark, frames will be considered unfinished (corrupted) by decoders.
- You can then reuse `ZSTD_CCtx` (ZSTD_compressBegin()) to compress some new frame.
+ `ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress some new frame.
*/
/*===== Buffer-less streaming compression functions =====*/
@@ -553,6 +580,7 @@
ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize);
ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize);
+ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize);
ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
--- a/contrib/python-zstandard/zstd_cffi.py Sat Feb 25 12:48:50 2017 +0900
+++ b/contrib/python-zstandard/zstd_cffi.py Tue Feb 28 11:13:25 2017 -0800
@@ -8,145 +8,1035 @@
from __future__ import absolute_import, unicode_literals
-import io
+import sys
from _zstd_cffi import (
ffi,
lib,
)
+if sys.version_info[0] == 2:
+ bytes_type = str
+ int_type = long
+else:
+ bytes_type = bytes
+ int_type = int
-_CSTREAM_IN_SIZE = lib.ZSTD_CStreamInSize()
-_CSTREAM_OUT_SIZE = lib.ZSTD_CStreamOutSize()
+
+COMPRESSION_RECOMMENDED_INPUT_SIZE = lib.ZSTD_CStreamInSize()
+COMPRESSION_RECOMMENDED_OUTPUT_SIZE = lib.ZSTD_CStreamOutSize()
+DECOMPRESSION_RECOMMENDED_INPUT_SIZE = lib.ZSTD_DStreamInSize()
+DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE = lib.ZSTD_DStreamOutSize()
+
+new_nonzero = ffi.new_allocator(should_clear_after_alloc=False)
+
+
+MAX_COMPRESSION_LEVEL = lib.ZSTD_maxCLevel()
+MAGIC_NUMBER = lib.ZSTD_MAGICNUMBER
+FRAME_HEADER = b'\x28\xb5\x2f\xfd'
+ZSTD_VERSION = (lib.ZSTD_VERSION_MAJOR, lib.ZSTD_VERSION_MINOR, lib.ZSTD_VERSION_RELEASE)
+
+WINDOWLOG_MIN = lib.ZSTD_WINDOWLOG_MIN
+WINDOWLOG_MAX = lib.ZSTD_WINDOWLOG_MAX
+CHAINLOG_MIN = lib.ZSTD_CHAINLOG_MIN
+CHAINLOG_MAX = lib.ZSTD_CHAINLOG_MAX
+HASHLOG_MIN = lib.ZSTD_HASHLOG_MIN
+HASHLOG_MAX = lib.ZSTD_HASHLOG_MAX
+HASHLOG3_MAX = lib.ZSTD_HASHLOG3_MAX
+SEARCHLOG_MIN = lib.ZSTD_SEARCHLOG_MIN
+SEARCHLOG_MAX = lib.ZSTD_SEARCHLOG_MAX
+SEARCHLENGTH_MIN = lib.ZSTD_SEARCHLENGTH_MIN
+SEARCHLENGTH_MAX = lib.ZSTD_SEARCHLENGTH_MAX
+TARGETLENGTH_MIN = lib.ZSTD_TARGETLENGTH_MIN
+TARGETLENGTH_MAX = lib.ZSTD_TARGETLENGTH_MAX
+
+STRATEGY_FAST = lib.ZSTD_fast
+STRATEGY_DFAST = lib.ZSTD_dfast
+STRATEGY_GREEDY = lib.ZSTD_greedy
+STRATEGY_LAZY = lib.ZSTD_lazy
+STRATEGY_LAZY2 = lib.ZSTD_lazy2
+STRATEGY_BTLAZY2 = lib.ZSTD_btlazy2
+STRATEGY_BTOPT = lib.ZSTD_btopt
+
+COMPRESSOBJ_FLUSH_FINISH = 0
+COMPRESSOBJ_FLUSH_BLOCK = 1
+
+
+class ZstdError(Exception):
+ pass
-class _ZstdCompressionWriter(object):
- def __init__(self, cstream, writer):
- self._cstream = cstream
+class CompressionParameters(object):
+ def __init__(self, window_log, chain_log, hash_log, search_log,
+ search_length, target_length, strategy):
+ if window_log < WINDOWLOG_MIN or window_log > WINDOWLOG_MAX:
+ raise ValueError('invalid window log value')
+
+ if chain_log < CHAINLOG_MIN or chain_log > CHAINLOG_MAX:
+ raise ValueError('invalid chain log value')
+
+ if hash_log < HASHLOG_MIN or hash_log > HASHLOG_MAX:
+ raise ValueError('invalid hash log value')
+
+ if search_log < SEARCHLOG_MIN or search_log > SEARCHLOG_MAX:
+ raise ValueError('invalid search log value')
+
+ if search_length < SEARCHLENGTH_MIN or search_length > SEARCHLENGTH_MAX:
+ raise ValueError('invalid search length value')
+
+ if target_length < TARGETLENGTH_MIN or target_length > TARGETLENGTH_MAX:
+ raise ValueError('invalid target length value')
+
+ if strategy < STRATEGY_FAST or strategy > STRATEGY_BTOPT:
+ raise ValueError('invalid strategy value')
+
+ self.window_log = window_log
+ self.chain_log = chain_log
+ self.hash_log = hash_log
+ self.search_log = search_log
+ self.search_length = search_length
+ self.target_length = target_length
+ self.strategy = strategy
+
+ def as_compression_parameters(self):
+ p = ffi.new('ZSTD_compressionParameters *')[0]
+ p.windowLog = self.window_log
+ p.chainLog = self.chain_log
+ p.hashLog = self.hash_log
+ p.searchLog = self.search_log
+ p.searchLength = self.search_length
+ p.targetLength = self.target_length
+ p.strategy = self.strategy
+
+ return p
+
+def get_compression_parameters(level, source_size=0, dict_size=0):
+ params = lib.ZSTD_getCParams(level, source_size, dict_size)
+ return CompressionParameters(window_log=params.windowLog,
+ chain_log=params.chainLog,
+ hash_log=params.hashLog,
+ search_log=params.searchLog,
+ search_length=params.searchLength,
+ target_length=params.targetLength,
+ strategy=params.strategy)
+
+
+def estimate_compression_context_size(params):
+ if not isinstance(params, CompressionParameters):
+ raise ValueError('argument must be a CompressionParameters')
+
+ cparams = params.as_compression_parameters()
+ return lib.ZSTD_estimateCCtxSize(cparams)
+
+
+def estimate_decompression_context_size():
+ return lib.ZSTD_estimateDCtxSize()
+
+
+class ZstdCompressionWriter(object):
+ def __init__(self, compressor, writer, source_size, write_size):
+ self._compressor = compressor
self._writer = writer
+ self._source_size = source_size
+ self._write_size = write_size
+ self._entered = False
def __enter__(self):
+ if self._entered:
+ raise ZstdError('cannot __enter__ multiple times')
+
+ self._cstream = self._compressor._get_cstream(self._source_size)
+ self._entered = True
return self
def __exit__(self, exc_type, exc_value, exc_tb):
+ self._entered = False
+
if not exc_type and not exc_value and not exc_tb:
out_buffer = ffi.new('ZSTD_outBuffer *')
- out_buffer.dst = ffi.new('char[]', _CSTREAM_OUT_SIZE)
- out_buffer.size = _CSTREAM_OUT_SIZE
+ dst_buffer = ffi.new('char[]', self._write_size)
+ out_buffer.dst = dst_buffer
+ out_buffer.size = self._write_size
out_buffer.pos = 0
while True:
- res = lib.ZSTD_endStream(self._cstream, out_buffer)
- if lib.ZSTD_isError(res):
- raise Exception('error ending compression stream: %s' % lib.ZSTD_getErrorName)
+ zresult = lib.ZSTD_endStream(self._cstream, out_buffer)
+ if lib.ZSTD_isError(zresult):
+ raise ZstdError('error ending compression stream: %s' %
+ ffi.string(lib.ZSTD_getErrorName(zresult)))
if out_buffer.pos:
- self._writer.write(ffi.buffer(out_buffer.dst, out_buffer.pos))
+ self._writer.write(ffi.buffer(out_buffer.dst, out_buffer.pos)[:])
out_buffer.pos = 0
- if res == 0:
+ if zresult == 0:
break
+ self._cstream = None
+ self._compressor = None
+
return False
+ def memory_size(self):
+ if not self._entered:
+ raise ZstdError('cannot determine size of an inactive compressor; '
+ 'call when a context manager is active')
+
+ return lib.ZSTD_sizeof_CStream(self._cstream)
+
def write(self, data):
+ if not self._entered:
+ raise ZstdError('write() must be called from an active context '
+ 'manager')
+
+ total_write = 0
+
+ data_buffer = ffi.from_buffer(data)
+
+ in_buffer = ffi.new('ZSTD_inBuffer *')
+ in_buffer.src = data_buffer
+ in_buffer.size = len(data_buffer)
+ in_buffer.pos = 0
+
out_buffer = ffi.new('ZSTD_outBuffer *')
- out_buffer.dst = ffi.new('char[]', _CSTREAM_OUT_SIZE)
- out_buffer.size = _CSTREAM_OUT_SIZE
+ dst_buffer = ffi.new('char[]', self._write_size)
+ out_buffer.dst = dst_buffer
+ out_buffer.size = self._write_size
+ out_buffer.pos = 0
+
+ while in_buffer.pos < in_buffer.size:
+ zresult = lib.ZSTD_compressStream(self._cstream, out_buffer, in_buffer)
+ if lib.ZSTD_isError(zresult):
+ raise ZstdError('zstd compress error: %s' %
+ ffi.string(lib.ZSTD_getErrorName(zresult)))
+
+ if out_buffer.pos:
+ self._writer.write(ffi.buffer(out_buffer.dst, out_buffer.pos)[:])
+ total_write += out_buffer.pos
+ out_buffer.pos = 0
+
+ return total_write
+
+ def flush(self):
+ if not self._entered:
+ raise ZstdError('flush must be called from an active context manager')
+
+ total_write = 0
+
+ out_buffer = ffi.new('ZSTD_outBuffer *')
+ dst_buffer = ffi.new('char[]', self._write_size)
+ out_buffer.dst = dst_buffer
+ out_buffer.size = self._write_size
out_buffer.pos = 0
- # TODO can we reuse existing memory?
- in_buffer = ffi.new('ZSTD_inBuffer *')
- in_buffer.src = ffi.new('char[]', data)
- in_buffer.size = len(data)
- in_buffer.pos = 0
- while in_buffer.pos < in_buffer.size:
- res = lib.ZSTD_compressStream(self._cstream, out_buffer, in_buffer)
- if lib.ZSTD_isError(res):
- raise Exception('zstd compress error: %s' % lib.ZSTD_getErrorName(res))
+ while True:
+ zresult = lib.ZSTD_flushStream(self._cstream, out_buffer)
+ if lib.ZSTD_isError(zresult):
+ raise ZstdError('zstd compress error: %s' %
+ ffi.string(lib.ZSTD_getErrorName(zresult)))
+
+ if not out_buffer.pos:
+ break
+
+ self._writer.write(ffi.buffer(out_buffer.dst, out_buffer.pos)[:])
+ total_write += out_buffer.pos
+ out_buffer.pos = 0
+
+ return total_write
+
+
+class ZstdCompressionObj(object):
+ def compress(self, data):
+ if self._finished:
+ raise ZstdError('cannot call compress() after compressor finished')
+
+ data_buffer = ffi.from_buffer(data)
+ source = ffi.new('ZSTD_inBuffer *')
+ source.src = data_buffer
+ source.size = len(data_buffer)
+ source.pos = 0
+
+ chunks = []
+
+ while source.pos < len(data):
+ zresult = lib.ZSTD_compressStream(self._cstream, self._out, source)
+ if lib.ZSTD_isError(zresult):
+ raise ZstdError('zstd compress error: %s' %
+ ffi.string(lib.ZSTD_getErrorName(zresult)))
+
+ if self._out.pos:
+ chunks.append(ffi.buffer(self._out.dst, self._out.pos)[:])
+ self._out.pos = 0
+
+ return b''.join(chunks)
- if out_buffer.pos:
- self._writer.write(ffi.buffer(out_buffer.dst, out_buffer.pos))
- out_buffer.pos = 0
+ def flush(self, flush_mode=COMPRESSOBJ_FLUSH_FINISH):
+ if flush_mode not in (COMPRESSOBJ_FLUSH_FINISH, COMPRESSOBJ_FLUSH_BLOCK):
+ raise ValueError('flush mode not recognized')
+
+ if self._finished:
+ raise ZstdError('compressor object already finished')
+
+ assert self._out.pos == 0
+
+ if flush_mode == COMPRESSOBJ_FLUSH_BLOCK:
+ zresult = lib.ZSTD_flushStream(self._cstream, self._out)
+ if lib.ZSTD_isError(zresult):
+ raise ZstdError('zstd compress error: %s' %
+ ffi.string(lib.ZSTD_getErrorName(zresult)))
+
+ # Output buffer is guaranteed to hold full block.
+ assert zresult == 0
+
+ if self._out.pos:
+ result = ffi.buffer(self._out.dst, self._out.pos)[:]
+ self._out.pos = 0
+ return result
+ else:
+ return b''
+
+ assert flush_mode == COMPRESSOBJ_FLUSH_FINISH
+ self._finished = True
+
+ chunks = []
+
+ while True:
+ zresult = lib.ZSTD_endStream(self._cstream, self._out)
+ if lib.ZSTD_isError(zresult):
+ raise ZstdError('error ending compression stream: %s' %
+ ffi.string(lib.ZSTD_getErroName(zresult)))
+
+ if self._out.pos:
+ chunks.append(ffi.buffer(self._out.dst, self._out.pos)[:])
+ self._out.pos = 0
+
+ if not zresult:
+ break
+
+ # GC compression stream immediately.
+ self._cstream = None
+
+ return b''.join(chunks)
class ZstdCompressor(object):
- def __init__(self, level=3, dict_data=None, compression_params=None):
- if dict_data:
- raise Exception('dict_data not yet supported')
- if compression_params:
- raise Exception('compression_params not yet supported')
+ def __init__(self, level=3, dict_data=None, compression_params=None,
+ write_checksum=False, write_content_size=False,
+ write_dict_id=True):
+ if level < 1:
+ raise ValueError('level must be greater than 0')
+ elif level > lib.ZSTD_maxCLevel():
+ raise ValueError('level must be less than %d' % lib.ZSTD_maxCLevel())
self._compression_level = level
+ self._dict_data = dict_data
+ self._cparams = compression_params
+ self._fparams = ffi.new('ZSTD_frameParameters *')[0]
+ self._fparams.checksumFlag = write_checksum
+ self._fparams.contentSizeFlag = write_content_size
+ self._fparams.noDictIDFlag = not write_dict_id
- def compress(self, data):
- # Just use the stream API for now.
- output = io.BytesIO()
- with self.write_to(output) as compressor:
- compressor.write(data)
- return output.getvalue()
+ cctx = lib.ZSTD_createCCtx()
+ if cctx == ffi.NULL:
+ raise MemoryError()
+
+ self._cctx = ffi.gc(cctx, lib.ZSTD_freeCCtx)
+
+ def compress(self, data, allow_empty=False):
+ if len(data) == 0 and self._fparams.contentSizeFlag and not allow_empty:
+ raise ValueError('cannot write empty inputs when writing content sizes')
+
+ # TODO use a CDict for performance.
+ dict_data = ffi.NULL
+ dict_size = 0
+
+ if self._dict_data:
+ dict_data = self._dict_data.as_bytes()
+ dict_size = len(self._dict_data)
+
+ params = ffi.new('ZSTD_parameters *')[0]
+ if self._cparams:
+ params.cParams = self._cparams.as_compression_parameters()
+ else:
+ params.cParams = lib.ZSTD_getCParams(self._compression_level, len(data),
+ dict_size)
+ params.fParams = self._fparams
+
+ dest_size = lib.ZSTD_compressBound(len(data))
+ out = new_nonzero('char[]', dest_size)
- def copy_stream(self, ifh, ofh):
- cstream = self._get_cstream()
+ zresult = lib.ZSTD_compress_advanced(self._cctx,
+ ffi.addressof(out), dest_size,
+ data, len(data),
+ dict_data, dict_size,
+ params)
+
+ if lib.ZSTD_isError(zresult):
+ raise ZstdError('cannot compress: %s' %
+ ffi.string(lib.ZSTD_getErrorName(zresult)))
+
+ return ffi.buffer(out, zresult)[:]
+
+ def compressobj(self, size=0):
+ cstream = self._get_cstream(size)
+ cobj = ZstdCompressionObj()
+ cobj._cstream = cstream
+ cobj._out = ffi.new('ZSTD_outBuffer *')
+ cobj._dst_buffer = ffi.new('char[]', COMPRESSION_RECOMMENDED_OUTPUT_SIZE)
+ cobj._out.dst = cobj._dst_buffer
+ cobj._out.size = COMPRESSION_RECOMMENDED_OUTPUT_SIZE
+ cobj._out.pos = 0
+ cobj._compressor = self
+ cobj._finished = False
+
+ return cobj
+
+ def copy_stream(self, ifh, ofh, size=0,
+ read_size=COMPRESSION_RECOMMENDED_INPUT_SIZE,
+ write_size=COMPRESSION_RECOMMENDED_OUTPUT_SIZE):
+
+ if not hasattr(ifh, 'read'):
+ raise ValueError('first argument must have a read() method')
+ if not hasattr(ofh, 'write'):
+ raise ValueError('second argument must have a write() method')
+
+ cstream = self._get_cstream(size)
in_buffer = ffi.new('ZSTD_inBuffer *')
out_buffer = ffi.new('ZSTD_outBuffer *')
- out_buffer.dst = ffi.new('char[]', _CSTREAM_OUT_SIZE)
- out_buffer.size = _CSTREAM_OUT_SIZE
+ dst_buffer = ffi.new('char[]', write_size)
+ out_buffer.dst = dst_buffer
+ out_buffer.size = write_size
out_buffer.pos = 0
total_read, total_write = 0, 0
while True:
- data = ifh.read(_CSTREAM_IN_SIZE)
+ data = ifh.read(read_size)
if not data:
break
- total_read += len(data)
-
- in_buffer.src = ffi.new('char[]', data)
- in_buffer.size = len(data)
+ data_buffer = ffi.from_buffer(data)
+ total_read += len(data_buffer)
+ in_buffer.src = data_buffer
+ in_buffer.size = len(data_buffer)
in_buffer.pos = 0
while in_buffer.pos < in_buffer.size:
- res = lib.ZSTD_compressStream(cstream, out_buffer, in_buffer)
- if lib.ZSTD_isError(res):
- raise Exception('zstd compress error: %s' %
- lib.ZSTD_getErrorName(res))
+ zresult = lib.ZSTD_compressStream(cstream, out_buffer, in_buffer)
+ if lib.ZSTD_isError(zresult):
+ raise ZstdError('zstd compress error: %s' %
+ ffi.string(lib.ZSTD_getErrorName(zresult)))
if out_buffer.pos:
ofh.write(ffi.buffer(out_buffer.dst, out_buffer.pos))
- total_write = out_buffer.pos
+ total_write += out_buffer.pos
out_buffer.pos = 0
# We've finished reading. Flush the compressor.
while True:
- res = lib.ZSTD_endStream(cstream, out_buffer)
- if lib.ZSTD_isError(res):
- raise Exception('error ending compression stream: %s' %
- lib.ZSTD_getErrorName(res))
+ zresult = lib.ZSTD_endStream(cstream, out_buffer)
+ if lib.ZSTD_isError(zresult):
+ raise ZstdError('error ending compression stream: %s' %
+ ffi.string(lib.ZSTD_getErrorName(zresult)))
if out_buffer.pos:
ofh.write(ffi.buffer(out_buffer.dst, out_buffer.pos))
total_write += out_buffer.pos
out_buffer.pos = 0
- if res == 0:
+ if zresult == 0:
break
return total_read, total_write
- def write_to(self, writer):
- return _ZstdCompressionWriter(self._get_cstream(), writer)
+ def write_to(self, writer, size=0,
+ write_size=COMPRESSION_RECOMMENDED_OUTPUT_SIZE):
+
+ if not hasattr(writer, 'write'):
+ raise ValueError('must pass an object with a write() method')
+
+ return ZstdCompressionWriter(self, writer, size, write_size)
+
+ def read_from(self, reader, size=0,
+ read_size=COMPRESSION_RECOMMENDED_INPUT_SIZE,
+ write_size=COMPRESSION_RECOMMENDED_OUTPUT_SIZE):
+ if hasattr(reader, 'read'):
+ have_read = True
+ elif hasattr(reader, '__getitem__'):
+ have_read = False
+ buffer_offset = 0
+ size = len(reader)
+ else:
+ raise ValueError('must pass an object with a read() method or '
+ 'conforms to buffer protocol')
+
+ cstream = self._get_cstream(size)
+
+ in_buffer = ffi.new('ZSTD_inBuffer *')
+ out_buffer = ffi.new('ZSTD_outBuffer *')
+
+ in_buffer.src = ffi.NULL
+ in_buffer.size = 0
+ in_buffer.pos = 0
+
+ dst_buffer = ffi.new('char[]', write_size)
+ out_buffer.dst = dst_buffer
+ out_buffer.size = write_size
+ out_buffer.pos = 0
+
+ while True:
+ # We should never have output data sitting around after a previous
+ # iteration.
+ assert out_buffer.pos == 0
+
+ # Collect input data.
+ if have_read:
+ read_result = reader.read(read_size)
+ else:
+ remaining = len(reader) - buffer_offset
+ slice_size = min(remaining, read_size)
+ read_result = reader[buffer_offset:buffer_offset + slice_size]
+ buffer_offset += slice_size
- def _get_cstream(self):
+ # No new input data. Break out of the read loop.
+ if not read_result:
+ break
+
+ # Feed all read data into the compressor and emit output until
+ # exhausted.
+ read_buffer = ffi.from_buffer(read_result)
+ in_buffer.src = read_buffer
+ in_buffer.size = len(read_buffer)
+ in_buffer.pos = 0
+
+ while in_buffer.pos < in_buffer.size:
+ zresult = lib.ZSTD_compressStream(cstream, out_buffer, in_buffer)
+ if lib.ZSTD_isError(zresult):
+ raise ZstdError('zstd compress error: %s' %
+ ffi.string(lib.ZSTD_getErrorName(zresult)))
+
+ if out_buffer.pos:
+ data = ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
+ out_buffer.pos = 0
+ yield data
+
+ assert out_buffer.pos == 0
+
+ # And repeat the loop to collect more data.
+ continue
+
+ # If we get here, input is exhausted. End the stream and emit what
+ # remains.
+ while True:
+ assert out_buffer.pos == 0
+ zresult = lib.ZSTD_endStream(cstream, out_buffer)
+ if lib.ZSTD_isError(zresult):
+ raise ZstdError('error ending compression stream: %s' %
+ ffi.string(lib.ZSTD_getErrorName(zresult)))
+
+ if out_buffer.pos:
+ data = ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
+ out_buffer.pos = 0
+ yield data
+
+ if zresult == 0:
+ break
+
+ def _get_cstream(self, size):
cstream = lib.ZSTD_createCStream()
+ if cstream == ffi.NULL:
+ raise MemoryError()
+
cstream = ffi.gc(cstream, lib.ZSTD_freeCStream)
- res = lib.ZSTD_initCStream(cstream, self._compression_level)
- if lib.ZSTD_isError(res):
+ dict_data = ffi.NULL
+ dict_size = 0
+ if self._dict_data:
+ dict_data = self._dict_data.as_bytes()
+ dict_size = len(self._dict_data)
+
+ zparams = ffi.new('ZSTD_parameters *')[0]
+ if self._cparams:
+ zparams.cParams = self._cparams.as_compression_parameters()
+ else:
+ zparams.cParams = lib.ZSTD_getCParams(self._compression_level,
+ size, dict_size)
+ zparams.fParams = self._fparams
+
+ zresult = lib.ZSTD_initCStream_advanced(cstream, dict_data, dict_size,
+ zparams, size)
+ if lib.ZSTD_isError(zresult):
raise Exception('cannot init CStream: %s' %
- lib.ZSTD_getErrorName(res))
+ ffi.string(lib.ZSTD_getErrorName(zresult)))
return cstream
+
+
+class FrameParameters(object):
+ def __init__(self, fparams):
+ self.content_size = fparams.frameContentSize
+ self.window_size = fparams.windowSize
+ self.dict_id = fparams.dictID
+ self.has_checksum = bool(fparams.checksumFlag)
+
+
+def get_frame_parameters(data):
+ if not isinstance(data, bytes_type):
+ raise TypeError('argument must be bytes')
+
+ params = ffi.new('ZSTD_frameParams *')
+
+ zresult = lib.ZSTD_getFrameParams(params, data, len(data))
+ if lib.ZSTD_isError(zresult):
+ raise ZstdError('cannot get frame parameters: %s' %
+ ffi.string(lib.ZSTD_getErrorName(zresult)))
+
+ if zresult:
+ raise ZstdError('not enough data for frame parameters; need %d bytes' %
+ zresult)
+
+ return FrameParameters(params[0])
+
+
+class ZstdCompressionDict(object):
+ def __init__(self, data):
+ assert isinstance(data, bytes_type)
+ self._data = data
+
+ def __len__(self):
+ return len(self._data)
+
+ def dict_id(self):
+ return int_type(lib.ZDICT_getDictID(self._data, len(self._data)))
+
+ def as_bytes(self):
+ return self._data
+
+
+def train_dictionary(dict_size, samples, parameters=None):
+ if not isinstance(samples, list):
+ raise TypeError('samples must be a list')
+
+ total_size = sum(map(len, samples))
+
+ samples_buffer = new_nonzero('char[]', total_size)
+ sample_sizes = new_nonzero('size_t[]', len(samples))
+
+ offset = 0
+ for i, sample in enumerate(samples):
+ if not isinstance(sample, bytes_type):
+ raise ValueError('samples must be bytes')
+
+ l = len(sample)
+ ffi.memmove(samples_buffer + offset, sample, l)
+ offset += l
+ sample_sizes[i] = l
+
+ dict_data = new_nonzero('char[]', dict_size)
+
+ zresult = lib.ZDICT_trainFromBuffer(ffi.addressof(dict_data), dict_size,
+ ffi.addressof(samples_buffer),
+ ffi.addressof(sample_sizes, 0),
+ len(samples))
+ if lib.ZDICT_isError(zresult):
+ raise ZstdError('Cannot train dict: %s' %
+ ffi.string(lib.ZDICT_getErrorName(zresult)))
+
+ return ZstdCompressionDict(ffi.buffer(dict_data, zresult)[:])
+
+
+class ZstdDecompressionObj(object):
+ def __init__(self, decompressor):
+ self._decompressor = decompressor
+ self._dstream = self._decompressor._get_dstream()
+ self._finished = False
+
+ def decompress(self, data):
+ if self._finished:
+ raise ZstdError('cannot use a decompressobj multiple times')
+
+ in_buffer = ffi.new('ZSTD_inBuffer *')
+ out_buffer = ffi.new('ZSTD_outBuffer *')
+
+ data_buffer = ffi.from_buffer(data)
+ in_buffer.src = data_buffer
+ in_buffer.size = len(data_buffer)
+ in_buffer.pos = 0
+
+ dst_buffer = ffi.new('char[]', DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE)
+ out_buffer.dst = dst_buffer
+ out_buffer.size = len(dst_buffer)
+ out_buffer.pos = 0
+
+ chunks = []
+
+ while in_buffer.pos < in_buffer.size:
+ zresult = lib.ZSTD_decompressStream(self._dstream, out_buffer, in_buffer)
+ if lib.ZSTD_isError(zresult):
+ raise ZstdError('zstd decompressor error: %s' %
+ ffi.string(lib.ZSTD_getErrorName(zresult)))
+
+ if zresult == 0:
+ self._finished = True
+ self._dstream = None
+ self._decompressor = None
+
+ if out_buffer.pos:
+ chunks.append(ffi.buffer(out_buffer.dst, out_buffer.pos)[:])
+ out_buffer.pos = 0
+
+ return b''.join(chunks)
+
+
+class ZstdDecompressionWriter(object):
+ def __init__(self, decompressor, writer, write_size):
+ self._decompressor = decompressor
+ self._writer = writer
+ self._write_size = write_size
+ self._dstream = None
+ self._entered = False
+
+ def __enter__(self):
+ if self._entered:
+ raise ZstdError('cannot __enter__ multiple times')
+
+ self._dstream = self._decompressor._get_dstream()
+ self._entered = True
+
+ return self
+
+ def __exit__(self, exc_type, exc_value, exc_tb):
+ self._entered = False
+ self._dstream = None
+
+ def memory_size(self):
+ if not self._dstream:
+ raise ZstdError('cannot determine size of inactive decompressor '
+ 'call when context manager is active')
+
+ return lib.ZSTD_sizeof_DStream(self._dstream)
+
+ def write(self, data):
+ if not self._entered:
+ raise ZstdError('write must be called from an active context manager')
+
+ total_write = 0
+
+ in_buffer = ffi.new('ZSTD_inBuffer *')
+ out_buffer = ffi.new('ZSTD_outBuffer *')
+
+ data_buffer = ffi.from_buffer(data)
+ in_buffer.src = data_buffer
+ in_buffer.size = len(data_buffer)
+ in_buffer.pos = 0
+
+ dst_buffer = ffi.new('char[]', self._write_size)
+ out_buffer.dst = dst_buffer
+ out_buffer.size = len(dst_buffer)
+ out_buffer.pos = 0
+
+ while in_buffer.pos < in_buffer.size:
+ zresult = lib.ZSTD_decompressStream(self._dstream, out_buffer, in_buffer)
+ if lib.ZSTD_isError(zresult):
+ raise ZstdError('zstd decompress error: %s' %
+ ffi.string(lib.ZSTD_getErrorName(zresult)))
+
+ if out_buffer.pos:
+ self._writer.write(ffi.buffer(out_buffer.dst, out_buffer.pos)[:])
+ total_write += out_buffer.pos
+ out_buffer.pos = 0
+
+ return total_write
+
+
+class ZstdDecompressor(object):
+ def __init__(self, dict_data=None):
+ self._dict_data = dict_data
+
+ dctx = lib.ZSTD_createDCtx()
+ if dctx == ffi.NULL:
+ raise MemoryError()
+
+ self._refdctx = ffi.gc(dctx, lib.ZSTD_freeDCtx)
+
+ @property
+ def _ddict(self):
+ if self._dict_data:
+ dict_data = self._dict_data.as_bytes()
+ dict_size = len(self._dict_data)
+
+ ddict = lib.ZSTD_createDDict(dict_data, dict_size)
+ if ddict == ffi.NULL:
+ raise ZstdError('could not create decompression dict')
+ else:
+ ddict = None
+
+ self.__dict__['_ddict'] = ddict
+ return ddict
+
+ def decompress(self, data, max_output_size=0):
+ data_buffer = ffi.from_buffer(data)
+
+ orig_dctx = new_nonzero('char[]', lib.ZSTD_sizeof_DCtx(self._refdctx))
+ dctx = ffi.cast('ZSTD_DCtx *', orig_dctx)
+ lib.ZSTD_copyDCtx(dctx, self._refdctx)
+
+ ddict = self._ddict
+
+ output_size = lib.ZSTD_getDecompressedSize(data_buffer, len(data_buffer))
+ if output_size:
+ result_buffer = ffi.new('char[]', output_size)
+ result_size = output_size
+ else:
+ if not max_output_size:
+ raise ZstdError('input data invalid or missing content size '
+ 'in frame header')
+
+ result_buffer = ffi.new('char[]', max_output_size)
+ result_size = max_output_size
+
+ if ddict:
+ zresult = lib.ZSTD_decompress_usingDDict(dctx,
+ result_buffer, result_size,
+ data_buffer, len(data_buffer),
+ ddict)
+ else:
+ zresult = lib.ZSTD_decompressDCtx(dctx,
+ result_buffer, result_size,
+ data_buffer, len(data_buffer))
+ if lib.ZSTD_isError(zresult):
+ raise ZstdError('decompression error: %s' %
+ ffi.string(lib.ZSTD_getErrorName(zresult)))
+ elif output_size and zresult != output_size:
+ raise ZstdError('decompression error: decompressed %d bytes; expected %d' %
+ (zresult, output_size))
+
+ return ffi.buffer(result_buffer, zresult)[:]
+
+ def decompressobj(self):
+ return ZstdDecompressionObj(self)
+
+ def read_from(self, reader, read_size=DECOMPRESSION_RECOMMENDED_INPUT_SIZE,
+ write_size=DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE,
+ skip_bytes=0):
+ if skip_bytes >= read_size:
+ raise ValueError('skip_bytes must be smaller than read_size')
+
+ if hasattr(reader, 'read'):
+ have_read = True
+ elif hasattr(reader, '__getitem__'):
+ have_read = False
+ buffer_offset = 0
+ size = len(reader)
+ else:
+ raise ValueError('must pass an object with a read() method or '
+ 'conforms to buffer protocol')
+
+ if skip_bytes:
+ if have_read:
+ reader.read(skip_bytes)
+ else:
+ if skip_bytes > size:
+ raise ValueError('skip_bytes larger than first input chunk')
+
+ buffer_offset = skip_bytes
+
+ dstream = self._get_dstream()
+
+ in_buffer = ffi.new('ZSTD_inBuffer *')
+ out_buffer = ffi.new('ZSTD_outBuffer *')
+
+ dst_buffer = ffi.new('char[]', write_size)
+ out_buffer.dst = dst_buffer
+ out_buffer.size = len(dst_buffer)
+ out_buffer.pos = 0
+
+ while True:
+ assert out_buffer.pos == 0
+
+ if have_read:
+ read_result = reader.read(read_size)
+ else:
+ remaining = size - buffer_offset
+ slice_size = min(remaining, read_size)
+ read_result = reader[buffer_offset:buffer_offset + slice_size]
+ buffer_offset += slice_size
+
+ # No new input. Break out of read loop.
+ if not read_result:
+ break
+
+ # Feed all read data into decompressor and emit output until
+ # exhausted.
+ read_buffer = ffi.from_buffer(read_result)
+ in_buffer.src = read_buffer
+ in_buffer.size = len(read_buffer)
+ in_buffer.pos = 0
+
+ while in_buffer.pos < in_buffer.size:
+ assert out_buffer.pos == 0
+
+ zresult = lib.ZSTD_decompressStream(dstream, out_buffer, in_buffer)
+ if lib.ZSTD_isError(zresult):
+ raise ZstdError('zstd decompress error: %s' %
+ ffi.string(lib.ZSTD_getErrorName(zresult)))
+
+ if out_buffer.pos:
+ data = ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
+ out_buffer.pos = 0
+ yield data
+
+ if zresult == 0:
+ return
+
+ # Repeat loop to collect more input data.
+ continue
+
+ # If we get here, input is exhausted.
+
+ def write_to(self, writer, write_size=DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE):
+ if not hasattr(writer, 'write'):
+ raise ValueError('must pass an object with a write() method')
+
+ return ZstdDecompressionWriter(self, writer, write_size)
+
+ def copy_stream(self, ifh, ofh,
+ read_size=DECOMPRESSION_RECOMMENDED_INPUT_SIZE,
+ write_size=DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE):
+ if not hasattr(ifh, 'read'):
+ raise ValueError('first argument must have a read() method')
+ if not hasattr(ofh, 'write'):
+ raise ValueError('second argument must have a write() method')
+
+ dstream = self._get_dstream()
+
+ in_buffer = ffi.new('ZSTD_inBuffer *')
+ out_buffer = ffi.new('ZSTD_outBuffer *')
+
+ dst_buffer = ffi.new('char[]', write_size)
+ out_buffer.dst = dst_buffer
+ out_buffer.size = write_size
+ out_buffer.pos = 0
+
+ total_read, total_write = 0, 0
+
+ # Read all available input.
+ while True:
+ data = ifh.read(read_size)
+ if not data:
+ break
+
+ data_buffer = ffi.from_buffer(data)
+ total_read += len(data_buffer)
+ in_buffer.src = data_buffer
+ in_buffer.size = len(data_buffer)
+ in_buffer.pos = 0
+
+ # Flush all read data to output.
+ while in_buffer.pos < in_buffer.size:
+ zresult = lib.ZSTD_decompressStream(dstream, out_buffer, in_buffer)
+ if lib.ZSTD_isError(zresult):
+ raise ZstdError('zstd decompressor error: %s' %
+ ffi.string(lib.ZSTD_getErrorName(zresult)))
+
+ if out_buffer.pos:
+ ofh.write(ffi.buffer(out_buffer.dst, out_buffer.pos))
+ total_write += out_buffer.pos
+ out_buffer.pos = 0
+
+ # Continue loop to keep reading.
+
+ return total_read, total_write
+
+ def decompress_content_dict_chain(self, frames):
+ if not isinstance(frames, list):
+ raise TypeError('argument must be a list')
+
+ if not frames:
+ raise ValueError('empty input chain')
+
+ # First chunk should not be using a dictionary. We handle it specially.
+ chunk = frames[0]
+ if not isinstance(chunk, bytes_type):
+ raise ValueError('chunk 0 must be bytes')
+
+ # All chunks should be zstd frames and should have content size set.
+ chunk_buffer = ffi.from_buffer(chunk)
+ params = ffi.new('ZSTD_frameParams *')
+ zresult = lib.ZSTD_getFrameParams(params, chunk_buffer, len(chunk_buffer))
+ if lib.ZSTD_isError(zresult):
+ raise ValueError('chunk 0 is not a valid zstd frame')
+ elif zresult:
+ raise ValueError('chunk 0 is too small to contain a zstd frame')
+
+ if not params.frameContentSize:
+ raise ValueError('chunk 0 missing content size in frame')
+
+ dctx = lib.ZSTD_createDCtx()
+ if dctx == ffi.NULL:
+ raise MemoryError()
+
+ dctx = ffi.gc(dctx, lib.ZSTD_freeDCtx)
+
+ last_buffer = ffi.new('char[]', params.frameContentSize)
+
+ zresult = lib.ZSTD_decompressDCtx(dctx, last_buffer, len(last_buffer),
+ chunk_buffer, len(chunk_buffer))
+ if lib.ZSTD_isError(zresult):
+ raise ZstdError('could not decompress chunk 0: %s' %
+ ffi.string(lib.ZSTD_getErrorName(zresult)))
+
+ # Special case of chain length of 1
+ if len(frames) == 1:
+ return ffi.buffer(last_buffer, len(last_buffer))[:]
+
+ i = 1
+ while i < len(frames):
+ chunk = frames[i]
+ if not isinstance(chunk, bytes_type):
+ raise ValueError('chunk %d must be bytes' % i)
+
+ chunk_buffer = ffi.from_buffer(chunk)
+ zresult = lib.ZSTD_getFrameParams(params, chunk_buffer, len(chunk_buffer))
+ if lib.ZSTD_isError(zresult):
+ raise ValueError('chunk %d is not a valid zstd frame' % i)
+ elif zresult:
+ raise ValueError('chunk %d is too small to contain a zstd frame' % i)
+
+ if not params.frameContentSize:
+ raise ValueError('chunk %d missing content size in frame' % i)
+
+ dest_buffer = ffi.new('char[]', params.frameContentSize)
+
+ zresult = lib.ZSTD_decompress_usingDict(dctx, dest_buffer, len(dest_buffer),
+ chunk_buffer, len(chunk_buffer),
+ last_buffer, len(last_buffer))
+ if lib.ZSTD_isError(zresult):
+ raise ZstdError('could not decompress chunk %d' % i)
+
+ last_buffer = dest_buffer
+ i += 1
+
+ return ffi.buffer(last_buffer, len(last_buffer))[:]
+
+ def _get_dstream(self):
+ dstream = lib.ZSTD_createDStream()
+ if dstream == ffi.NULL:
+ raise MemoryError()
+
+ dstream = ffi.gc(dstream, lib.ZSTD_freeDStream)
+
+ if self._dict_data:
+ zresult = lib.ZSTD_initDStream_usingDict(dstream,
+ self._dict_data.as_bytes(),
+ len(self._dict_data))
+ else:
+ zresult = lib.ZSTD_initDStream(dstream)
+
+ if lib.ZSTD_isError(zresult):
+ raise ZstdError('could not initialize DStream: %s' %
+ ffi.string(lib.ZSTD_getErrorName(zresult)))
+
+ return dstream
--- a/contrib/wix/help.wxs Sat Feb 25 12:48:50 2017 +0900
+++ b/contrib/wix/help.wxs Tue Feb 28 11:13:25 2017 -0800
@@ -25,6 +25,7 @@
<File Name="hgignore.txt" />
<File Name="hgweb.txt" />
<File Name="merge-tools.txt" />
+ <File Name="pager.txt" />
<File Name="patterns.txt" />
<File Name="phases.txt" />
<File Name="revisions.txt" />
--- a/hgext/bugzilla.py Sat Feb 25 12:48:50 2017 +0900
+++ b/hgext/bugzilla.py Tue Feb 28 11:13:25 2017 -0800
@@ -15,14 +15,16 @@
The bug references can optionally include an update for Bugzilla of the
hours spent working on the bug. Bugs can also be marked fixed.
-Three basic modes of access to Bugzilla are provided:
+Four basic modes of access to Bugzilla are provided:
+
+1. Access via the Bugzilla REST-API. Requires bugzilla 5.0 or later.
-1. Access via the Bugzilla XMLRPC interface. Requires Bugzilla 3.4 or later.
+2. Access via the Bugzilla XMLRPC interface. Requires Bugzilla 3.4 or later.
-2. Check data via the Bugzilla XMLRPC interface and submit bug change
+3. Check data via the Bugzilla XMLRPC interface and submit bug change
via email to Bugzilla email interface. Requires Bugzilla 3.4 or later.
-3. Writing directly to the Bugzilla database. Only Bugzilla installations
+4. Writing directly to the Bugzilla database. Only Bugzilla installations
using MySQL are supported. Requires Python MySQLdb.
Writing directly to the database is susceptible to schema changes, and
@@ -50,11 +52,16 @@
Bugzilla is used instead as the source of the comment. Marking bugs fixed
works on all supported Bugzilla versions.
+Access via the REST-API needs either a Bugzilla username and password
+or an apikey specified in the configuration. Comments are made under
+the given username or the user assoicated with the apikey in Bugzilla.
+
Configuration items common to all access modes:
bugzilla.version
The access type to use. Values recognized are:
+ :``restapi``: Bugzilla REST-API, Bugzilla 5.0 and later.
:``xmlrpc``: Bugzilla XMLRPC interface.
:``xmlrpc+email``: Bugzilla XMLRPC and email interfaces.
:``3.0``: MySQL access, Bugzilla 3.0 and later.
@@ -135,7 +142,7 @@
committer email to Bugzilla user email. See also ``bugzilla.usermap``.
Contains entries of the form ``committer = Bugzilla user``.
-XMLRPC access mode configuration:
+XMLRPC and REST-API access mode configuration:
bugzilla.bzurl
The base URL for the Bugzilla installation.
@@ -148,6 +155,13 @@
bugzilla.password
The password for Bugzilla login.
+REST-API access mode uses the options listed above as well as:
+
+bugzilla.apikey
+ An apikey generated on the Bugzilla instance for api access.
+ Using an apikey removes the need to store the user and password
+ options.
+
XMLRPC+email access mode uses the XMLRPC access mode configuration items,
and also:
@@ -279,6 +293,7 @@
from __future__ import absolute_import
+import json
import re
import time
@@ -288,6 +303,7 @@
cmdutil,
error,
mail,
+ url,
util,
)
@@ -773,6 +789,136 @@
cmds.append(self.makecommandline("resolution", self.fixresolution))
self.send_bug_modify_email(bugid, cmds, text, committer)
+class NotFound(LookupError):
+ pass
+
+class bzrestapi(bzaccess):
+ """Read and write bugzilla data using the REST API available since
+ Bugzilla 5.0.
+ """
+ def __init__(self, ui):
+ bzaccess.__init__(self, ui)
+ bz = self.ui.config('bugzilla', 'bzurl',
+ 'http://localhost/bugzilla/')
+ self.bzroot = '/'.join([bz, 'rest'])
+ self.apikey = self.ui.config('bugzilla', 'apikey', '')
+ self.user = self.ui.config('bugzilla', 'user', 'bugs')
+ self.passwd = self.ui.config('bugzilla', 'password')
+ self.fixstatus = self.ui.config('bugzilla', 'fixstatus', 'RESOLVED')
+ self.fixresolution = self.ui.config('bugzilla', 'fixresolution',
+ 'FIXED')
+
+ def apiurl(self, targets, include_fields=None):
+ url = '/'.join([self.bzroot] + [str(t) for t in targets])
+ qv = {}
+ if self.apikey:
+ qv['api_key'] = self.apikey
+ elif self.user and self.passwd:
+ qv['login'] = self.user
+ qv['password'] = self.passwd
+ if include_fields:
+ qv['include_fields'] = include_fields
+ if qv:
+ url = '%s?%s' % (url, util.urlreq.urlencode(qv))
+ return url
+
+ def _fetch(self, burl):
+ try:
+ resp = url.open(self.ui, burl)
+ return json.loads(resp.read())
+ except util.urlerr.httperror as inst:
+ if inst.code == 401:
+ raise error.Abort(_('authorization failed'))
+ if inst.code == 404:
+ raise NotFound()
+ else:
+ raise
+
+ def _submit(self, burl, data, method='POST'):
+ data = json.dumps(data)
+ if method == 'PUT':
+ class putrequest(util.urlreq.request):
+ def get_method(self):
+ return 'PUT'
+ request_type = putrequest
+ else:
+ request_type = util.urlreq.request
+ req = request_type(burl, data,
+ {'Content-Type': 'application/json'})
+ try:
+ resp = url.opener(self.ui).open(req)
+ return json.loads(resp.read())
+ except util.urlerr.httperror as inst:
+ if inst.code == 401:
+ raise error.Abort(_('authorization failed'))
+ if inst.code == 404:
+ raise NotFound()
+ else:
+ raise
+
+ def filter_real_bug_ids(self, bugs):
+ '''remove bug IDs that do not exist in Bugzilla from bugs.'''
+ badbugs = set()
+ for bugid in bugs:
+ burl = self.apiurl(('bug', bugid), include_fields='status')
+ try:
+ self._fetch(burl)
+ except NotFound:
+ badbugs.add(bugid)
+ for bugid in badbugs:
+ del bugs[bugid]
+
+ def filter_cset_known_bug_ids(self, node, bugs):
+ '''remove bug IDs where node occurs in comment text from bugs.'''
+ sn = short(node)
+ for bugid in bugs.keys():
+ burl = self.apiurl(('bug', bugid, 'comment'), include_fields='text')
+ result = self._fetch(burl)
+ comments = result['bugs'][str(bugid)]['comments']
+ if any(sn in c['text'] for c in comments):
+ self.ui.status(_('bug %d already knows about changeset %s\n') %
+ (bugid, sn))
+ del bugs[bugid]
+
+ def updatebug(self, bugid, newstate, text, committer):
+ '''update the specified bug. Add comment text and set new states.
+
+ If possible add the comment as being from the committer of
+ the changeset. Otherwise use the default Bugzilla user.
+ '''
+ bugmod = {}
+ if 'hours' in newstate:
+ bugmod['work_time'] = newstate['hours']
+ if 'fix' in newstate:
+ bugmod['status'] = self.fixstatus
+ bugmod['resolution'] = self.fixresolution
+ if bugmod:
+ # if we have to change the bugs state do it here
+ bugmod['comment'] = {
+ 'comment': text,
+ 'is_private': False,
+ 'is_markdown': False,
+ }
+ burl = self.apiurl(('bug', bugid))
+ self._submit(burl, bugmod, method='PUT')
+ self.ui.debug('updated bug %s\n' % bugid)
+ else:
+ burl = self.apiurl(('bug', bugid, 'comment'))
+ self._submit(burl, {
+ 'comment': text,
+ 'is_private': False,
+ 'is_markdown': False,
+ })
+ self.ui.debug('added comment to bug %s\n' % bugid)
+
+ def notify(self, bugs, committer):
+ '''Force sending of Bugzilla notification emails.
+
+ Only required if the access method does not trigger notification
+ emails automatically.
+ '''
+ pass
+
class bugzilla(object):
# supported versions of bugzilla. different versions have
# different schemas.
@@ -781,7 +927,8 @@
'2.18': bzmysql_2_18,
'3.0': bzmysql_3_0,
'xmlrpc': bzxmlrpc,
- 'xmlrpc+email': bzxmlrpcemail
+ 'xmlrpc+email': bzxmlrpcemail,
+ 'restapi': bzrestapi,
}
_default_bug_re = (r'bugs?\s*,?\s*(?:#|nos?\.?|num(?:ber)?s?)?\s*'
--- a/hgext/color.py Sat Feb 25 12:48:50 2017 +0900
+++ b/hgext/color.py Tue Feb 28 11:13:25 2017 -0800
@@ -164,18 +164,17 @@
from __future__ import absolute_import
+try:
+ import curses
+ curses.COLOR_BLACK # force import
+except ImportError:
+ curses = None
+
from mercurial.i18n import _
from mercurial import (
cmdutil,
color,
commands,
- dispatch,
- encoding,
- extensions,
- pycompat,
- subrepo,
- ui as uimod,
- util,
)
cmdtable = {}
@@ -186,294 +185,15 @@
# leave the attribute unspecified.
testedwith = 'ships-with-hg-core'
-# start and stop parameters for effects
-_effects = {'none': 0, 'black': 30, 'red': 31, 'green': 32, 'yellow': 33,
- 'blue': 34, 'magenta': 35, 'cyan': 36, 'white': 37, 'bold': 1,
- 'italic': 3, 'underline': 4, 'inverse': 7, 'dim': 2,
- 'black_background': 40, 'red_background': 41,
- 'green_background': 42, 'yellow_background': 43,
- 'blue_background': 44, 'purple_background': 45,
- 'cyan_background': 46, 'white_background': 47}
-
-def _terminfosetup(ui, mode):
- '''Initialize terminfo data and the terminal if we're in terminfo mode.'''
-
- # If we failed to load curses, we go ahead and return.
- if not _terminfo_params:
- return
- # Otherwise, see what the config file says.
- if mode not in ('auto', 'terminfo'):
- return
-
- _terminfo_params.update((key[6:], (False, int(val), ''))
- for key, val in ui.configitems('color')
- if key.startswith('color.'))
- _terminfo_params.update((key[9:], (True, '', val.replace('\\E', '\x1b')))
- for key, val in ui.configitems('color')
- if key.startswith('terminfo.'))
-
- try:
- curses.setupterm()
- except curses.error as e:
- _terminfo_params.clear()
- return
-
- for key, (b, e, c) in _terminfo_params.items():
- if not b:
- continue
- if not c and not curses.tigetstr(e):
- # Most terminals don't support dim, invis, etc, so don't be
- # noisy and use ui.debug().
- ui.debug("no terminfo entry for %s\n" % e)
- del _terminfo_params[key]
- if not curses.tigetstr('setaf') or not curses.tigetstr('setab'):
- # Only warn about missing terminfo entries if we explicitly asked for
- # terminfo mode.
- if mode == "terminfo":
- ui.warn(_("no terminfo entry for setab/setaf: reverting to "
- "ECMA-48 color\n"))
- _terminfo_params.clear()
-
-def _modesetup(ui, coloropt):
- if coloropt == 'debug':
- return 'debug'
-
- auto = (coloropt == 'auto')
- always = not auto and util.parsebool(coloropt)
- if not always and not auto:
- return None
-
- formatted = (always or (encoding.environ.get('TERM') != 'dumb'
- and ui.formatted()))
-
- mode = ui.config('color', 'mode', 'auto')
-
- # If pager is active, color.pagermode overrides color.mode.
- if getattr(ui, 'pageractive', False):
- mode = ui.config('color', 'pagermode', mode)
-
- realmode = mode
- if mode == 'auto':
- if pycompat.osname == 'nt':
- term = encoding.environ.get('TERM')
- # TERM won't be defined in a vanilla cmd.exe environment.
-
- # UNIX-like environments on Windows such as Cygwin and MSYS will
- # set TERM. They appear to make a best effort attempt at setting it
- # to something appropriate. However, not all environments with TERM
- # defined support ANSI. Since "ansi" could result in terminal
- # gibberish, we error on the side of selecting "win32". However, if
- # w32effects is not defined, we almost certainly don't support
- # "win32", so don't even try.
- if (term and 'xterm' in term) or not w32effects:
- realmode = 'ansi'
- else:
- realmode = 'win32'
- else:
- realmode = 'ansi'
-
- def modewarn():
- # only warn if color.mode was explicitly set and we're in
- # a formatted terminal
- if mode == realmode and ui.formatted():
- ui.warn(_('warning: failed to set color mode to %s\n') % mode)
-
- if realmode == 'win32':
- _terminfo_params.clear()
- if not w32effects:
- modewarn()
- return None
- _effects.update(w32effects)
- elif realmode == 'ansi':
- _terminfo_params.clear()
- elif realmode == 'terminfo':
- _terminfosetup(ui, mode)
- if not _terminfo_params:
- ## FIXME Shouldn't we return None in this case too?
- modewarn()
- realmode = 'ansi'
- else:
- return None
-
- if always or (auto and formatted):
- return realmode
- return None
-
-try:
- import curses
- # Mapping from effect name to terminfo attribute name (or raw code) or
- # color number. This will also force-load the curses module.
- _terminfo_params = {'none': (True, 'sgr0', ''),
- 'standout': (True, 'smso', ''),
- 'underline': (True, 'smul', ''),
- 'reverse': (True, 'rev', ''),
- 'inverse': (True, 'rev', ''),
- 'blink': (True, 'blink', ''),
- 'dim': (True, 'dim', ''),
- 'bold': (True, 'bold', ''),
- 'invisible': (True, 'invis', ''),
- 'italic': (True, 'sitm', ''),
- 'black': (False, curses.COLOR_BLACK, ''),
- 'red': (False, curses.COLOR_RED, ''),
- 'green': (False, curses.COLOR_GREEN, ''),
- 'yellow': (False, curses.COLOR_YELLOW, ''),
- 'blue': (False, curses.COLOR_BLUE, ''),
- 'magenta': (False, curses.COLOR_MAGENTA, ''),
- 'cyan': (False, curses.COLOR_CYAN, ''),
- 'white': (False, curses.COLOR_WHITE, '')}
-except ImportError:
- _terminfo_params = {}
-
-def _effect_str(effect):
- '''Helper function for render_effects().'''
-
- bg = False
- if effect.endswith('_background'):
- bg = True
- effect = effect[:-11]
- try:
- attr, val, termcode = _terminfo_params[effect]
- except KeyError:
- return ''
- if attr:
- if termcode:
- return termcode
- else:
- return curses.tigetstr(val)
- elif bg:
- return curses.tparm(curses.tigetstr('setab'), val)
- else:
- return curses.tparm(curses.tigetstr('setaf'), val)
-
-def render_effects(text, effects):
- 'Wrap text in commands to turn on each effect.'
- if not text:
- return text
- if not _terminfo_params:
- start = [str(_effects[e]) for e in ['none'] + effects.split()]
- start = '\033[' + ';'.join(start) + 'm'
- stop = '\033[' + str(_effects['none']) + 'm'
- else:
- start = ''.join(_effect_str(effect)
- for effect in ['none'] + effects.split())
- stop = _effect_str('none')
- return ''.join([start, text, stop])
-
-def valideffect(effect):
- 'Determine if the effect is valid or not.'
- good = False
- if not _terminfo_params and effect in _effects:
- good = True
- elif effect in _terminfo_params or effect[:-11] in _terminfo_params:
- good = True
- return good
-
-def configstyles(ui):
- for status, cfgeffects in ui.configitems('color'):
- if '.' not in status or status.startswith(('color.', 'terminfo.')):
- continue
- cfgeffects = ui.configlist('color', status)
- if cfgeffects:
- good = []
- for e in cfgeffects:
- if valideffect(e):
- good.append(e)
- else:
- ui.warn(_("ignoring unknown color/effect %r "
- "(configured in color.%s)\n")
- % (e, status))
- color._styles[status] = ' '.join(good)
-
-class colorui(uimod.ui):
- _colormode = 'ansi'
- def write(self, *args, **opts):
- if self._colormode is None:
- return super(colorui, self).write(*args, **opts)
-
- label = opts.get('label', '')
- if self._buffers and not opts.get('prompt', False):
- if self._bufferapplylabels:
- self._buffers[-1].extend(self.label(a, label) for a in args)
- else:
- self._buffers[-1].extend(args)
- elif self._colormode == 'win32':
- for a in args:
- win32print(a, super(colorui, self).write, **opts)
- else:
- return super(colorui, self).write(
- *[self.label(a, label) for a in args], **opts)
-
- def write_err(self, *args, **opts):
- if self._colormode is None:
- return super(colorui, self).write_err(*args, **opts)
-
- label = opts.get('label', '')
- if self._bufferstates and self._bufferstates[-1][0]:
- return self.write(*args, **opts)
- if self._colormode == 'win32':
- for a in args:
- win32print(a, super(colorui, self).write_err, **opts)
- else:
- return super(colorui, self).write_err(
- *[self.label(a, label) for a in args], **opts)
-
- def showlabel(self, msg, label):
- if label and msg:
- if msg[-1] == '\n':
- return "[%s|%s]\n" % (label, msg[:-1])
- else:
- return "[%s|%s]" % (label, msg)
- else:
- return msg
-
- def label(self, msg, label):
- if self._colormode is None:
- return super(colorui, self).label(msg, label)
-
- if self._colormode == 'debug':
- return self.showlabel(msg, label)
-
- effects = []
- for l in label.split():
- s = color._styles.get(l, '')
- if s:
- effects.append(s)
- elif valideffect(l):
- effects.append(l)
- effects = ' '.join(effects)
- if effects:
- return '\n'.join([render_effects(line, effects)
- for line in msg.split('\n')])
- return msg
-
-def uisetup(ui):
- if ui.plain():
- return
- if not isinstance(ui, colorui):
- colorui.__bases__ = (ui.__class__,)
- ui.__class__ = colorui
- def colorcmd(orig, ui_, opts, cmd, cmdfunc):
- mode = _modesetup(ui_, opts['color'])
- colorui._colormode = mode
- if mode and mode != 'debug':
- configstyles(ui_)
- return orig(ui_, opts, cmd, cmdfunc)
- def colorgit(orig, gitsub, commands, env=None, stream=False, cwd=None):
- if gitsub.ui._colormode and len(commands) and commands[0] == "diff":
- # insert the argument in the front,
- # the end of git diff arguments is used for paths
- commands.insert(1, '--color')
- return orig(gitsub, commands, env, stream, cwd)
- extensions.wrapfunction(dispatch, '_runcommand', colorcmd)
- extensions.wrapfunction(subrepo.gitsubrepo, '_gitnodir', colorgit)
-
def extsetup(ui):
- commands.globalopts.append(
- ('', 'color', 'auto',
- # i18n: 'always', 'auto', 'never', and 'debug' are keywords
- # and should not be translated
- _("when to colorize (boolean, always, auto, never, or debug)"),
- _('TYPE')))
+ # change default color config
+ color._enabledbydefault = True
+ for idx, entry in enumerate(commands.globalopts):
+ if entry[1] == 'color':
+ patch = (entry[3].replace(' (EXPERIMENTAL)', ''),)
+ new = entry[:3] + patch + entry[4:]
+ commands.globalopts[idx] = new
+ break
@command('debugcolor',
[('', 'style', None, _('show all configured styles'))],
@@ -487,31 +207,31 @@
return _debugdisplaycolor(ui)
def _debugdisplaycolor(ui):
- oldstyle = color._styles.copy()
+ oldstyle = ui._styles.copy()
try:
- color._styles.clear()
- for effect in _effects.keys():
- color._styles[effect] = effect
- if _terminfo_params:
+ ui._styles.clear()
+ for effect in color._effects.keys():
+ ui._styles[effect] = effect
+ if ui._terminfoparams:
for k, v in ui.configitems('color'):
if k.startswith('color.'):
- color._styles[k] = k[6:]
+ ui._styles[k] = k[6:]
elif k.startswith('terminfo.'):
- color._styles[k] = k[9:]
+ ui._styles[k] = k[9:]
ui.write(_('available colors:\n'))
# sort label with a '_' after the other to group '_background' entry.
- items = sorted(color._styles.items(),
+ items = sorted(ui._styles.items(),
key=lambda i: ('_' in i[0], i[0], i[1]))
for colorname, label in items:
ui.write(('%s\n') % colorname, label=label)
finally:
- color._styles.clear()
- color._styles.update(oldstyle)
+ ui._styles.clear()
+ ui._styles.update(oldstyle)
def _debugdisplaystyle(ui):
ui.write(_('available style:\n'))
- width = max(len(s) for s in color._styles)
- for label, effects in sorted(color._styles.items()):
+ width = max(len(s) for s in ui._styles)
+ for label, effects in sorted(ui._styles.items()):
ui.write('%s' % label, label=label)
if effects:
# 50
@@ -519,138 +239,3 @@
ui.write(' ' * (max(0, width - len(label))))
ui.write(', '.join(ui.label(e, e) for e in effects.split()))
ui.write('\n')
-
-if pycompat.osname != 'nt':
- w32effects = None
-else:
- import ctypes
- import re
-
- _kernel32 = ctypes.windll.kernel32
-
- _WORD = ctypes.c_ushort
-
- _INVALID_HANDLE_VALUE = -1
-
- class _COORD(ctypes.Structure):
- _fields_ = [('X', ctypes.c_short),
- ('Y', ctypes.c_short)]
-
- class _SMALL_RECT(ctypes.Structure):
- _fields_ = [('Left', ctypes.c_short),
- ('Top', ctypes.c_short),
- ('Right', ctypes.c_short),
- ('Bottom', ctypes.c_short)]
-
- class _CONSOLE_SCREEN_BUFFER_INFO(ctypes.Structure):
- _fields_ = [('dwSize', _COORD),
- ('dwCursorPosition', _COORD),
- ('wAttributes', _WORD),
- ('srWindow', _SMALL_RECT),
- ('dwMaximumWindowSize', _COORD)]
-
- _STD_OUTPUT_HANDLE = 0xfffffff5 # (DWORD)-11
- _STD_ERROR_HANDLE = 0xfffffff4 # (DWORD)-12
-
- _FOREGROUND_BLUE = 0x0001
- _FOREGROUND_GREEN = 0x0002
- _FOREGROUND_RED = 0x0004
- _FOREGROUND_INTENSITY = 0x0008
-
- _BACKGROUND_BLUE = 0x0010
- _BACKGROUND_GREEN = 0x0020
- _BACKGROUND_RED = 0x0040
- _BACKGROUND_INTENSITY = 0x0080
-
- _COMMON_LVB_REVERSE_VIDEO = 0x4000
- _COMMON_LVB_UNDERSCORE = 0x8000
-
- # http://msdn.microsoft.com/en-us/library/ms682088%28VS.85%29.aspx
- w32effects = {
- 'none': -1,
- 'black': 0,
- 'red': _FOREGROUND_RED,
- 'green': _FOREGROUND_GREEN,
- 'yellow': _FOREGROUND_RED | _FOREGROUND_GREEN,
- 'blue': _FOREGROUND_BLUE,
- 'magenta': _FOREGROUND_BLUE | _FOREGROUND_RED,
- 'cyan': _FOREGROUND_BLUE | _FOREGROUND_GREEN,
- 'white': _FOREGROUND_RED | _FOREGROUND_GREEN | _FOREGROUND_BLUE,
- 'bold': _FOREGROUND_INTENSITY,
- 'black_background': 0x100, # unused value > 0x0f
- 'red_background': _BACKGROUND_RED,
- 'green_background': _BACKGROUND_GREEN,
- 'yellow_background': _BACKGROUND_RED | _BACKGROUND_GREEN,
- 'blue_background': _BACKGROUND_BLUE,
- 'purple_background': _BACKGROUND_BLUE | _BACKGROUND_RED,
- 'cyan_background': _BACKGROUND_BLUE | _BACKGROUND_GREEN,
- 'white_background': (_BACKGROUND_RED | _BACKGROUND_GREEN |
- _BACKGROUND_BLUE),
- 'bold_background': _BACKGROUND_INTENSITY,
- 'underline': _COMMON_LVB_UNDERSCORE, # double-byte charsets only
- 'inverse': _COMMON_LVB_REVERSE_VIDEO, # double-byte charsets only
- }
-
- passthrough = set([_FOREGROUND_INTENSITY,
- _BACKGROUND_INTENSITY,
- _COMMON_LVB_UNDERSCORE,
- _COMMON_LVB_REVERSE_VIDEO])
-
- stdout = _kernel32.GetStdHandle(
- _STD_OUTPUT_HANDLE) # don't close the handle returned
- if stdout is None or stdout == _INVALID_HANDLE_VALUE:
- w32effects = None
- else:
- csbi = _CONSOLE_SCREEN_BUFFER_INFO()
- if not _kernel32.GetConsoleScreenBufferInfo(
- stdout, ctypes.byref(csbi)):
- # stdout may not support GetConsoleScreenBufferInfo()
- # when called from subprocess or redirected
- w32effects = None
- else:
- origattr = csbi.wAttributes
- ansire = re.compile('\033\[([^m]*)m([^\033]*)(.*)',
- re.MULTILINE | re.DOTALL)
-
- def win32print(text, orig, **opts):
- label = opts.get('label', '')
- attr = origattr
-
- def mapcolor(val, attr):
- if val == -1:
- return origattr
- elif val in passthrough:
- return attr | val
- elif val > 0x0f:
- return (val & 0x70) | (attr & 0x8f)
- else:
- return (val & 0x07) | (attr & 0xf8)
-
- # determine console attributes based on labels
- for l in label.split():
- style = color._styles.get(l, '')
- for effect in style.split():
- try:
- attr = mapcolor(w32effects[effect], attr)
- except KeyError:
- # w32effects could not have certain attributes so we skip
- # them if not found
- pass
- # hack to ensure regexp finds data
- if not text.startswith('\033['):
- text = '\033[m' + text
-
- # Look for ANSI-like codes embedded in text
- m = re.match(ansire, text)
-
- try:
- while m:
- for sattr in m.group(1).split(';'):
- if sattr:
- attr = mapcolor(int(sattr), attr)
- _kernel32.SetConsoleTextAttribute(stdout, attr)
- orig(m.group(2), **opts)
- m = re.match(ansire, m.group(3))
- finally:
- # Explicitly reset original attributes
- _kernel32.SetConsoleTextAttribute(stdout, origattr)
--- a/hgext/convert/subversion.py Sat Feb 25 12:48:50 2017 +0900
+++ b/hgext/convert/subversion.py Tue Feb 28 11:13:25 2017 -0800
@@ -1306,7 +1306,7 @@
self.setexec = []
fd, messagefile = tempfile.mkstemp(prefix='hg-convert-')
- fp = os.fdopen(fd, 'w')
+ fp = os.fdopen(fd, pycompat.sysstr('w'))
fp.write(commit.desc)
fp.close()
try:
--- a/hgext/extdiff.py Sat Feb 25 12:48:50 2017 +0900
+++ b/hgext/extdiff.py Tue Feb 28 11:13:25 2017 -0800
@@ -273,7 +273,7 @@
cmdline = re.sub(regex, quote, cmdline)
ui.debug('running %r in %s\n' % (cmdline, tmproot))
- ui.system(cmdline, cwd=tmproot)
+ ui.system(cmdline, cwd=tmproot, blockedtag='extdiff')
for copy_fn, working_fn, mtime in fns_and_mtime:
if os.lstat(copy_fn).st_mtime != mtime:
--- a/hgext/gpg.py Sat Feb 25 12:48:50 2017 +0900
+++ b/hgext/gpg.py Tue Feb 28 11:13:25 2017 -0800
@@ -18,6 +18,7 @@
error,
match,
node as hgnode,
+ pycompat,
util,
)
@@ -44,11 +45,11 @@
try:
# create temporary files
fd, sigfile = tempfile.mkstemp(prefix="hg-gpg-", suffix=".sig")
- fp = os.fdopen(fd, 'wb')
+ fp = os.fdopen(fd, pycompat.sysstr('wb'))
fp.write(sig)
fp.close()
fd, datafile = tempfile.mkstemp(prefix="hg-gpg-", suffix=".txt")
- fp = os.fdopen(fd, 'wb')
+ fp = os.fdopen(fd, pycompat.sysstr('wb'))
fp.write(data)
fp.close()
gpgcmd = ("%s --logger-fd 1 --status-fd 1 --verify "
--- a/hgext/histedit.py Sat Feb 25 12:48:50 2017 +0900
+++ b/hgext/histedit.py Tue Feb 28 11:13:25 2017 -0800
@@ -36,7 +36,7 @@
# p, pick = use commit
# e, edit = use commit, but stop for amending
# f, fold = use commit, but combine it with the one above
- # r, roll = like fold, but discard this commit's description
+ # r, roll = like fold, but discard this commit's description and date
# d, drop = remove commit from history
# m, mess = edit commit message without changing commit content
#
@@ -58,7 +58,7 @@
# p, pick = use commit
# e, edit = use commit, but stop for amending
# f, fold = use commit, but combine it with the one above
- # r, roll = like fold, but discard this commit's description
+ # r, roll = like fold, but discard this commit's description and date
# d, drop = remove commit from history
# m, mess = edit commit message without changing commit content
#
@@ -71,11 +71,11 @@
***
Add delta
-Edit the commit message to your liking, then close the editor. For
-this example, let's assume that the commit message was changed to
-``Add beta and delta.`` After histedit has run and had a chance to
-remove any old or temporary revisions it needed, the history looks
-like this::
+Edit the commit message to your liking, then close the editor. The date used
+for the commit will be the later of the two commits' dates. For this example,
+let's assume that the commit message was changed to ``Add beta and delta.``
+After histedit has run and had a chance to remove any old or temporary
+revisions it needed, the history looks like this::
@ 2[tip] 989b4d060121 2009-04-27 18:04 -0500 durin42
| Add beta and delta.
@@ -97,9 +97,10 @@
allowing you to edit files freely, or even use ``hg record`` to commit
some changes as a separate commit. When you're done, any remaining
uncommitted changes will be committed as well. When done, run ``hg
-histedit --continue`` to finish this step. You'll be prompted for a
-new commit message, but the default commit message will be the
-original message for the ``edit`` ed revision.
+histedit --continue`` to finish this step. If there are uncommitted
+changes, you'll be prompted for a new commit message, but the default
+commit message will be the original message for the ``edit`` ed
+revision, and the date of the original commit will be preserved.
The ``message`` operation will give you a chance to revise a commit
message without changing the contents. It's a shortcut for doing
@@ -724,6 +725,15 @@
"""
return True
+ def firstdate(self):
+ """Returns true if the rule should preserve the date of the first
+ change.
+
+ This exists mainly so that 'rollup' rules can be a subclass of
+ 'fold'.
+ """
+ return False
+
def finishfold(self, ui, repo, ctx, oldctx, newnode, internalchanges):
parent = ctx.parents()[0].node()
repo.ui.pushbuffer()
@@ -742,7 +752,10 @@
[oldctx.description()]) + '\n'
commitopts['message'] = newmessage
# date
- commitopts['date'] = max(ctx.date(), oldctx.date())
+ if self.firstdate():
+ commitopts['date'] = ctx.date()
+ else:
+ commitopts['date'] = max(ctx.date(), oldctx.date())
extra = ctx.extra().copy()
# histedit_source
# note: ctx is likely a temporary commit but that the best we can do
@@ -809,7 +822,7 @@
return True
@action(["roll", "r"],
- _("like fold, but discard this commit's description"))
+ _("like fold, but discard this commit's description and date"))
class rollup(fold):
def mergedescs(self):
return False
@@ -817,6 +830,9 @@
def skipprompt(self):
return True
+ def firstdate(self):
+ return True
+
@action(["drop", "d"],
_('remove commit from history'))
class drop(histeditaction):
@@ -884,11 +900,11 @@
- `mess` to reword the changeset commit message
- - `fold` to combine it with the preceding changeset
+ - `fold` to combine it with the preceding changeset (using the later date)
- - `roll` like fold, but discarding this commit's description
+ - `roll` like fold, but discarding this commit's description and date
- - `edit` to edit this changeset
+ - `edit` to edit this changeset (preserving date)
There are a number of ways to select the root changeset:
@@ -992,7 +1008,8 @@
def _readfile(ui, path):
if path == '-':
- return ui.fin.read()
+ with ui.timeblockedsection('histedit'):
+ return ui.fin.read()
else:
with open(path, 'rb') as f:
return f.read()
--- a/hgext/largefiles/overrides.py Sat Feb 25 12:48:50 2017 +0900
+++ b/hgext/largefiles/overrides.py Tue Feb 28 11:13:25 2017 -0800
@@ -22,8 +22,8 @@
match as matchmod,
pathutil,
registrar,
- revset,
scmutil,
+ smartset,
util,
)
@@ -855,7 +855,7 @@
firstpulled = repo.firstpulled
except AttributeError:
raise error.Abort(_("pulled() only available in --lfrev"))
- return revset.baseset([r for r in subset if r >= firstpulled])
+ return smartset.baseset([r for r in subset if r >= firstpulled])
def overrideclone(orig, ui, source, dest=None, **opts):
d = dest
@@ -993,9 +993,9 @@
archiver.done()
-def hgsubrepoarchive(orig, repo, archiver, prefix, match=None):
+def hgsubrepoarchive(orig, repo, archiver, prefix, match=None, decode=True):
if not repo._repo.lfstatus:
- return orig(repo, archiver, prefix, match)
+ return orig(repo, archiver, prefix, match, decode)
repo._get(repo._state + ('hg',))
rev = repo._state[1]
@@ -1010,6 +1010,8 @@
if match and not match(f):
return
data = getdata()
+ if decode:
+ data = repo._repo.wwritedata(name, data)
archiver.addfile(prefix + repo._path + '/' + name, mode, islink, data)
@@ -1037,7 +1039,7 @@
sub = ctx.workingsub(subpath)
submatch = matchmod.subdirmatcher(subpath, match)
sub._repo.lfstatus = True
- sub.archive(archiver, prefix + repo._path + '/', submatch)
+ sub.archive(archiver, prefix + repo._path + '/', submatch, decode)
# If a largefile is modified, the change is not reflected in its
# standin until a commit. cmdutil.bailifchanged() raises an exception
--- a/hgext/mq.py Sat Feb 25 12:48:50 2017 +0900
+++ b/hgext/mq.py Tue Feb 28 11:13:25 2017 -0800
@@ -14,7 +14,7 @@
Known patches are represented as patch files in the .hg/patches
directory. Applied patches are both patch files and changesets.
-Common tasks (use :hg:`help command` for more details)::
+Common tasks (use :hg:`help COMMAND` for more details)::
create new patch qnew
import existing patch qimport
@@ -89,8 +89,9 @@
phases,
pycompat,
registrar,
- revset,
+ revsetlang,
scmutil,
+ smartset,
subrepo,
util,
)
@@ -2675,6 +2676,7 @@
Returns 0 on success.
"""
+ ui.pager('qdiff')
repo.mq.diff(repo, pats, opts)
return 0
@@ -3567,9 +3569,9 @@
def revsetmq(repo, subset, x):
"""Changesets managed by MQ.
"""
- revset.getargs(x, 0, 0, _("mq takes no arguments"))
+ revsetlang.getargs(x, 0, 0, _("mq takes no arguments"))
applied = set([repo[r.node].rev() for r in repo.mq.applied])
- return revset.baseset([r for r in subset if r in applied])
+ return smartset.baseset([r for r in subset if r in applied])
# tell hggettext to extract docstrings from these functions:
i18nfunctions = [revsetmq]
--- a/hgext/pager.py Sat Feb 25 12:48:50 2017 +0900
+++ b/hgext/pager.py Tue Feb 28 11:13:25 2017 -0800
@@ -12,68 +12,22 @@
#
# Run 'hg help pager' to get info on configuration.
-'''browse command output with an external pager
-
-To set the pager that should be used, set the application variable::
-
- [pager]
- pager = less -FRX
-
-If no pager is set, the pager extensions uses the environment variable
-$PAGER. If neither pager.pager, nor $PAGER is set, no pager is used.
-
-You can disable the pager for certain commands by adding them to the
-pager.ignore list::
+'''browse command output with an external pager (DEPRECATED)
- [pager]
- ignore = version, help, update
-
-You can also enable the pager only for certain commands using
-pager.attend. Below is the default list of commands to be paged::
-
- [pager]
- attend = annotate, cat, diff, export, glog, log, qdiff
-
-Setting pager.attend to an empty value will cause all commands to be
-paged.
-
-If pager.attend is present, pager.ignore will be ignored.
-
-Lastly, you can enable and disable paging for individual commands with
-the attend-<command> option. This setting takes precedence over
-existing attend and ignore options and defaults::
+Forcibly enable paging for individual commands that don't typically
+request pagination with the attend-<command> option. This setting
+takes precedence over ignore options and defaults::
[pager]
attend-cat = false
-
-To ignore global commands like :hg:`version` or :hg:`help`, you have
-to specify them in your user configuration file.
-
-To control whether the pager is used at all for an individual command,
-you can use --pager=<value>::
-
- - use as needed: `auto`.
- - require the pager: `yes` or `on`.
- - suppress the pager: `no` or `off` (any unrecognized value
- will also work).
-
'''
from __future__ import absolute_import
-import atexit
-import os
-import signal
-import subprocess
-import sys
-
-from mercurial.i18n import _
from mercurial import (
cmdutil,
commands,
dispatch,
- encoding,
extensions,
- util,
)
# Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
@@ -82,58 +36,12 @@
# leave the attribute unspecified.
testedwith = 'ships-with-hg-core'
-def _runpager(ui, p):
- pager = subprocess.Popen(p, shell=True, bufsize=-1,
- close_fds=util.closefds, stdin=subprocess.PIPE,
- stdout=util.stdout, stderr=util.stderr)
-
- # back up original file objects and descriptors
- olduifout = ui.fout
- oldstdout = util.stdout
- stdoutfd = os.dup(util.stdout.fileno())
- stderrfd = os.dup(util.stderr.fileno())
-
- # create new line-buffered stdout so that output can show up immediately
- ui.fout = util.stdout = newstdout = os.fdopen(util.stdout.fileno(), 'wb', 1)
- os.dup2(pager.stdin.fileno(), util.stdout.fileno())
- if ui._isatty(util.stderr):
- os.dup2(pager.stdin.fileno(), util.stderr.fileno())
-
- @atexit.register
- def killpager():
- if util.safehasattr(signal, "SIGINT"):
- signal.signal(signal.SIGINT, signal.SIG_IGN)
- pager.stdin.close()
- ui.fout = olduifout
- util.stdout = oldstdout
- # close new stdout while it's associated with pager; otherwise stdout
- # fd would be closed when newstdout is deleted
- newstdout.close()
- # restore original fds: stdout is open again
- os.dup2(stdoutfd, util.stdout.fileno())
- os.dup2(stderrfd, util.stderr.fileno())
- pager.wait()
-
def uisetup(ui):
- class pagerui(ui.__class__):
- def _runpager(self, pagercmd):
- _runpager(self, pagercmd)
-
- ui.__class__ = pagerui
def pagecmd(orig, ui, options, cmd, cmdfunc):
- p = ui.config("pager", "pager", encoding.environ.get("PAGER"))
- usepager = False
- always = util.parsebool(options['pager'])
auto = options['pager'] == 'auto'
-
- if not p or '--debugger' in sys.argv or not ui.formatted():
- pass
- elif always:
- usepager = True
- elif not auto:
+ if auto and not ui.pageractive:
usepager = False
- else:
attend = ui.configlist('pager', 'attend', attended)
ignore = ui.configlist('pager', 'ignore')
cmds, _ = cmdutil.findcmd(cmd, commands.table)
@@ -148,14 +56,14 @@
usepager = True
break
- setattr(ui, 'pageractive', usepager)
-
- if usepager:
- ui.setconfig('ui', 'formatted', ui.formatted(), 'pager')
- ui.setconfig('ui', 'interactive', False, 'pager')
- if util.safehasattr(signal, "SIGPIPE"):
- signal.signal(signal.SIGPIPE, signal.SIG_DFL)
- ui._runpager(p)
+ if usepager:
+ # Slight hack: the attend list is supposed to override
+ # the ignore list for the pager extension, but the
+ # core code doesn't know about attend, so we have to
+ # lobotomize the ignore list so that the extension's
+ # behavior is preserved.
+ ui.setconfig('pager', 'ignore', '', 'pager')
+ ui.pager('extension-via-attend-' + cmd)
return orig(ui, options, cmd, cmdfunc)
# Wrap dispatch._runcommand after color is loaded so color can see
@@ -165,10 +73,6 @@
extensions.wrapfunction(dispatch, '_runcommand', pagecmd)
extensions.afterloaded('color', afterloaded)
-def extsetup(ui):
- commands.globalopts.append(
- ('', 'pager', 'auto',
- _("when to paginate (boolean, always, auto, or never)"),
- _('TYPE')))
-
-attended = ['annotate', 'cat', 'diff', 'export', 'glog', 'log', 'qdiff']
+attended = [
+ 'the-default-attend-list-is-now-empty-but-that-breaks-the-extension',
+]
--- a/hgext/rebase.py Sat Feb 25 12:48:50 2017 +0900
+++ b/hgext/rebase.py Tue Feb 28 11:13:25 2017 -0800
@@ -47,6 +47,7 @@
repoview,
revset,
scmutil,
+ smartset,
util,
)
@@ -118,8 +119,8 @@
# i18n: "_rebasedefaultdest" is a keyword
sourceset = None
if x is not None:
- sourceset = revset.getset(repo, revset.fullreposet(repo), x)
- return subset & revset.baseset([_destrebase(repo, sourceset)])
+ sourceset = revset.getset(repo, smartset.fullreposet(repo), x)
+ return subset & smartset.baseset([_destrebase(repo, sourceset)])
class rebaseruntime(object):
"""This class is a container for rebase runtime state"""
@@ -1367,7 +1368,7 @@
"""store the currently rebased set on the repo object
This is used by another function to prevent rebased revision to because
- hidden (see issue4505)"""
+ hidden (see issue4504)"""
repo = repo.unfiltered()
revs = set(revs)
repo._rebaseset = revs
@@ -1383,7 +1384,7 @@
del repo._rebaseset
def _rebasedvisible(orig, repo):
- """ensure rebased revs stay visible (see issue4505)"""
+ """ensure rebased revs stay visible (see issue4504)"""
blockers = orig(repo)
blockers.update(getattr(repo, '_rebaseset', ()))
return blockers
--- a/hgext/share.py Sat Feb 25 12:48:50 2017 +0900
+++ b/hgext/share.py Tue Feb 28 11:13:25 2017 -0800
@@ -48,6 +48,7 @@
error,
extensions,
hg,
+ txnutil,
util,
)
@@ -171,7 +172,28 @@
if _hassharedbookmarks(repo):
srcrepo = _getsrcrepo(repo)
if srcrepo is not None:
+ # just orig(srcrepo) doesn't work as expected, because
+ # HG_PENDING refers repo.root.
+ try:
+ fp, pending = txnutil.trypending(repo.root, repo.vfs,
+ 'bookmarks')
+ if pending:
+ # only in this case, bookmark information in repo
+ # is up-to-date.
+ return fp
+ fp.close()
+ except IOError as inst:
+ if inst.errno != errno.ENOENT:
+ raise
+
+ # otherwise, we should read bookmarks from srcrepo,
+ # because .hg/bookmarks in srcrepo might be already
+ # changed via another sharing repo
repo = srcrepo
+
+ # TODO: Pending changes in repo are still invisible in
+ # srcrepo, because bookmarks.pending is written only into repo.
+ # See also https://www.mercurial-scm.org/wiki/SharedRepository
return orig(repo)
def recordchange(orig, self, tr):
--- a/hgext/shelve.py Sat Feb 25 12:48:50 2017 +0900
+++ b/hgext/shelve.py Tue Feb 28 11:13:25 2017 -0800
@@ -485,6 +485,7 @@
if not ui.plain():
width = ui.termwidth()
namelabel = 'shelve.newest'
+ ui.pager('shelve')
for mtime, name in listshelves(repo):
sname = util.split(name)[1]
if pats and sname not in pats:
@@ -747,10 +748,12 @@
_('continue an incomplete unshelve operation')),
('k', 'keep', None,
_('keep shelve after unshelving')),
+ ('n', 'name', '',
+ _('restore shelved change with given name'), _('NAME')),
('t', 'tool', '', _('specify merge tool')),
('', 'date', '',
_('set date for temporary commits (DEPRECATED)'), _('DATE'))],
- _('hg unshelve [SHELVED]'))
+ _('hg unshelve [[-n] SHELVED]'))
def unshelve(ui, repo, *shelved, **opts):
"""restore a shelved change to the working directory
@@ -795,6 +798,9 @@
continuef = opts.get('continue')
if not abortf and not continuef:
cmdutil.checkunfinished(repo)
+ shelved = list(shelved)
+ if opts.get("name"):
+ shelved.append(opts["name"])
if abortf or continuef:
if abortf and continuef:
--- a/hgext/transplant.py Sat Feb 25 12:48:50 2017 +0900
+++ b/hgext/transplant.py Tue Feb 28 11:13:25 2017 -0800
@@ -28,10 +28,12 @@
merge,
node as nodemod,
patch,
+ pycompat,
registrar,
revlog,
revset,
scmutil,
+ smartset,
util,
)
@@ -197,7 +199,7 @@
patchfile = None
else:
fd, patchfile = tempfile.mkstemp(prefix='hg-transplant-')
- fp = os.fdopen(fd, 'w')
+ fp = os.fdopen(fd, pycompat.sysstr('w'))
gen = patch.diff(source, parent, node, opts=diffopts)
for chunk in gen:
fp.write(chunk)
@@ -245,7 +247,7 @@
self.ui.status(_('filtering %s\n') % patchfile)
user, date, msg = (changelog[1], changelog[2], changelog[4])
fd, headerfile = tempfile.mkstemp(prefix='hg-transplant-')
- fp = os.fdopen(fd, 'w')
+ fp = os.fdopen(fd, pycompat.sysstr('w'))
fp.write("# HG changeset patch\n")
fp.write("# User %s\n" % user)
fp.write("# Date %d %d\n" % date)
@@ -722,7 +724,7 @@
s = revset.getset(repo, subset, x)
else:
s = subset
- return revset.baseset([r for r in s if
+ return smartset.baseset([r for r in s if
repo[r].extra().get('transplant_source')])
templatekeyword = registrar.templatekeyword()
--- a/hgext/zeroconf/__init__.py Sat Feb 25 12:48:50 2017 +0900
+++ b/hgext/zeroconf/__init__.py Tue Feb 28 11:13:25 2017 -0800
@@ -64,7 +64,9 @@
# Generic method, sometimes gives useless results
try:
dumbip = socket.gethostbyaddr(socket.gethostname())[2][0]
- if not dumbip.startswith('127.') and ':' not in dumbip:
+ if ':' in dumbip:
+ dumbip = '127.0.0.1'
+ if not dumbip.startswith('127.'):
return dumbip
except (socket.gaierror, socket.herror):
dumbip = '127.0.0.1'
--- a/mercurial/archival.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/archival.py Tue Feb 28 11:13:25 2017 -0800
@@ -331,7 +331,7 @@
for subpath in sorted(ctx.substate):
sub = ctx.workingsub(subpath)
submatch = matchmod.subdirmatcher(subpath, matchfn)
- total += sub.archive(archiver, prefix, submatch)
+ total += sub.archive(archiver, prefix, submatch, decode)
if total == 0:
raise error.Abort(_('no files match the archive pattern'))
--- a/mercurial/bookmarks.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/bookmarks.py Tue Feb 28 11:13:25 2017 -0800
@@ -19,6 +19,7 @@
error,
lock as lockmod,
obsolete,
+ txnutil,
util,
)
@@ -29,17 +30,8 @@
bookmarks or the committed ones. Other extensions (like share)
may need to tweak this behavior further.
"""
- bkfile = None
- if 'HG_PENDING' in encoding.environ:
- try:
- bkfile = repo.vfs('bookmarks.pending')
- except IOError as inst:
- if inst.errno != errno.ENOENT:
- raise
- if bkfile is None:
- bkfile = repo.vfs('bookmarks')
- return bkfile
-
+ fp, pending = txnutil.trypending(repo.root, repo.vfs, 'bookmarks')
+ return fp
class bmstore(dict):
"""Storage for bookmarks.
--- a/mercurial/branchmap.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/branchmap.py Tue Feb 28 11:13:25 2017 -0800
@@ -9,7 +9,6 @@
import array
import struct
-import time
from .node import (
bin,
@@ -21,6 +20,7 @@
encoding,
error,
scmutil,
+ util,
)
array = array.array
@@ -261,7 +261,7 @@
missing heads, and a generator of nodes that are strictly a superset of
heads missing, this function updates self to be correct.
"""
- starttime = time.time()
+ starttime = util.timer()
cl = repo.changelog
# collect new branch entries
newbranches = {}
@@ -314,7 +314,7 @@
self.tiprev = tiprev
self.filteredhash = scmutil.filteredhash(repo, self.tiprev)
- duration = time.time() - starttime
+ duration = util.timer() - starttime
repo.ui.log('branchcache', 'updated %s branch cache in %.4f seconds\n',
repo.filtername, duration)
--- a/mercurial/bundle2.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/bundle2.py Tue Feb 28 11:13:25 2017 -0800
@@ -320,9 +320,6 @@
It iterates over each part then searches for and uses the proper handling
code to process the part. Parts are processed in order.
- This is very early version of this function that will be strongly reworked
- before final usage.
-
Unknown Mandatory part will abort the process.
It is temporarily possible to provide a prebuilt bundleoperation to the
@@ -865,6 +862,11 @@
self._generated = None
self.mandatory = mandatory
+ def __repr__(self):
+ cls = "%s.%s" % (self.__class__.__module__, self.__class__.__name__)
+ return ('<%s object at %x; id: %s; type: %s; mandatory: %s>'
+ % (cls, id(self), self.id, self.type, self.mandatory))
+
def copy(self):
"""return a copy of the part
--- a/mercurial/bundlerepo.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/bundlerepo.py Tue Feb 28 11:13:25 2017 -0800
@@ -272,7 +272,7 @@
suffix=".hg10un")
self.tempfile = temp
- with os.fdopen(fdtemp, 'wb') as fptemp:
+ with os.fdopen(fdtemp, pycompat.sysstr('wb')) as fptemp:
fptemp.write(header)
while True:
chunk = read(2**18)
--- a/mercurial/changegroup.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/changegroup.py Tue Feb 28 11:13:25 2017 -0800
@@ -26,6 +26,7 @@
error,
mdiff,
phases,
+ pycompat,
util,
)
@@ -98,7 +99,7 @@
fh = open(filename, "wb", 131072)
else:
fd, filename = tempfile.mkstemp(prefix="hg-bundle-", suffix=".hg")
- fh = os.fdopen(fd, "wb")
+ fh = os.fdopen(fd, pycompat.sysstr("wb"))
cleanup = filename
for c in chunks:
fh.write(c)
--- a/mercurial/chgserver.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/chgserver.py Tue Feb 28 11:13:25 2017 -0800
@@ -31,8 +31,11 @@
::
[chgserver]
- idletimeout = 3600 # seconds, after which an idle server will exit
- skiphash = False # whether to skip config or env change checks
+ # how long (in seconds) should an idle chg server exit
+ idletimeout = 3600
+
+ # whether to skip config or env change checks
+ skiphash = False
"""
from __future__ import absolute_import
@@ -176,26 +179,17 @@
else:
self._csystem = csystem
- def system(self, cmd, environ=None, cwd=None, onerr=None,
- errprefix=None):
+ def _runsystem(self, cmd, environ, cwd, out):
# fallback to the original system method if the output needs to be
# captured (to self._buffers), or the output stream is not stdout
# (e.g. stderr, cStringIO), because the chg client is not aware of
# these situations and will behave differently (write to stdout).
- if (any(s[1] for s in self._bufferstates)
+ if (out is not self.fout
or not util.safehasattr(self.fout, 'fileno')
or self.fout.fileno() != util.stdout.fileno()):
- return super(chgui, self).system(cmd, environ, cwd, onerr,
- errprefix)
+ return util.system(cmd, environ=environ, cwd=cwd, out=out)
self.flush()
- rc = self._csystem(cmd, util.shellenviron(environ), cwd)
- if rc and onerr:
- errmsg = '%s %s' % (os.path.basename(cmd.split(None, 1)[0]),
- util.explainexit(rc)[0])
- if errprefix:
- errmsg = '%s: %s' % (errprefix, errmsg)
- raise onerr(errmsg)
- return rc
+ return self._csystem(cmd, util.shellenviron(environ), cwd)
def _runpager(self, cmd):
self._csystem(cmd, util.shellenviron(), type='pager',
@@ -287,9 +281,9 @@
_iochannels = [
# server.ch, ui.fp, mode
- ('cin', 'fin', 'rb'),
- ('cout', 'fout', 'wb'),
- ('cerr', 'ferr', 'wb'),
+ ('cin', 'fin', pycompat.sysstr('rb')),
+ ('cout', 'fout', pycompat.sysstr('wb')),
+ ('cerr', 'ferr', pycompat.sysstr('wb')),
]
class chgcmdserver(commandserver.server):
--- a/mercurial/cmdutil.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/cmdutil.py Tue Feb 28 11:13:25 2017 -0800
@@ -26,14 +26,12 @@
changelog,
copies,
crecord as crecordmod,
- dirstateguard as dirstateguardmod,
encoding,
error,
formatter,
graphmod,
lock as lockmod,
match as matchmod,
- mergeutil,
obsolete,
patch,
pathutil,
@@ -43,6 +41,7 @@
revlog,
revset,
scmutil,
+ smartset,
templatekw,
templater,
util,
@@ -2092,11 +2091,11 @@
if opts.get('rev'):
revs = scmutil.revrange(repo, opts['rev'])
elif follow and repo.dirstate.p1() == nullid:
- revs = revset.baseset()
+ revs = smartset.baseset()
elif follow:
revs = repo.revs('reverse(:.)')
else:
- revs = revset.spanset(repo)
+ revs = smartset.spanset(repo)
revs.reverse()
return revs
@@ -2111,7 +2110,7 @@
limit = loglimit(opts)
revs = _logrevs(repo, opts)
if not revs:
- return revset.baseset(), None, None
+ return smartset.baseset(), None, None
expr, filematcher = _makelogrevset(repo, pats, opts, revs)
if opts.get('rev'):
# User-specified revs might be unsorted, but don't sort before
@@ -2127,7 +2126,7 @@
if idx >= limit:
break
limitedrevs.append(rev)
- revs = revset.baseset(limitedrevs)
+ revs = smartset.baseset(limitedrevs)
return revs, expr, filematcher
@@ -2142,7 +2141,7 @@
limit = loglimit(opts)
revs = _logrevs(repo, opts)
if not revs:
- return revset.baseset([]), None, None
+ return smartset.baseset([]), None, None
expr, filematcher = _makelogrevset(repo, pats, opts, revs)
if expr:
matcher = revset.match(repo.ui, expr, order=revset.followorder)
@@ -2153,7 +2152,7 @@
if limit <= idx:
break
limitedrevs.append(r)
- revs = revset.baseset(limitedrevs)
+ revs = smartset.baseset(limitedrevs)
return revs, expr, filematcher
@@ -2236,6 +2235,8 @@
if opts.get('rev'):
endrev = scmutil.revrange(repo, opts.get('rev')).max() + 1
getrenamed = templatekw.getrenamedfn(repo, endrev=endrev)
+
+ ui.pager('log')
displayer = show_changeset(ui, repo, opts, buffered=True)
displaygraph(ui, repo, revdag, displayer, graphmod.asciiedges, getrenamed,
filematcher)
@@ -3366,11 +3367,6 @@
return cmd
-def checkunresolved(ms):
- ms._repo.ui.deprecwarn('checkunresolved moved from cmdutil to mergeutil',
- '4.1')
- return mergeutil.checkunresolved(ms)
-
# a list of (ui, repo, otherpeer, opts, missing) functions called by
# commands.outgoing. "missing" is "missing" of the result of
# "findcommonoutgoing()"
@@ -3477,10 +3473,3 @@
if after[1]:
hint = after[0]
raise error.Abort(_('no %s in progress') % task, hint=hint)
-
-class dirstateguard(dirstateguardmod.dirstateguard):
- def __init__(self, repo, name):
- dirstateguardmod.dirstateguard.__init__(self, repo, name)
- repo.ui.deprecwarn(
- 'dirstateguard has moved from cmdutil to dirstateguard',
- '4.1')
--- a/mercurial/color.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/color.py Tue Feb 28 11:13:25 2017 -0800
@@ -7,59 +7,466 @@
from __future__ import absolute_import
-_styles = {'grep.match': 'red bold',
- 'grep.linenumber': 'green',
- 'grep.rev': 'green',
- 'grep.change': 'green',
- 'grep.sep': 'cyan',
- 'grep.filename': 'magenta',
- 'grep.user': 'magenta',
- 'grep.date': 'magenta',
- 'bookmarks.active': 'green',
- 'branches.active': 'none',
- 'branches.closed': 'black bold',
- 'branches.current': 'green',
- 'branches.inactive': 'none',
- 'diff.changed': 'white',
- 'diff.deleted': 'red',
- 'diff.diffline': 'bold',
- 'diff.extended': 'cyan bold',
- 'diff.file_a': 'red bold',
- 'diff.file_b': 'green bold',
- 'diff.hunk': 'magenta',
- 'diff.inserted': 'green',
- 'diff.tab': '',
- 'diff.trailingwhitespace': 'bold red_background',
- 'changeset.public' : '',
- 'changeset.draft' : '',
- 'changeset.secret' : '',
- 'diffstat.deleted': 'red',
- 'diffstat.inserted': 'green',
- 'histedit.remaining': 'red bold',
- 'ui.prompt': 'yellow',
- 'log.changeset': 'yellow',
- 'patchbomb.finalsummary': '',
- 'patchbomb.from': 'magenta',
- 'patchbomb.to': 'cyan',
- 'patchbomb.subject': 'green',
- 'patchbomb.diffstats': '',
- 'rebase.rebased': 'blue',
- 'rebase.remaining': 'red bold',
- 'resolve.resolved': 'green bold',
- 'resolve.unresolved': 'red bold',
- 'shelve.age': 'cyan',
- 'shelve.newest': 'green bold',
- 'shelve.name': 'blue bold',
- 'status.added': 'green bold',
- 'status.clean': 'none',
- 'status.copied': 'none',
- 'status.deleted': 'cyan bold underline',
- 'status.ignored': 'black bold',
- 'status.modified': 'blue bold',
- 'status.removed': 'red bold',
- 'status.unknown': 'magenta bold underline',
- 'tags.normal': 'green',
- 'tags.local': 'black bold'}
+from .i18n import _
+
+from . import (
+ encoding,
+ pycompat,
+ util
+)
+
+try:
+ import curses
+ # Mapping from effect name to terminfo attribute name (or raw code) or
+ # color number. This will also force-load the curses module.
+ _baseterminfoparams = {
+ 'none': (True, 'sgr0', ''),
+ 'standout': (True, 'smso', ''),
+ 'underline': (True, 'smul', ''),
+ 'reverse': (True, 'rev', ''),
+ 'inverse': (True, 'rev', ''),
+ 'blink': (True, 'blink', ''),
+ 'dim': (True, 'dim', ''),
+ 'bold': (True, 'bold', ''),
+ 'invisible': (True, 'invis', ''),
+ 'italic': (True, 'sitm', ''),
+ 'black': (False, curses.COLOR_BLACK, ''),
+ 'red': (False, curses.COLOR_RED, ''),
+ 'green': (False, curses.COLOR_GREEN, ''),
+ 'yellow': (False, curses.COLOR_YELLOW, ''),
+ 'blue': (False, curses.COLOR_BLUE, ''),
+ 'magenta': (False, curses.COLOR_MAGENTA, ''),
+ 'cyan': (False, curses.COLOR_CYAN, ''),
+ 'white': (False, curses.COLOR_WHITE, ''),
+ }
+except ImportError:
+ curses = None
+ _baseterminfoparams = {}
+
+# allow the extensions to change the default
+_enabledbydefault = False
+
+# start and stop parameters for effects
+_effects = {
+ 'none': 0,
+ 'black': 30,
+ 'red': 31,
+ 'green': 32,
+ 'yellow': 33,
+ 'blue': 34,
+ 'magenta': 35,
+ 'cyan': 36,
+ 'white': 37,
+ 'bold': 1,
+ 'italic': 3,
+ 'underline': 4,
+ 'inverse': 7,
+ 'dim': 2,
+ 'black_background': 40,
+ 'red_background': 41,
+ 'green_background': 42,
+ 'yellow_background': 43,
+ 'blue_background': 44,
+ 'purple_background': 45,
+ 'cyan_background': 46,
+ 'white_background': 47,
+ }
+
+_defaultstyles = {
+ 'grep.match': 'red bold',
+ 'grep.linenumber': 'green',
+ 'grep.rev': 'green',
+ 'grep.change': 'green',
+ 'grep.sep': 'cyan',
+ 'grep.filename': 'magenta',
+ 'grep.user': 'magenta',
+ 'grep.date': 'magenta',
+ 'bookmarks.active': 'green',
+ 'branches.active': 'none',
+ 'branches.closed': 'black bold',
+ 'branches.current': 'green',
+ 'branches.inactive': 'none',
+ 'diff.changed': 'white',
+ 'diff.deleted': 'red',
+ 'diff.diffline': 'bold',
+ 'diff.extended': 'cyan bold',
+ 'diff.file_a': 'red bold',
+ 'diff.file_b': 'green bold',
+ 'diff.hunk': 'magenta',
+ 'diff.inserted': 'green',
+ 'diff.tab': '',
+ 'diff.trailingwhitespace': 'bold red_background',
+ 'changeset.public' : '',
+ 'changeset.draft' : '',
+ 'changeset.secret' : '',
+ 'diffstat.deleted': 'red',
+ 'diffstat.inserted': 'green',
+ 'histedit.remaining': 'red bold',
+ 'ui.prompt': 'yellow',
+ 'log.changeset': 'yellow',
+ 'patchbomb.finalsummary': '',
+ 'patchbomb.from': 'magenta',
+ 'patchbomb.to': 'cyan',
+ 'patchbomb.subject': 'green',
+ 'patchbomb.diffstats': '',
+ 'rebase.rebased': 'blue',
+ 'rebase.remaining': 'red bold',
+ 'resolve.resolved': 'green bold',
+ 'resolve.unresolved': 'red bold',
+ 'shelve.age': 'cyan',
+ 'shelve.newest': 'green bold',
+ 'shelve.name': 'blue bold',
+ 'status.added': 'green bold',
+ 'status.clean': 'none',
+ 'status.copied': 'none',
+ 'status.deleted': 'cyan bold underline',
+ 'status.ignored': 'black bold',
+ 'status.modified': 'blue bold',
+ 'status.removed': 'red bold',
+ 'status.unknown': 'magenta bold underline',
+ 'tags.normal': 'green',
+ 'tags.local': 'black bold',
+}
def loadcolortable(ui, extname, colortable):
- _styles.update(colortable)
+ _defaultstyles.update(colortable)
+
+def _terminfosetup(ui, mode):
+ '''Initialize terminfo data and the terminal if we're in terminfo mode.'''
+
+ # If we failed to load curses, we go ahead and return.
+ if curses is None:
+ return
+ # Otherwise, see what the config file says.
+ if mode not in ('auto', 'terminfo'):
+ return
+ ui._terminfoparams.update(_baseterminfoparams)
+
+ for key, val in ui.configitems('color'):
+ if key.startswith('color.'):
+ newval = (False, int(val), '')
+ ui._terminfoparams[key[6:]] = newval
+ elif key.startswith('terminfo.'):
+ newval = (True, '', val.replace('\\E', '\x1b'))
+ ui._terminfoparams[key[9:]] = newval
+ try:
+ curses.setupterm()
+ except curses.error as e:
+ ui._terminfoparams.clear()
+ return
+
+ for key, (b, e, c) in ui._terminfoparams.items():
+ if not b:
+ continue
+ if not c and not curses.tigetstr(e):
+ # Most terminals don't support dim, invis, etc, so don't be
+ # noisy and use ui.debug().
+ ui.debug("no terminfo entry for %s\n" % e)
+ del ui._terminfoparams[key]
+ if not curses.tigetstr('setaf') or not curses.tigetstr('setab'):
+ # Only warn about missing terminfo entries if we explicitly asked for
+ # terminfo mode.
+ if mode == "terminfo":
+ ui.warn(_("no terminfo entry for setab/setaf: reverting to "
+ "ECMA-48 color\n"))
+ ui._terminfoparams.clear()
+
+def setup(ui):
+ """configure color on a ui
+
+ That function both set the colormode for the ui object and read
+ the configuration looking for custom colors and effect definitions."""
+ mode = _modesetup(ui)
+ ui._colormode = mode
+ if mode and mode != 'debug':
+ configstyles(ui)
+
+def _modesetup(ui):
+ if ui.plain():
+ return None
+ default = 'never'
+ if _enabledbydefault:
+ default = 'auto'
+ # experimental config: ui.color
+ config = ui.config('ui', 'color', default)
+ if config == 'debug':
+ return 'debug'
+
+ auto = (config == 'auto')
+ always = not auto and util.parsebool(config)
+ if not always and not auto:
+ return None
+
+ formatted = (always or (encoding.environ.get('TERM') != 'dumb'
+ and ui.formatted()))
+
+ mode = ui.config('color', 'mode', 'auto')
+
+ # If pager is active, color.pagermode overrides color.mode.
+ if getattr(ui, 'pageractive', False):
+ mode = ui.config('color', 'pagermode', mode)
+
+ realmode = mode
+ if mode == 'auto':
+ if pycompat.osname == 'nt':
+ term = encoding.environ.get('TERM')
+ # TERM won't be defined in a vanilla cmd.exe environment.
+
+ # UNIX-like environments on Windows such as Cygwin and MSYS will
+ # set TERM. They appear to make a best effort attempt at setting it
+ # to something appropriate. However, not all environments with TERM
+ # defined support ANSI. Since "ansi" could result in terminal
+ # gibberish, we error on the side of selecting "win32". However, if
+ # w32effects is not defined, we almost certainly don't support
+ # "win32", so don't even try.
+ if (term and 'xterm' in term) or not w32effects:
+ realmode = 'ansi'
+ else:
+ realmode = 'win32'
+ else:
+ realmode = 'ansi'
+
+ def modewarn():
+ # only warn if color.mode was explicitly set and we're in
+ # a formatted terminal
+ if mode == realmode and ui.formatted():
+ ui.warn(_('warning: failed to set color mode to %s\n') % mode)
+
+ if realmode == 'win32':
+ ui._terminfoparams.clear()
+ if not w32effects:
+ modewarn()
+ return None
+ _effects.update(w32effects)
+ elif realmode == 'ansi':
+ ui._terminfoparams.clear()
+ elif realmode == 'terminfo':
+ _terminfosetup(ui, mode)
+ if not ui._terminfoparams:
+ ## FIXME Shouldn't we return None in this case too?
+ modewarn()
+ realmode = 'ansi'
+ else:
+ return None
+
+ if always or (auto and formatted):
+ return realmode
+ return None
+
+def configstyles(ui):
+ ui._styles.update(_defaultstyles)
+ for status, cfgeffects in ui.configitems('color'):
+ if '.' not in status or status.startswith(('color.', 'terminfo.')):
+ continue
+ cfgeffects = ui.configlist('color', status)
+ if cfgeffects:
+ good = []
+ for e in cfgeffects:
+ if valideffect(ui, e):
+ good.append(e)
+ else:
+ ui.warn(_("ignoring unknown color/effect %r "
+ "(configured in color.%s)\n")
+ % (e, status))
+ ui._styles[status] = ' '.join(good)
+
+def valideffect(ui, effect):
+ 'Determine if the effect is valid or not.'
+ return ((not ui._terminfoparams and effect in _effects)
+ or (effect in ui._terminfoparams
+ or effect[:-11] in ui._terminfoparams))
+
+def _effect_str(ui, effect):
+ '''Helper function for render_effects().'''
+
+ bg = False
+ if effect.endswith('_background'):
+ bg = True
+ effect = effect[:-11]
+ try:
+ attr, val, termcode = ui._terminfoparams[effect]
+ except KeyError:
+ return ''
+ if attr:
+ if termcode:
+ return termcode
+ else:
+ return curses.tigetstr(val)
+ elif bg:
+ return curses.tparm(curses.tigetstr('setab'), val)
+ else:
+ return curses.tparm(curses.tigetstr('setaf'), val)
+
+def _render_effects(ui, text, effects):
+ 'Wrap text in commands to turn on each effect.'
+ if not text:
+ return text
+ if ui._terminfoparams:
+ start = ''.join(_effect_str(ui, effect)
+ for effect in ['none'] + effects.split())
+ stop = _effect_str(ui, 'none')
+ else:
+ start = [str(_effects[e]) for e in ['none'] + effects.split()]
+ start = '\033[' + ';'.join(start) + 'm'
+ stop = '\033[' + str(_effects['none']) + 'm'
+ return ''.join([start, text, stop])
+
+def colorlabel(ui, msg, label):
+ """add color control code according to the mode"""
+ if ui._colormode == 'debug':
+ if label and msg:
+ if msg[-1] == '\n':
+ msg = "[%s|%s]\n" % (label, msg[:-1])
+ else:
+ msg = "[%s|%s]" % (label, msg)
+ elif ui._colormode is not None:
+ effects = []
+ for l in label.split():
+ s = ui._styles.get(l, '')
+ if s:
+ effects.append(s)
+ elif valideffect(ui, l):
+ effects.append(l)
+ effects = ' '.join(effects)
+ if effects:
+ msg = '\n'.join([_render_effects(ui, line, effects)
+ for line in msg.split('\n')])
+ return msg
+
+w32effects = None
+if pycompat.osname == 'nt':
+ import ctypes
+ import re
+
+ _kernel32 = ctypes.windll.kernel32
+
+ _WORD = ctypes.c_ushort
+
+ _INVALID_HANDLE_VALUE = -1
+
+ class _COORD(ctypes.Structure):
+ _fields_ = [('X', ctypes.c_short),
+ ('Y', ctypes.c_short)]
+
+ class _SMALL_RECT(ctypes.Structure):
+ _fields_ = [('Left', ctypes.c_short),
+ ('Top', ctypes.c_short),
+ ('Right', ctypes.c_short),
+ ('Bottom', ctypes.c_short)]
+
+ class _CONSOLE_SCREEN_BUFFER_INFO(ctypes.Structure):
+ _fields_ = [('dwSize', _COORD),
+ ('dwCursorPosition', _COORD),
+ ('wAttributes', _WORD),
+ ('srWindow', _SMALL_RECT),
+ ('dwMaximumWindowSize', _COORD)]
+
+ _STD_OUTPUT_HANDLE = 0xfffffff5 # (DWORD)-11
+ _STD_ERROR_HANDLE = 0xfffffff4 # (DWORD)-12
+
+ _FOREGROUND_BLUE = 0x0001
+ _FOREGROUND_GREEN = 0x0002
+ _FOREGROUND_RED = 0x0004
+ _FOREGROUND_INTENSITY = 0x0008
+
+ _BACKGROUND_BLUE = 0x0010
+ _BACKGROUND_GREEN = 0x0020
+ _BACKGROUND_RED = 0x0040
+ _BACKGROUND_INTENSITY = 0x0080
+
+ _COMMON_LVB_REVERSE_VIDEO = 0x4000
+ _COMMON_LVB_UNDERSCORE = 0x8000
+
+ # http://msdn.microsoft.com/en-us/library/ms682088%28VS.85%29.aspx
+ w32effects = {
+ 'none': -1,
+ 'black': 0,
+ 'red': _FOREGROUND_RED,
+ 'green': _FOREGROUND_GREEN,
+ 'yellow': _FOREGROUND_RED | _FOREGROUND_GREEN,
+ 'blue': _FOREGROUND_BLUE,
+ 'magenta': _FOREGROUND_BLUE | _FOREGROUND_RED,
+ 'cyan': _FOREGROUND_BLUE | _FOREGROUND_GREEN,
+ 'white': _FOREGROUND_RED | _FOREGROUND_GREEN | _FOREGROUND_BLUE,
+ 'bold': _FOREGROUND_INTENSITY,
+ 'black_background': 0x100, # unused value > 0x0f
+ 'red_background': _BACKGROUND_RED,
+ 'green_background': _BACKGROUND_GREEN,
+ 'yellow_background': _BACKGROUND_RED | _BACKGROUND_GREEN,
+ 'blue_background': _BACKGROUND_BLUE,
+ 'purple_background': _BACKGROUND_BLUE | _BACKGROUND_RED,
+ 'cyan_background': _BACKGROUND_BLUE | _BACKGROUND_GREEN,
+ 'white_background': (_BACKGROUND_RED | _BACKGROUND_GREEN |
+ _BACKGROUND_BLUE),
+ 'bold_background': _BACKGROUND_INTENSITY,
+ 'underline': _COMMON_LVB_UNDERSCORE, # double-byte charsets only
+ 'inverse': _COMMON_LVB_REVERSE_VIDEO, # double-byte charsets only
+ }
+
+ passthrough = set([_FOREGROUND_INTENSITY,
+ _BACKGROUND_INTENSITY,
+ _COMMON_LVB_UNDERSCORE,
+ _COMMON_LVB_REVERSE_VIDEO])
+
+ stdout = _kernel32.GetStdHandle(
+ _STD_OUTPUT_HANDLE) # don't close the handle returned
+ if stdout is None or stdout == _INVALID_HANDLE_VALUE:
+ w32effects = None
+ else:
+ csbi = _CONSOLE_SCREEN_BUFFER_INFO()
+ if not _kernel32.GetConsoleScreenBufferInfo(
+ stdout, ctypes.byref(csbi)):
+ # stdout may not support GetConsoleScreenBufferInfo()
+ # when called from subprocess or redirected
+ w32effects = None
+ else:
+ origattr = csbi.wAttributes
+ ansire = re.compile('\033\[([^m]*)m([^\033]*)(.*)',
+ re.MULTILINE | re.DOTALL)
+
+ def win32print(ui, writefunc, *msgs, **opts):
+ for text in msgs:
+ _win32print(ui, text, writefunc, **opts)
+
+ def _win32print(ui, text, writefunc, **opts):
+ label = opts.get('label', '')
+ attr = origattr
+
+ def mapcolor(val, attr):
+ if val == -1:
+ return origattr
+ elif val in passthrough:
+ return attr | val
+ elif val > 0x0f:
+ return (val & 0x70) | (attr & 0x8f)
+ else:
+ return (val & 0x07) | (attr & 0xf8)
+
+ # determine console attributes based on labels
+ for l in label.split():
+ style = ui._styles.get(l, '')
+ for effect in style.split():
+ try:
+ attr = mapcolor(w32effects[effect], attr)
+ except KeyError:
+ # w32effects could not have certain attributes so we skip
+ # them if not found
+ pass
+ # hack to ensure regexp finds data
+ if not text.startswith('\033['):
+ text = '\033[m' + text
+
+ # Look for ANSI-like codes embedded in text
+ m = re.match(ansire, text)
+
+ try:
+ while m:
+ for sattr in m.group(1).split(';'):
+ if sattr:
+ attr = mapcolor(int(sattr), attr)
+ _kernel32.SetConsoleTextAttribute(stdout, attr)
+ writefunc(m.group(2), **opts)
+ m = re.match(ansire, m.group(3))
+ finally:
+ # Explicitly reset original attributes
+ _kernel32.SetConsoleTextAttribute(stdout, origattr)
--- a/mercurial/commands.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/commands.py Tue Feb 28 11:13:25 2017 -0800
@@ -11,17 +11,10 @@
import errno
import os
import re
-import socket
-import string
-import sys
-import tempfile
-import time
from .i18n import _
from .node import (
- bin,
hex,
- nullhex,
nullid,
nullrev,
short,
@@ -40,30 +33,22 @@
error,
exchange,
extensions,
- formatter,
graphmod,
hbisect,
help,
hg,
lock as lockmod,
merge as mergemod,
- minirst,
obsolete,
patch,
phases,
- policy,
- pvec,
pycompat,
- repair,
- revlog,
- revset,
+ revsetlang,
scmutil,
server,
sshserver,
- sslutil,
streamclone,
templatekw,
- templater,
ui as uimod,
util,
)
@@ -92,6 +77,12 @@
_('do not prompt, automatically pick the first choice for all prompts')),
('q', 'quiet', None, _('suppress output')),
('v', 'verbose', None, _('enable additional output')),
+ ('', 'color', '',
+ # i18n: 'always', 'auto', 'never', and 'debug' are keywords
+ # and should not be translated
+ _("when to colorize (boolean, always, auto, never, or debug)"
+ " (EXPERIMENTAL)"),
+ _('TYPE')),
('', 'config', [],
_('set/override config option (use \'section.name=value\')'),
_('CONFIG')),
@@ -107,6 +98,8 @@
('', 'version', None, _('output version information and exit')),
('h', 'help', None, _('display help and exit')),
('', 'hidden', False, _('consider hidden changesets')),
+ ('', 'pager', 'auto',
+ _("when to paginate (boolean, always, auto, or never)"), _('TYPE')),
]
dryrunopts = [('n', 'dry-run', None,
@@ -433,6 +426,8 @@
if linenumber and (not opts.get('changeset')) and (not opts.get('number')):
raise error.Abort(_('at least one of -n/-c is required for -l'))
+ ui.pager('annotate')
+
if fm.isplain():
def makefunc(get, fmt):
return lambda x: fmt(get(x))
@@ -1427,6 +1422,7 @@
ctx = scmutil.revsingle(repo, opts.get('rev'))
m = scmutil.match(ctx, (file1,) + pats, opts)
+ ui.pager('cat')
return cmdutil.cat(ui, repo, ctx, m, '', **opts)
@command('^clone',
@@ -1801,7 +1797,7 @@
ui.system("%s \"%s\"" % (editor, f),
onerr=error.Abort, errprefix=_("edit failed"))
return
-
+ ui.pager('config')
fm = ui.formatter('config', opts)
for f in scmutil.rcpath():
ui.debug('read config from: %s\n' % f)
@@ -1866,1176 +1862,6 @@
with repo.wlock(False):
return cmdutil.copy(ui, repo, pats, opts)
-@command('debuginstall', [] + formatteropts, '', norepo=True)
-def debuginstall(ui, **opts):
- '''test Mercurial installation
-
- Returns 0 on success.
- '''
-
- def writetemp(contents):
- (fd, name) = tempfile.mkstemp(prefix="hg-debuginstall-")
- f = os.fdopen(fd, "wb")
- f.write(contents)
- f.close()
- return name
-
- problems = 0
-
- fm = ui.formatter('debuginstall', opts)
- fm.startitem()
-
- # encoding
- fm.write('encoding', _("checking encoding (%s)...\n"), encoding.encoding)
- err = None
- try:
- encoding.fromlocal("test")
- except error.Abort as inst:
- err = inst
- problems += 1
- fm.condwrite(err, 'encodingerror', _(" %s\n"
- " (check that your locale is properly set)\n"), err)
-
- # Python
- fm.write('pythonexe', _("checking Python executable (%s)\n"),
- pycompat.sysexecutable)
- fm.write('pythonver', _("checking Python version (%s)\n"),
- ("%d.%d.%d" % sys.version_info[:3]))
- fm.write('pythonlib', _("checking Python lib (%s)...\n"),
- os.path.dirname(os.__file__))
-
- security = set(sslutil.supportedprotocols)
- if sslutil.hassni:
- security.add('sni')
-
- fm.write('pythonsecurity', _("checking Python security support (%s)\n"),
- fm.formatlist(sorted(security), name='protocol',
- fmt='%s', sep=','))
-
- # These are warnings, not errors. So don't increment problem count. This
- # may change in the future.
- if 'tls1.2' not in security:
- fm.plain(_(' TLS 1.2 not supported by Python install; '
- 'network connections lack modern security\n'))
- if 'sni' not in security:
- fm.plain(_(' SNI not supported by Python install; may have '
- 'connectivity issues with some servers\n'))
-
- # TODO print CA cert info
-
- # hg version
- hgver = util.version()
- fm.write('hgver', _("checking Mercurial version (%s)\n"),
- hgver.split('+')[0])
- fm.write('hgverextra', _("checking Mercurial custom build (%s)\n"),
- '+'.join(hgver.split('+')[1:]))
-
- # compiled modules
- fm.write('hgmodulepolicy', _("checking module policy (%s)\n"),
- policy.policy)
- fm.write('hgmodules', _("checking installed modules (%s)...\n"),
- os.path.dirname(__file__))
-
- err = None
- try:
- from . import (
- base85,
- bdiff,
- mpatch,
- osutil,
- )
- dir(bdiff), dir(mpatch), dir(base85), dir(osutil) # quiet pyflakes
- except Exception as inst:
- err = inst
- problems += 1
- fm.condwrite(err, 'extensionserror', " %s\n", err)
-
- compengines = util.compengines._engines.values()
- fm.write('compengines', _('checking registered compression engines (%s)\n'),
- fm.formatlist(sorted(e.name() for e in compengines),
- name='compengine', fmt='%s', sep=', '))
- fm.write('compenginesavail', _('checking available compression engines '
- '(%s)\n'),
- fm.formatlist(sorted(e.name() for e in compengines
- if e.available()),
- name='compengine', fmt='%s', sep=', '))
- wirecompengines = util.compengines.supportedwireengines(util.SERVERROLE)
- fm.write('compenginesserver', _('checking available compression engines '
- 'for wire protocol (%s)\n'),
- fm.formatlist([e.name() for e in wirecompengines
- if e.wireprotosupport()],
- name='compengine', fmt='%s', sep=', '))
-
- # templates
- p = templater.templatepaths()
- fm.write('templatedirs', 'checking templates (%s)...\n', ' '.join(p))
- fm.condwrite(not p, '', _(" no template directories found\n"))
- if p:
- m = templater.templatepath("map-cmdline.default")
- if m:
- # template found, check if it is working
- err = None
- try:
- templater.templater.frommapfile(m)
- except Exception as inst:
- err = inst
- p = None
- fm.condwrite(err, 'defaulttemplateerror', " %s\n", err)
- else:
- p = None
- fm.condwrite(p, 'defaulttemplate',
- _("checking default template (%s)\n"), m)
- fm.condwrite(not m, 'defaulttemplatenotfound',
- _(" template '%s' not found\n"), "default")
- if not p:
- problems += 1
- fm.condwrite(not p, '',
- _(" (templates seem to have been installed incorrectly)\n"))
-
- # editor
- editor = ui.geteditor()
- editor = util.expandpath(editor)
- fm.write('editor', _("checking commit editor... (%s)\n"), editor)
- cmdpath = util.findexe(pycompat.shlexsplit(editor)[0])
- fm.condwrite(not cmdpath and editor == 'vi', 'vinotfound',
- _(" No commit editor set and can't find %s in PATH\n"
- " (specify a commit editor in your configuration"
- " file)\n"), not cmdpath and editor == 'vi' and editor)
- fm.condwrite(not cmdpath and editor != 'vi', 'editornotfound',
- _(" Can't find editor '%s' in PATH\n"
- " (specify a commit editor in your configuration"
- " file)\n"), not cmdpath and editor)
- if not cmdpath and editor != 'vi':
- problems += 1
-
- # check username
- username = None
- err = None
- try:
- username = ui.username()
- except error.Abort as e:
- err = e
- problems += 1
-
- fm.condwrite(username, 'username', _("checking username (%s)\n"), username)
- fm.condwrite(err, 'usernameerror', _("checking username...\n %s\n"
- " (specify a username in your configuration file)\n"), err)
-
- fm.condwrite(not problems, '',
- _("no problems detected\n"))
- if not problems:
- fm.data(problems=problems)
- fm.condwrite(problems, 'problems',
- _("%d problems detected,"
- " please check your install!\n"), problems)
- fm.end()
-
- return problems
-
-@command('debugknown', [], _('REPO ID...'), norepo=True)
-def debugknown(ui, repopath, *ids, **opts):
- """test whether node ids are known to a repo
-
- Every ID must be a full-length hex node id string. Returns a list of 0s
- and 1s indicating unknown/known.
- """
- repo = hg.peer(ui, opts, repopath)
- if not repo.capable('known'):
- raise error.Abort("known() not supported by target repository")
- flags = repo.known([bin(s) for s in ids])
- ui.write("%s\n" % ("".join([f and "1" or "0" for f in flags])))
-
-@command('debuglabelcomplete', [], _('LABEL...'))
-def debuglabelcomplete(ui, repo, *args):
- '''backwards compatibility with old bash completion scripts (DEPRECATED)'''
- debugnamecomplete(ui, repo, *args)
-
-@command('debugmergestate', [], '')
-def debugmergestate(ui, repo, *args):
- """print merge state
-
- Use --verbose to print out information about whether v1 or v2 merge state
- was chosen."""
- def _hashornull(h):
- if h == nullhex:
- return 'null'
- else:
- return h
-
- def printrecords(version):
- ui.write(('* version %s records\n') % version)
- if version == 1:
- records = v1records
- else:
- records = v2records
-
- for rtype, record in records:
- # pretty print some record types
- if rtype == 'L':
- ui.write(('local: %s\n') % record)
- elif rtype == 'O':
- ui.write(('other: %s\n') % record)
- elif rtype == 'm':
- driver, mdstate = record.split('\0', 1)
- ui.write(('merge driver: %s (state "%s")\n')
- % (driver, mdstate))
- elif rtype in 'FDC':
- r = record.split('\0')
- f, state, hash, lfile, afile, anode, ofile = r[0:7]
- if version == 1:
- onode = 'not stored in v1 format'
- flags = r[7]
- else:
- onode, flags = r[7:9]
- ui.write(('file: %s (record type "%s", state "%s", hash %s)\n')
- % (f, rtype, state, _hashornull(hash)))
- ui.write((' local path: %s (flags "%s")\n') % (lfile, flags))
- ui.write((' ancestor path: %s (node %s)\n')
- % (afile, _hashornull(anode)))
- ui.write((' other path: %s (node %s)\n')
- % (ofile, _hashornull(onode)))
- elif rtype == 'f':
- filename, rawextras = record.split('\0', 1)
- extras = rawextras.split('\0')
- i = 0
- extrastrings = []
- while i < len(extras):
- extrastrings.append('%s = %s' % (extras[i], extras[i + 1]))
- i += 2
-
- ui.write(('file extras: %s (%s)\n')
- % (filename, ', '.join(extrastrings)))
- elif rtype == 'l':
- labels = record.split('\0', 2)
- labels = [l for l in labels if len(l) > 0]
- ui.write(('labels:\n'))
- ui.write((' local: %s\n' % labels[0]))
- ui.write((' other: %s\n' % labels[1]))
- if len(labels) > 2:
- ui.write((' base: %s\n' % labels[2]))
- else:
- ui.write(('unrecognized entry: %s\t%s\n')
- % (rtype, record.replace('\0', '\t')))
-
- # Avoid mergestate.read() since it may raise an exception for unsupported
- # merge state records. We shouldn't be doing this, but this is OK since this
- # command is pretty low-level.
- ms = mergemod.mergestate(repo)
-
- # sort so that reasonable information is on top
- v1records = ms._readrecordsv1()
- v2records = ms._readrecordsv2()
- order = 'LOml'
- def key(r):
- idx = order.find(r[0])
- if idx == -1:
- return (1, r[1])
- else:
- return (0, idx)
- v1records.sort(key=key)
- v2records.sort(key=key)
-
- if not v1records and not v2records:
- ui.write(('no merge state found\n'))
- elif not v2records:
- ui.note(('no version 2 merge state\n'))
- printrecords(1)
- elif ms._v1v2match(v1records, v2records):
- ui.note(('v1 and v2 states match: using v2\n'))
- printrecords(2)
- else:
- ui.note(('v1 and v2 states mismatch: using v1\n'))
- printrecords(1)
- if ui.verbose:
- printrecords(2)
-
-@command('debugnamecomplete', [], _('NAME...'))
-def debugnamecomplete(ui, repo, *args):
- '''complete "names" - tags, open branch names, bookmark names'''
-
- names = set()
- # since we previously only listed open branches, we will handle that
- # specially (after this for loop)
- for name, ns in repo.names.iteritems():
- if name != 'branches':
- names.update(ns.listnames(repo))
- names.update(tag for (tag, heads, tip, closed)
- in repo.branchmap().iterbranches() if not closed)
- completions = set()
- if not args:
- args = ['']
- for a in args:
- completions.update(n for n in names if n.startswith(a))
- ui.write('\n'.join(sorted(completions)))
- ui.write('\n')
-
-@command('debuglocks',
- [('L', 'force-lock', None, _('free the store lock (DANGEROUS)')),
- ('W', 'force-wlock', None,
- _('free the working state lock (DANGEROUS)'))],
- _('[OPTION]...'))
-def debuglocks(ui, repo, **opts):
- """show or modify state of locks
-
- By default, this command will show which locks are held. This
- includes the user and process holding the lock, the amount of time
- the lock has been held, and the machine name where the process is
- running if it's not local.
-
- Locks protect the integrity of Mercurial's data, so should be
- treated with care. System crashes or other interruptions may cause
- locks to not be properly released, though Mercurial will usually
- detect and remove such stale locks automatically.
-
- However, detecting stale locks may not always be possible (for
- instance, on a shared filesystem). Removing locks may also be
- blocked by filesystem permissions.
-
- Returns 0 if no locks are held.
-
- """
-
- if opts.get('force_lock'):
- repo.svfs.unlink('lock')
- if opts.get('force_wlock'):
- repo.vfs.unlink('wlock')
- if opts.get('force_lock') or opts.get('force_lock'):
- return 0
-
- now = time.time()
- held = 0
-
- def report(vfs, name, method):
- # this causes stale locks to get reaped for more accurate reporting
- try:
- l = method(False)
- except error.LockHeld:
- l = None
-
- if l:
- l.release()
- else:
- try:
- stat = vfs.lstat(name)
- age = now - stat.st_mtime
- user = util.username(stat.st_uid)
- locker = vfs.readlock(name)
- if ":" in locker:
- host, pid = locker.split(':')
- if host == socket.gethostname():
- locker = 'user %s, process %s' % (user, pid)
- else:
- locker = 'user %s, process %s, host %s' \
- % (user, pid, host)
- ui.write(("%-6s %s (%ds)\n") % (name + ":", locker, age))
- return 1
- except OSError as e:
- if e.errno != errno.ENOENT:
- raise
-
- ui.write(("%-6s free\n") % (name + ":"))
- return 0
-
- held += report(repo.svfs, "lock", repo.lock)
- held += report(repo.vfs, "wlock", repo.wlock)
-
- return held
-
-@command('debugobsolete',
- [('', 'flags', 0, _('markers flag')),
- ('', 'record-parents', False,
- _('record parent information for the precursor')),
- ('r', 'rev', [], _('display markers relevant to REV')),
- ('', 'index', False, _('display index of the marker')),
- ('', 'delete', [], _('delete markers specified by indices')),
- ] + commitopts2 + formatteropts,
- _('[OBSOLETED [REPLACEMENT ...]]'))
-def debugobsolete(ui, repo, precursor=None, *successors, **opts):
- """create arbitrary obsolete marker
-
- With no arguments, displays the list of obsolescence markers."""
-
- def parsenodeid(s):
- try:
- # We do not use revsingle/revrange functions here to accept
- # arbitrary node identifiers, possibly not present in the
- # local repository.
- n = bin(s)
- if len(n) != len(nullid):
- raise TypeError()
- return n
- except TypeError:
- raise error.Abort('changeset references must be full hexadecimal '
- 'node identifiers')
-
- if opts.get('delete'):
- indices = []
- for v in opts.get('delete'):
- try:
- indices.append(int(v))
- except ValueError:
- raise error.Abort(_('invalid index value: %r') % v,
- hint=_('use integers for indices'))
-
- if repo.currenttransaction():
- raise error.Abort(_('cannot delete obsmarkers in the middle '
- 'of transaction.'))
-
- with repo.lock():
- n = repair.deleteobsmarkers(repo.obsstore, indices)
- ui.write(_('deleted %i obsolescence markers\n') % n)
-
- return
-
- if precursor is not None:
- if opts['rev']:
- raise error.Abort('cannot select revision when creating marker')
- metadata = {}
- metadata['user'] = opts['user'] or ui.username()
- succs = tuple(parsenodeid(succ) for succ in successors)
- l = repo.lock()
- try:
- tr = repo.transaction('debugobsolete')
- try:
- date = opts.get('date')
- if date:
- date = util.parsedate(date)
- else:
- date = None
- prec = parsenodeid(precursor)
- parents = None
- if opts['record_parents']:
- if prec not in repo.unfiltered():
- raise error.Abort('cannot used --record-parents on '
- 'unknown changesets')
- parents = repo.unfiltered()[prec].parents()
- parents = tuple(p.node() for p in parents)
- repo.obsstore.create(tr, prec, succs, opts['flags'],
- parents=parents, date=date,
- metadata=metadata)
- tr.close()
- except ValueError as exc:
- raise error.Abort(_('bad obsmarker input: %s') % exc)
- finally:
- tr.release()
- finally:
- l.release()
- else:
- if opts['rev']:
- revs = scmutil.revrange(repo, opts['rev'])
- nodes = [repo[r].node() for r in revs]
- markers = list(obsolete.getmarkers(repo, nodes=nodes))
- markers.sort(key=lambda x: x._data)
- else:
- markers = obsolete.getmarkers(repo)
-
- markerstoiter = markers
- isrelevant = lambda m: True
- if opts.get('rev') and opts.get('index'):
- markerstoiter = obsolete.getmarkers(repo)
- markerset = set(markers)
- isrelevant = lambda m: m in markerset
-
- fm = ui.formatter('debugobsolete', opts)
- for i, m in enumerate(markerstoiter):
- if not isrelevant(m):
- # marker can be irrelevant when we're iterating over a set
- # of markers (markerstoiter) which is bigger than the set
- # of markers we want to display (markers)
- # this can happen if both --index and --rev options are
- # provided and thus we need to iterate over all of the markers
- # to get the correct indices, but only display the ones that
- # are relevant to --rev value
- continue
- fm.startitem()
- ind = i if opts.get('index') else None
- cmdutil.showmarker(fm, m, index=ind)
- fm.end()
-
-@command('debugpathcomplete',
- [('f', 'full', None, _('complete an entire path')),
- ('n', 'normal', None, _('show only normal files')),
- ('a', 'added', None, _('show only added files')),
- ('r', 'removed', None, _('show only removed files'))],
- _('FILESPEC...'))
-def debugpathcomplete(ui, repo, *specs, **opts):
- '''complete part or all of a tracked path
-
- This command supports shells that offer path name completion. It
- currently completes only files already known to the dirstate.
-
- Completion extends only to the next path segment unless
- --full is specified, in which case entire paths are used.'''
-
- def complete(path, acceptable):
- dirstate = repo.dirstate
- spec = os.path.normpath(os.path.join(pycompat.getcwd(), path))
- rootdir = repo.root + pycompat.ossep
- if spec != repo.root and not spec.startswith(rootdir):
- return [], []
- if os.path.isdir(spec):
- spec += '/'
- spec = spec[len(rootdir):]
- fixpaths = pycompat.ossep != '/'
- if fixpaths:
- spec = spec.replace(pycompat.ossep, '/')
- speclen = len(spec)
- fullpaths = opts['full']
- files, dirs = set(), set()
- adddir, addfile = dirs.add, files.add
- for f, st in dirstate.iteritems():
- if f.startswith(spec) and st[0] in acceptable:
- if fixpaths:
- f = f.replace('/', pycompat.ossep)
- if fullpaths:
- addfile(f)
- continue
- s = f.find(pycompat.ossep, speclen)
- if s >= 0:
- adddir(f[:s])
- else:
- addfile(f)
- return files, dirs
-
- acceptable = ''
- if opts['normal']:
- acceptable += 'nm'
- if opts['added']:
- acceptable += 'a'
- if opts['removed']:
- acceptable += 'r'
- cwd = repo.getcwd()
- if not specs:
- specs = ['.']
-
- files, dirs = set(), set()
- for spec in specs:
- f, d = complete(spec, acceptable or 'nmar')
- files.update(f)
- dirs.update(d)
- files.update(dirs)
- ui.write('\n'.join(repo.pathto(p, cwd) for p in sorted(files)))
- ui.write('\n')
-
-@command('debugpushkey', [], _('REPO NAMESPACE [KEY OLD NEW]'), norepo=True)
-def debugpushkey(ui, repopath, namespace, *keyinfo, **opts):
- '''access the pushkey key/value protocol
-
- With two args, list the keys in the given namespace.
-
- With five args, set a key to new if it currently is set to old.
- Reports success or failure.
- '''
-
- target = hg.peer(ui, {}, repopath)
- if keyinfo:
- key, old, new = keyinfo
- r = target.pushkey(namespace, key, old, new)
- ui.status(str(r) + '\n')
- return not r
- else:
- for k, v in sorted(target.listkeys(namespace).iteritems()):
- ui.write("%s\t%s\n" % (k.encode('string-escape'),
- v.encode('string-escape')))
-
-@command('debugpvec', [], _('A B'))
-def debugpvec(ui, repo, a, b=None):
- ca = scmutil.revsingle(repo, a)
- cb = scmutil.revsingle(repo, b)
- pa = pvec.ctxpvec(ca)
- pb = pvec.ctxpvec(cb)
- if pa == pb:
- rel = "="
- elif pa > pb:
- rel = ">"
- elif pa < pb:
- rel = "<"
- elif pa | pb:
- rel = "|"
- ui.write(_("a: %s\n") % pa)
- ui.write(_("b: %s\n") % pb)
- ui.write(_("depth(a): %d depth(b): %d\n") % (pa._depth, pb._depth))
- ui.write(_("delta: %d hdist: %d distance: %d relation: %s\n") %
- (abs(pa._depth - pb._depth), pvec._hamming(pa._vec, pb._vec),
- pa.distance(pb), rel))
-
-@command('debugrebuilddirstate|debugrebuildstate',
- [('r', 'rev', '', _('revision to rebuild to'), _('REV')),
- ('', 'minimal', None, _('only rebuild files that are inconsistent with '
- 'the working copy parent')),
- ],
- _('[-r REV]'))
-def debugrebuilddirstate(ui, repo, rev, **opts):
- """rebuild the dirstate as it would look like for the given revision
-
- If no revision is specified the first current parent will be used.
-
- The dirstate will be set to the files of the given revision.
- The actual working directory content or existing dirstate
- information such as adds or removes is not considered.
-
- ``minimal`` will only rebuild the dirstate status for files that claim to be
- tracked but are not in the parent manifest, or that exist in the parent
- manifest but are not in the dirstate. It will not change adds, removes, or
- modified files that are in the working copy parent.
-
- One use of this command is to make the next :hg:`status` invocation
- check the actual file content.
- """
- ctx = scmutil.revsingle(repo, rev)
- with repo.wlock():
- dirstate = repo.dirstate
- changedfiles = None
- # See command doc for what minimal does.
- if opts.get('minimal'):
- manifestfiles = set(ctx.manifest().keys())
- dirstatefiles = set(dirstate)
- manifestonly = manifestfiles - dirstatefiles
- dsonly = dirstatefiles - manifestfiles
- dsnotadded = set(f for f in dsonly if dirstate[f] != 'a')
- changedfiles = manifestonly | dsnotadded
-
- dirstate.rebuild(ctx.node(), ctx.manifest(), changedfiles)
-
-@command('debugrebuildfncache', [], '')
-def debugrebuildfncache(ui, repo):
- """rebuild the fncache file"""
- repair.rebuildfncache(ui, repo)
-
-@command('debugrename',
- [('r', 'rev', '', _('revision to debug'), _('REV'))],
- _('[-r REV] FILE'))
-def debugrename(ui, repo, file1, *pats, **opts):
- """dump rename information"""
-
- ctx = scmutil.revsingle(repo, opts.get('rev'))
- m = scmutil.match(ctx, (file1,) + pats, opts)
- for abs in ctx.walk(m):
- fctx = ctx[abs]
- o = fctx.filelog().renamed(fctx.filenode())
- rel = m.rel(abs)
- if o:
- ui.write(_("%s renamed from %s:%s\n") % (rel, o[0], hex(o[1])))
- else:
- ui.write(_("%s not renamed\n") % rel)
-
-@command('debugrevlog', debugrevlogopts +
- [('d', 'dump', False, _('dump index data'))],
- _('-c|-m|FILE'),
- optionalrepo=True)
-def debugrevlog(ui, repo, file_=None, **opts):
- """show data and statistics about a revlog"""
- r = cmdutil.openrevlog(repo, 'debugrevlog', file_, opts)
-
- if opts.get("dump"):
- numrevs = len(r)
- ui.write(("# rev p1rev p2rev start end deltastart base p1 p2"
- " rawsize totalsize compression heads chainlen\n"))
- ts = 0
- heads = set()
-
- for rev in xrange(numrevs):
- dbase = r.deltaparent(rev)
- if dbase == -1:
- dbase = rev
- cbase = r.chainbase(rev)
- clen = r.chainlen(rev)
- p1, p2 = r.parentrevs(rev)
- rs = r.rawsize(rev)
- ts = ts + rs
- heads -= set(r.parentrevs(rev))
- heads.add(rev)
- try:
- compression = ts / r.end(rev)
- except ZeroDivisionError:
- compression = 0
- ui.write("%5d %5d %5d %5d %5d %10d %4d %4d %4d %7d %9d "
- "%11d %5d %8d\n" %
- (rev, p1, p2, r.start(rev), r.end(rev),
- r.start(dbase), r.start(cbase),
- r.start(p1), r.start(p2),
- rs, ts, compression, len(heads), clen))
- return 0
-
- v = r.version
- format = v & 0xFFFF
- flags = []
- gdelta = False
- if v & revlog.REVLOGNGINLINEDATA:
- flags.append('inline')
- if v & revlog.REVLOGGENERALDELTA:
- gdelta = True
- flags.append('generaldelta')
- if not flags:
- flags = ['(none)']
-
- nummerges = 0
- numfull = 0
- numprev = 0
- nump1 = 0
- nump2 = 0
- numother = 0
- nump1prev = 0
- nump2prev = 0
- chainlengths = []
-
- datasize = [None, 0, 0]
- fullsize = [None, 0, 0]
- deltasize = [None, 0, 0]
- chunktypecounts = {}
- chunktypesizes = {}
-
- def addsize(size, l):
- if l[0] is None or size < l[0]:
- l[0] = size
- if size > l[1]:
- l[1] = size
- l[2] += size
-
- numrevs = len(r)
- for rev in xrange(numrevs):
- p1, p2 = r.parentrevs(rev)
- delta = r.deltaparent(rev)
- if format > 0:
- addsize(r.rawsize(rev), datasize)
- if p2 != nullrev:
- nummerges += 1
- size = r.length(rev)
- if delta == nullrev:
- chainlengths.append(0)
- numfull += 1
- addsize(size, fullsize)
- else:
- chainlengths.append(chainlengths[delta] + 1)
- addsize(size, deltasize)
- if delta == rev - 1:
- numprev += 1
- if delta == p1:
- nump1prev += 1
- elif delta == p2:
- nump2prev += 1
- elif delta == p1:
- nump1 += 1
- elif delta == p2:
- nump2 += 1
- elif delta != nullrev:
- numother += 1
-
- # Obtain data on the raw chunks in the revlog.
- chunk = r._chunkraw(rev, rev)[1]
- if chunk:
- chunktype = chunk[0]
- else:
- chunktype = 'empty'
-
- if chunktype not in chunktypecounts:
- chunktypecounts[chunktype] = 0
- chunktypesizes[chunktype] = 0
-
- chunktypecounts[chunktype] += 1
- chunktypesizes[chunktype] += size
-
- # Adjust size min value for empty cases
- for size in (datasize, fullsize, deltasize):
- if size[0] is None:
- size[0] = 0
-
- numdeltas = numrevs - numfull
- numoprev = numprev - nump1prev - nump2prev
- totalrawsize = datasize[2]
- datasize[2] /= numrevs
- fulltotal = fullsize[2]
- fullsize[2] /= numfull
- deltatotal = deltasize[2]
- if numrevs - numfull > 0:
- deltasize[2] /= numrevs - numfull
- totalsize = fulltotal + deltatotal
- avgchainlen = sum(chainlengths) / numrevs
- maxchainlen = max(chainlengths)
- compratio = 1
- if totalsize:
- compratio = totalrawsize / totalsize
-
- basedfmtstr = '%%%dd\n'
- basepcfmtstr = '%%%dd %s(%%5.2f%%%%)\n'
-
- def dfmtstr(max):
- return basedfmtstr % len(str(max))
- def pcfmtstr(max, padding=0):
- return basepcfmtstr % (len(str(max)), ' ' * padding)
-
- def pcfmt(value, total):
- if total:
- return (value, 100 * float(value) / total)
- else:
- return value, 100.0
-
- ui.write(('format : %d\n') % format)
- ui.write(('flags : %s\n') % ', '.join(flags))
-
- ui.write('\n')
- fmt = pcfmtstr(totalsize)
- fmt2 = dfmtstr(totalsize)
- ui.write(('revisions : ') + fmt2 % numrevs)
- ui.write((' merges : ') + fmt % pcfmt(nummerges, numrevs))
- ui.write((' normal : ') + fmt % pcfmt(numrevs - nummerges, numrevs))
- ui.write(('revisions : ') + fmt2 % numrevs)
- ui.write((' full : ') + fmt % pcfmt(numfull, numrevs))
- ui.write((' deltas : ') + fmt % pcfmt(numdeltas, numrevs))
- ui.write(('revision size : ') + fmt2 % totalsize)
- ui.write((' full : ') + fmt % pcfmt(fulltotal, totalsize))
- ui.write((' deltas : ') + fmt % pcfmt(deltatotal, totalsize))
-
- def fmtchunktype(chunktype):
- if chunktype == 'empty':
- return ' %s : ' % chunktype
- elif chunktype in string.ascii_letters:
- return ' 0x%s (%s) : ' % (hex(chunktype), chunktype)
- else:
- return ' 0x%s : ' % hex(chunktype)
-
- ui.write('\n')
- ui.write(('chunks : ') + fmt2 % numrevs)
- for chunktype in sorted(chunktypecounts):
- ui.write(fmtchunktype(chunktype))
- ui.write(fmt % pcfmt(chunktypecounts[chunktype], numrevs))
- ui.write(('chunks size : ') + fmt2 % totalsize)
- for chunktype in sorted(chunktypecounts):
- ui.write(fmtchunktype(chunktype))
- ui.write(fmt % pcfmt(chunktypesizes[chunktype], totalsize))
-
- ui.write('\n')
- fmt = dfmtstr(max(avgchainlen, compratio))
- ui.write(('avg chain length : ') + fmt % avgchainlen)
- ui.write(('max chain length : ') + fmt % maxchainlen)
- ui.write(('compression ratio : ') + fmt % compratio)
-
- if format > 0:
- ui.write('\n')
- ui.write(('uncompressed data size (min/max/avg) : %d / %d / %d\n')
- % tuple(datasize))
- ui.write(('full revision size (min/max/avg) : %d / %d / %d\n')
- % tuple(fullsize))
- ui.write(('delta size (min/max/avg) : %d / %d / %d\n')
- % tuple(deltasize))
-
- if numdeltas > 0:
- ui.write('\n')
- fmt = pcfmtstr(numdeltas)
- fmt2 = pcfmtstr(numdeltas, 4)
- ui.write(('deltas against prev : ') + fmt % pcfmt(numprev, numdeltas))
- if numprev > 0:
- ui.write((' where prev = p1 : ') + fmt2 % pcfmt(nump1prev,
- numprev))
- ui.write((' where prev = p2 : ') + fmt2 % pcfmt(nump2prev,
- numprev))
- ui.write((' other : ') + fmt2 % pcfmt(numoprev,
- numprev))
- if gdelta:
- ui.write(('deltas against p1 : ')
- + fmt % pcfmt(nump1, numdeltas))
- ui.write(('deltas against p2 : ')
- + fmt % pcfmt(nump2, numdeltas))
- ui.write(('deltas against other : ') + fmt % pcfmt(numother,
- numdeltas))
-
-@command('debugrevspec',
- [('', 'optimize', None,
- _('print parsed tree after optimizing (DEPRECATED)')),
- ('p', 'show-stage', [],
- _('print parsed tree at the given stage'), _('NAME')),
- ('', 'no-optimized', False, _('evaluate tree without optimization')),
- ('', 'verify-optimized', False, _('verify optimized result')),
- ],
- ('REVSPEC'))
-def debugrevspec(ui, repo, expr, **opts):
- """parse and apply a revision specification
-
- Use -p/--show-stage option to print the parsed tree at the given stages.
- Use -p all to print tree at every stage.
-
- Use --verify-optimized to compare the optimized result with the unoptimized
- one. Returns 1 if the optimized result differs.
- """
- stages = [
- ('parsed', lambda tree: tree),
- ('expanded', lambda tree: revset.expandaliases(ui, tree)),
- ('concatenated', revset.foldconcat),
- ('analyzed', revset.analyze),
- ('optimized', revset.optimize),
- ]
- if opts['no_optimized']:
- stages = stages[:-1]
- if opts['verify_optimized'] and opts['no_optimized']:
- raise error.Abort(_('cannot use --verify-optimized with '
- '--no-optimized'))
- stagenames = set(n for n, f in stages)
-
- showalways = set()
- showchanged = set()
- if ui.verbose and not opts['show_stage']:
- # show parsed tree by --verbose (deprecated)
- showalways.add('parsed')
- showchanged.update(['expanded', 'concatenated'])
- if opts['optimize']:
- showalways.add('optimized')
- if opts['show_stage'] and opts['optimize']:
- raise error.Abort(_('cannot use --optimize with --show-stage'))
- if opts['show_stage'] == ['all']:
- showalways.update(stagenames)
- else:
- for n in opts['show_stage']:
- if n not in stagenames:
- raise error.Abort(_('invalid stage name: %s') % n)
- showalways.update(opts['show_stage'])
-
- treebystage = {}
- printedtree = None
- tree = revset.parse(expr, lookup=repo.__contains__)
- for n, f in stages:
- treebystage[n] = tree = f(tree)
- if n in showalways or (n in showchanged and tree != printedtree):
- if opts['show_stage'] or n != 'parsed':
- ui.write(("* %s:\n") % n)
- ui.write(revset.prettyformat(tree), "\n")
- printedtree = tree
-
- if opts['verify_optimized']:
- arevs = revset.makematcher(treebystage['analyzed'])(repo)
- brevs = revset.makematcher(treebystage['optimized'])(repo)
- if ui.verbose:
- ui.note(("* analyzed set:\n"), revset.prettyformatset(arevs), "\n")
- ui.note(("* optimized set:\n"), revset.prettyformatset(brevs), "\n")
- arevs = list(arevs)
- brevs = list(brevs)
- if arevs == brevs:
- return 0
- ui.write(('--- analyzed\n'), label='diff.file_a')
- ui.write(('+++ optimized\n'), label='diff.file_b')
- sm = difflib.SequenceMatcher(None, arevs, brevs)
- for tag, alo, ahi, blo, bhi in sm.get_opcodes():
- if tag in ('delete', 'replace'):
- for c in arevs[alo:ahi]:
- ui.write('-%s\n' % c, label='diff.deleted')
- if tag in ('insert', 'replace'):
- for c in brevs[blo:bhi]:
- ui.write('+%s\n' % c, label='diff.inserted')
- if tag == 'equal':
- for c in arevs[alo:ahi]:
- ui.write(' %s\n' % c)
- return 1
-
- func = revset.makematcher(tree)
- revs = func(repo)
- if ui.verbose:
- ui.note(("* set:\n"), revset.prettyformatset(revs), "\n")
- for c in revs:
- ui.write("%s\n" % c)
-
-@command('debugsetparents', [], _('REV1 [REV2]'))
-def debugsetparents(ui, repo, rev1, rev2=None):
- """manually set the parents of the current working directory
-
- This is useful for writing repository conversion tools, but should
- be used with care. For example, neither the working directory nor the
- dirstate is updated, so file status may be incorrect after running this
- command.
-
- Returns 0 on success.
- """
-
- r1 = scmutil.revsingle(repo, rev1).node()
- r2 = scmutil.revsingle(repo, rev2, 'null').node()
-
- with repo.wlock():
- repo.setparents(r1, r2)
-
-@command('debugdirstate|debugstate',
- [('', 'nodates', None, _('do not display the saved mtime')),
- ('', 'datesort', None, _('sort by saved mtime'))],
- _('[OPTION]...'))
-def debugstate(ui, repo, **opts):
- """show the contents of the current dirstate"""
-
- nodates = opts.get('nodates')
- datesort = opts.get('datesort')
-
- timestr = ""
- if datesort:
- keyfunc = lambda x: (x[1][3], x[0]) # sort by mtime, then by filename
- else:
- keyfunc = None # sort by filename
- for file_, ent in sorted(repo.dirstate._map.iteritems(), key=keyfunc):
- if ent[3] == -1:
- timestr = 'unset '
- elif nodates:
- timestr = 'set '
- else:
- timestr = time.strftime("%Y-%m-%d %H:%M:%S ",
- time.localtime(ent[3]))
- if ent[1] & 0o20000:
- mode = 'lnk'
- else:
- mode = '%3o' % (ent[1] & 0o777 & ~util.umask)
- ui.write("%c %s %10d %s%s\n" % (ent[0], mode, ent[2], timestr, file_))
- for f in repo.dirstate.copies():
- ui.write(_("copy: %s -> %s\n") % (repo.dirstate.copied(f), f))
-
-@command('debugsub',
- [('r', 'rev', '',
- _('revision to check'), _('REV'))],
- _('[-r REV] [REV]'))
-def debugsub(ui, repo, rev=None):
- ctx = scmutil.revsingle(repo, rev, None)
- for k, v in sorted(ctx.substate.items()):
- ui.write(('path %s\n') % k)
- ui.write((' source %s\n') % v[0])
- ui.write((' revision %s\n') % v[1])
-
-@command('debugsuccessorssets',
- [],
- _('[REV]'))
-def debugsuccessorssets(ui, repo, *revs):
- """show set of successors for revision
-
- A successors set of changeset A is a consistent group of revisions that
- succeed A. It contains non-obsolete changesets only.
-
- In most cases a changeset A has a single successors set containing a single
- successor (changeset A replaced by A').
-
- A changeset that is made obsolete with no successors are called "pruned".
- Such changesets have no successors sets at all.
-
- A changeset that has been "split" will have a successors set containing
- more than one successor.
-
- A changeset that has been rewritten in multiple different ways is called
- "divergent". Such changesets have multiple successor sets (each of which
- may also be split, i.e. have multiple successors).
-
- Results are displayed as follows::
-
- <rev1>
- <successors-1A>
- <rev2>
- <successors-2A>
- <successors-2B1> <successors-2B2> <successors-2B3>
-
- Here rev2 has two possible (i.e. divergent) successors sets. The first
- holds one element, whereas the second holds three (i.e. the changeset has
- been split).
- """
- # passed to successorssets caching computation from one call to another
- cache = {}
- ctx2str = str
- node2str = short
- if ui.debug():
- def ctx2str(ctx):
- return ctx.hex()
- node2str = hex
- for rev in scmutil.revrange(repo, revs):
- ctx = repo[rev]
- ui.write('%s\n'% ctx2str(ctx))
- for succsset in obsolete.successorssets(repo, ctx.node(), cache):
- if succsset:
- ui.write(' ')
- ui.write(node2str(succsset[0]))
- for node in succsset[1:]:
- ui.write(' ')
- ui.write(node2str(node))
- ui.write('\n')
-
-@command('debugtemplate',
- [('r', 'rev', [], _('apply template on changesets'), _('REV')),
- ('D', 'define', [], _('define template keyword'), _('KEY=VALUE'))],
- _('[-r REV]... [-D KEY=VALUE]... TEMPLATE'),
- optionalrepo=True)
-def debugtemplate(ui, repo, tmpl, **opts):
- """parse and apply a template
-
- If -r/--rev is given, the template is processed as a log template and
- applied to the given changesets. Otherwise, it is processed as a generic
- template.
-
- Use --verbose to print the parsed tree.
- """
- revs = None
- if opts['rev']:
- if repo is None:
- raise error.RepoError(_('there is no Mercurial repository here '
- '(.hg not found)'))
- revs = scmutil.revrange(repo, opts['rev'])
-
- props = {}
- for d in opts['define']:
- try:
- k, v = (e.strip() for e in d.split('=', 1))
- if not k:
- raise ValueError
- props[k] = v
- except ValueError:
- raise error.Abort(_('malformed keyword definition: %s') % d)
-
- if ui.verbose:
- aliases = ui.configitems('templatealias')
- tree = templater.parse(tmpl)
- ui.note(templater.prettyformat(tree), '\n')
- newtree = templater.expandaliases(tree, aliases)
- if newtree != tree:
- ui.note(("* expanded:\n"), templater.prettyformat(newtree), '\n')
-
- mapfile = None
- if revs is None:
- k = 'debugtemplate'
- t = formatter.maketemplater(ui, k, tmpl)
- ui.write(templater.stringify(t(k, **props)))
- else:
- displayer = cmdutil.changeset_templater(ui, repo, None, opts, tmpl,
- mapfile, buffered=False)
- for r in revs:
- displayer.show(repo[r], **props)
- displayer.close()
-
-@command('debugwalk', walkopts, _('[OPTION]... [FILE]...'), inferrepo=True)
-def debugwalk(ui, repo, *pats, **opts):
- """show how files match on given patterns"""
- m = scmutil.match(repo[None], pats, opts)
- items = list(repo.walk(m))
- if not items:
- return
- f = lambda fn: fn
- if ui.configbool('ui', 'slash') and pycompat.ossep != '/':
- f = lambda fn: util.normpath(fn)
- fmt = 'f %%-%ds %%-%ds %%s' % (
- max([len(abs) for abs in items]),
- max([len(m.rel(abs)) for abs in items]))
- for abs in items:
- line = fmt % (abs, f(m.rel(abs)), m.exact(abs) and 'exact' or '')
- ui.write("%s\n" % line.rstrip())
-
-@command('debugwireargs',
- [('', 'three', '', 'three'),
- ('', 'four', '', 'four'),
- ('', 'five', '', 'five'),
- ] + remoteopts,
- _('REPO [OPTIONS]... [ONE [TWO]]'),
- norepo=True)
-def debugwireargs(ui, repopath, *vals, **opts):
- repo = hg.peer(ui, opts, repopath)
- for opt in remoteopts:
- del opts[opt[1]]
- args = {}
- for k, v in opts.iteritems():
- if v:
- args[k] = v
- # run twice to check that we don't mess up the stream for the next command
- res1 = repo.debugwireargs(*vals, **args)
- res2 = repo.debugwireargs(*vals, **args)
- ui.write("%s\n" % res1)
- if res1 != res2:
- ui.warn("%s\n" % res2)
-
@command('^diff',
[('r', 'rev', [], _('revision'), _('REV')),
('c', 'change', '', _('change made by revision'), _('REV'))
@@ -3119,6 +1945,7 @@
diffopts = patch.diffallopts(ui, opts)
m = scmutil.match(repo[node2], pats, opts)
+ ui.pager('diff')
cmdutil.diffordiffstat(ui, repo, diffopts, node1, node2, m, stat=stat,
listsubrepos=opts.get('subrepos'),
root=opts.get('root'))
@@ -3200,6 +2027,7 @@
ui.note(_('exporting patches:\n'))
else:
ui.note(_('exporting patch:\n'))
+ ui.pager('export')
cmdutil.export(repo, revs, template=opts.get('output'),
switch_parent=opts.get('switch_parent'),
opts=patch.diffallopts(ui, opts))
@@ -3261,6 +2089,7 @@
fmt = '%s' + end
m = scmutil.match(ctx, pats, opts)
+ ui.pager('files')
with ui.formatter('files', opts) as fm:
return cmdutil.files(ui, ctx, m, fm, fmt, opts.get('subrepos'))
@@ -3782,6 +2611,7 @@
except error.LookupError:
pass
+ ui.pager('grep')
fm = ui.formatter('grep', opts)
for ctx in cmdutil.walkchangerevs(repo, matchfn, opts, prep):
rev = ctx.rev()
@@ -3897,11 +2727,6 @@
Returns 0 if successful.
"""
- textwidth = ui.configint('ui', 'textwidth', 78)
- termwidth = ui.termwidth() - 2
- if textwidth <= 0 or termwidth < textwidth:
- textwidth = termwidth
-
keep = opts.get('system') or []
if len(keep) == 0:
if pycompat.sysplatform.startswith('win'):
@@ -3916,36 +2741,8 @@
if ui.verbose:
keep.append('verbose')
- section = None
- subtopic = None
- if name and '.' in name:
- name, remaining = name.split('.', 1)
- remaining = encoding.lower(remaining)
- if '.' in remaining:
- subtopic, section = remaining.split('.', 1)
- else:
- if name in help.subtopics:
- subtopic = remaining
- else:
- section = remaining
-
- text = help.help_(ui, name, subtopic=subtopic, **opts)
-
- formatted, pruned = minirst.format(text, textwidth, keep=keep,
- section=section)
-
- # We could have been given a weird ".foo" section without a name
- # to look for, or we could have simply failed to found "foo.bar"
- # because bar isn't a section of foo
- if section and not (formatted and name):
- raise error.Abort(_("help section not found"))
-
- if 'verbose' in pruned:
- keep.append('omitted')
- else:
- keep.append('notomitted')
- formatted, pruned = minirst.format(text, textwidth, keep=keep,
- section=section)
+ formatted = help.formattedhelp(ui, name, keep=keep, **opts)
+ ui.pager('help')
ui.write(formatted)
@@ -4127,8 +2924,9 @@
Import a list of patches and commit them individually (unless
--no-commit is specified).
- To read a patch from standard input, use "-" as the patch name. If
- a URL is specified, the patch will be downloaded from there.
+ To read a patch from standard input (stdin), use "-" as the patch
+ name. If a URL is specified, the patch will be downloaded from
+ there.
Import first applies changes to the working directory (unless
--bypass is specified), import will abort if there are outstanding
@@ -4198,6 +2996,10 @@
hg import incoming-patches.mbox
+ - import patches from stdin::
+
+ hg import -
+
- attempt to exactly restore an exported changeset (not always
possible)::
@@ -4392,6 +3194,7 @@
if 'bookmarks' not in other.listkeys('namespaces'):
ui.warn(_("remote doesn't support bookmarks\n"))
return 0
+ ui.pager('incoming')
ui.status(_('comparing with %s\n') % util.hidepassword(source))
return bookmarks.incoming(ui, repo, other)
@@ -4458,6 +3261,7 @@
m = scmutil.match(ctx, pats, opts, default='relglob',
badfn=lambda x, y: False)
+ ui.pager('locate')
for abs in ctx.matches(m):
if opts.get('fullpath'):
ui.write(repo.wjoin(abs), end)
@@ -4589,7 +3393,7 @@
"""
if opts.get('follow') and opts.get('rev'):
- opts['rev'] = [revset.formatspec('reverse(::%lr)', opts.get('rev'))]
+ opts['rev'] = [revsetlang.formatspec('reverse(::%lr)', opts.get('rev'))]
del opts['follow']
if opts.get('graph'):
@@ -4606,6 +3410,7 @@
endrev = scmutil.revrange(repo, opts.get('rev')).max() + 1
getrenamed = templatekw.getrenamedfn(repo, endrev=endrev)
+ ui.pager('log')
displayer = cmdutil.show_changeset(ui, repo, opts, buffered=True)
for rev in revs:
if count == limit:
@@ -4648,7 +3453,6 @@
Returns 0 on success.
"""
-
fm = ui.formatter('manifest', opts)
if opts.get('all'):
@@ -4664,6 +3468,7 @@
for fn, b, size in repo.store.datafiles():
if size != 0 and fn[-slen:] == suffix and fn[:plen] == prefix:
res.append(fn[plen:-slen])
+ ui.pager('manifest')
for f in res:
fm.startitem()
fm.write("path", '%s\n', f)
@@ -4680,6 +3485,7 @@
mode = {'l': '644', 'x': '755', '': '644'}
ctx = scmutil.revsingle(repo, node)
mf = ctx.manifest()
+ ui.pager('manifest')
for f in ctx:
fm.startitem()
fl = ctx[f].flags()
@@ -4812,6 +3618,7 @@
return
revdag = cmdutil.graphrevs(repo, o, opts)
+ ui.pager('outgoing')
displayer = cmdutil.show_changeset(ui, repo, opts, buffered=True)
cmdutil.displaygraph(ui, repo, revdag, displayer, graphmod.asciiedges)
cmdutil.outgoinghooks(ui, repo, other, opts, o)
@@ -4825,6 +3632,7 @@
ui.warn(_("remote doesn't support bookmarks\n"))
return 0
ui.status(_('comparing with %s\n') % util.hidepassword(dest))
+ ui.pager('outgoing')
return bookmarks.outgoing(ui, repo, other)
repo._subtoppath = ui.expandpath(dest or 'default-push', dest or 'default')
@@ -4921,6 +3729,7 @@
Returns 0 on success.
"""
+ ui.pager('paths')
if search:
pathitems = [(name, path) for name, path in ui.paths.iteritems()
if name == search]
@@ -5268,7 +4077,7 @@
elif path.pushrev:
# It doesn't make any sense to specify ancestor revisions. So limit
# to DAG heads to make discovery simpler.
- expr = revset.formatspec('heads(%r)', path.pushrev)
+ expr = revsetlang.formatspec('heads(%r)', path.pushrev)
revs = scmutil.revrange(repo, [expr])
revs = [repo[rev].node() for rev in revs]
if not revs:
@@ -5434,6 +4243,8 @@
- :hg:`resolve -l`: list files which had or still have conflicts.
In the printed list, ``U`` = unresolved and ``R`` = resolved.
+ You can use ``set:unresolved()`` or ``set:resolved()`` to filter
+ the list. See :hg:`help filesets` for details.
.. note::
@@ -5457,6 +4268,7 @@
hint=('use --all to re-merge all unresolved files'))
if show:
+ ui.pager('resolve')
fm = ui.formatter('resolve', opts)
ms = mergemod.mergestate.read(repo)
m = scmutil.match(repo[None], pats, opts)
@@ -5780,8 +4592,8 @@
('', 'webdir-conf', '', _('name of the hgweb config file (DEPRECATED)'),
_('FILE')),
('', 'pid-file', '', _('name of file to write process ID to'), _('FILE')),
- ('', 'stdio', None, _('for remote clients')),
- ('', 'cmdserver', '', _('for remote clients'), _('MODE')),
+ ('', 'stdio', None, _('for remote clients (ADVANCED)')),
+ ('', 'cmdserver', '', _('for remote clients (ADVANCED)'), _('MODE')),
('t', 'templates', '', _('web templates to use'), _('TEMPLATE')),
('', 'style', '', _('template style to use'), _('STYLE')),
('6', 'ipv6', None, _('use IPv6 in addition to IPv4')),
@@ -5946,6 +4758,7 @@
or ui.configbool('ui', 'statuscopies')) and not opts.get('no_status'):
copy = copies.pathcopies(repo[node1], repo[node2], m)
+ ui.pager('status')
fm = ui.formatter('status', opts)
fmt = '%s' + end
showchar = not opts.get('no_status')
@@ -5976,6 +4789,7 @@
Returns 0 on success.
"""
+ ui.pager('summary')
ctx = repo[None]
parents = ctx.parents()
pnode = parents[0].node()
@@ -6368,6 +5182,7 @@
Returns 0 on success.
"""
+ ui.pager('tags')
fm = ui.formatter('tags', opts)
hexfunc = fm.hexfunc
tagtype = ""
@@ -6467,7 +5282,7 @@
('d', 'date', '', _('tipmost revision matching date'), _('DATE')),
('r', 'rev', '', _('revision'), _('REV'))
] + mergetoolopts,
- _('[-c] [-C] [-d DATE] [[-r] REV]'))
+ _('[-C|-c] [-d DATE] [[-r] REV]'))
def update(ui, repo, node=None, rev=None, clean=False, date=None, check=False,
tool=None):
"""update working directory (or switch revisions)
@@ -6488,10 +5303,11 @@
.. container:: verbose
- The following rules apply when the working directory contains
- uncommitted changes:
-
- 1. If neither -c/--check nor -C/--clean is specified, and if
+ The -C/--clean and -c/--check options control what happens if the
+ working directory contains uncommitted changes.
+ At most of one of them can be specified.
+
+ 1. If no option is specified, and if
the requested changeset is an ancestor or descendant of
the working directory's parent, the uncommitted changes
are merged into the requested changeset and the merged
@@ -6541,9 +5357,6 @@
brev = rev
rev = scmutil.revsingle(repo, rev, rev).rev()
- if check:
- cmdutil.bailifchanged(repo, merge=False)
-
repo.ui.setconfig('ui', 'forcemerge', tool, 'update')
return hg.updatetotally(ui, repo, rev, brev, clean=clean, check=check)
@@ -6570,6 +5383,8 @@
@command('version', [] + formatteropts, norepo=True)
def version_(ui, **opts):
"""output version and copyright information"""
+ if ui.verbose:
+ ui.pager('version')
fm = ui.formatter("version", opts)
fm.startitem()
fm.write("ver", _("Mercurial Distributed SCM (version %s)\n"),
--- a/mercurial/commandserver.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/commandserver.py Tue Feb 28 11:13:25 2017 -0800
@@ -304,8 +304,8 @@
ui.flush()
newfiles = []
nullfd = os.open(os.devnull, os.O_RDWR)
- for f, sysf, mode in [(ui.fin, util.stdin, 'rb'),
- (ui.fout, util.stdout, 'wb')]:
+ for f, sysf, mode in [(ui.fin, util.stdin, pycompat.sysstr('rb')),
+ (ui.fout, util.stdout, pycompat.sysstr('wb'))]:
if f is sysf:
newfd = os.dup(f.fileno())
os.dup2(nullfd, f.fileno())
@@ -447,6 +447,7 @@
self._sock = None
self._oldsigchldhandler = None
self._workerpids = set() # updated by signal handler; do not iterate
+ self._socketunlinked = None
def init(self):
self._sock = socket.socket(socket.AF_UNIX)
@@ -455,11 +456,17 @@
o = signal.signal(signal.SIGCHLD, self._sigchldhandler)
self._oldsigchldhandler = o
self._servicehandler.printbanner(self.address)
+ self._socketunlinked = False
+
+ def _unlinksocket(self):
+ if not self._socketunlinked:
+ self._servicehandler.unlinksocket(self.address)
+ self._socketunlinked = True
def _cleanup(self):
signal.signal(signal.SIGCHLD, self._oldsigchldhandler)
self._sock.close()
- self._servicehandler.unlinksocket(self.address)
+ self._unlinksocket()
# don't kill child processes as they have active clients, just wait
self._reapworkers(0)
@@ -470,11 +477,23 @@
self._cleanup()
def _mainloop(self):
+ exiting = False
h = self._servicehandler
- while not h.shouldexit():
+ while True:
+ if not exiting and h.shouldexit():
+ # clients can no longer connect() to the domain socket, so
+ # we stop queuing new requests.
+ # for requests that are queued (connect()-ed, but haven't been
+ # accept()-ed), handle them before exit. otherwise, clients
+ # waiting for recv() will receive ECONNRESET.
+ self._unlinksocket()
+ exiting = True
try:
ready = select.select([self._sock], [], [], h.pollinterval)[0]
if not ready:
+ # only exit if we completed all queued requests
+ if exiting:
+ break
continue
conn, _addr = self._sock.accept()
except (select.error, socket.error) as inst:
--- a/mercurial/context.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/context.py Tue Feb 28 11:13:25 2017 -0800
@@ -1166,7 +1166,7 @@
diffinrange = any(stype == '!' for _, stype in filteredblocks)
return diffinrange, linerange1
-def blockancestors(fctx, fromline, toline):
+def blockancestors(fctx, fromline, toline, followfirst=False):
"""Yield ancestors of `fctx` with respect to the block of lines within
`fromline`-`toline` range.
"""
@@ -1175,9 +1175,11 @@
while visit:
c, linerange2 = visit.pop(max(visit))
pl = c.parents()
+ if followfirst:
+ pl = pl[:1]
if not pl:
# The block originates from the initial revision.
- yield c
+ yield c, linerange2
continue
inrange = False
for p in pl:
@@ -1190,7 +1192,7 @@
continue
visit[p.linkrev(), p.filenode()] = p, linerange1
if inrange:
- yield c
+ yield c, linerange2
class committablectx(basectx):
"""A committablectx object provides common functionality for a context that
--- a/mercurial/crecord.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/crecord.py Tue Feb 28 11:13:25 2017 -0800
@@ -1375,7 +1375,8 @@
pass
helpwin.refresh()
try:
- helpwin.getkey()
+ with self.ui.timeblockedsection('crecord'):
+ helpwin.getkey()
except curses.error:
pass
@@ -1392,7 +1393,8 @@
self.stdscr.refresh()
confirmwin.refresh()
try:
- response = chr(self.stdscr.getch())
+ with self.ui.timeblockedsection('crecord'):
+ response = chr(self.stdscr.getch())
except ValueError:
response = None
@@ -1412,7 +1414,8 @@
are you sure you want to review/edit and confirm the selected changes [yn]?
""")
- response = self.confirmationwindow(confirmtext)
+ with self.ui.timeblockedsection('crecord'):
+ response = self.confirmationwindow(confirmtext)
if response is None:
response = "n"
if response.lower().startswith("y"):
@@ -1655,7 +1658,8 @@
while True:
self.updatescreen()
try:
- keypressed = self.statuswin.getkey()
+ with self.ui.timeblockedsection('crecord'):
+ keypressed = self.statuswin.getkey()
if self.errorstr is not None:
self.errorstr = None
continue
--- a/mercurial/debugcommands.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/debugcommands.py Tue Feb 28 11:13:25 2017 -0800
@@ -7,15 +7,24 @@
from __future__ import absolute_import
+import difflib
+import errno
import operator
import os
import random
+import socket
+import string
+import sys
+import tempfile
+import time
from .i18n import _
from .node import (
bin,
hex,
+ nullhex,
nullid,
+ nullrev,
short,
)
from . import (
@@ -26,20 +35,31 @@
context,
dagparser,
dagutil,
+ encoding,
error,
exchange,
extensions,
fileset,
+ formatter,
hg,
localrepo,
lock as lockmod,
+ merge as mergemod,
+ obsolete,
+ policy,
+ pvec,
pycompat,
repair,
revlog,
+ revset,
+ revsetlang,
scmutil,
setdiscovery,
simplemerge,
+ smartset,
+ sslutil,
streamclone,
+ templater,
treediscovery,
util,
)
@@ -567,6 +587,37 @@
fm.end()
+@command('debugdirstate|debugstate',
+ [('', 'nodates', None, _('do not display the saved mtime')),
+ ('', 'datesort', None, _('sort by saved mtime'))],
+ _('[OPTION]...'))
+def debugstate(ui, repo, **opts):
+ """show the contents of the current dirstate"""
+
+ nodates = opts.get('nodates')
+ datesort = opts.get('datesort')
+
+ timestr = ""
+ if datesort:
+ keyfunc = lambda x: (x[1][3], x[0]) # sort by mtime, then by filename
+ else:
+ keyfunc = None # sort by filename
+ for file_, ent in sorted(repo.dirstate._map.iteritems(), key=keyfunc):
+ if ent[3] == -1:
+ timestr = 'unset '
+ elif nodates:
+ timestr = 'set '
+ else:
+ timestr = time.strftime("%Y-%m-%d %H:%M:%S ",
+ time.localtime(ent[3]))
+ if ent[1] & 0o20000:
+ mode = 'lnk'
+ else:
+ mode = '%3o' % (ent[1] & 0o777 & ~util.umask)
+ ui.write("%c %s %10d %s%s\n" % (ent[0], mode, ent[2], timestr, file_))
+ for f in repo.dirstate.copies():
+ ui.write(_("copy: %s -> %s\n") % (repo.dirstate.copied(f), f))
+
@command('debugdiscovery',
[('', 'old', None, _('use old-style discovery')),
('', 'nonheads', None,
@@ -641,7 +692,7 @@
fm = ui.formatter('debugextensions', opts)
for extname, extmod in sorted(exts, key=operator.itemgetter(0)):
isinternal = extensions.ismoduleinternal(extmod)
- extsource = extmod.__file__
+ extsource = pycompat.fsencode(extmod.__file__)
if isinternal:
exttestedwith = [] # never expose magic string to users
else:
@@ -851,6 +902,1106 @@
ui.write("\t%d -> %d\n" % (r.rev(pp[1]), i))
ui.write("}\n")
+@command('debuginstall', [] + commands.formatteropts, '', norepo=True)
+def debuginstall(ui, **opts):
+ '''test Mercurial installation
+
+ Returns 0 on success.
+ '''
+
+ def writetemp(contents):
+ (fd, name) = tempfile.mkstemp(prefix="hg-debuginstall-")
+ f = os.fdopen(fd, pycompat.sysstr("wb"))
+ f.write(contents)
+ f.close()
+ return name
+
+ problems = 0
+
+ fm = ui.formatter('debuginstall', opts)
+ fm.startitem()
+
+ # encoding
+ fm.write('encoding', _("checking encoding (%s)...\n"), encoding.encoding)
+ err = None
+ try:
+ encoding.fromlocal("test")
+ except error.Abort as inst:
+ err = inst
+ problems += 1
+ fm.condwrite(err, 'encodingerror', _(" %s\n"
+ " (check that your locale is properly set)\n"), err)
+
+ # Python
+ fm.write('pythonexe', _("checking Python executable (%s)\n"),
+ pycompat.sysexecutable)
+ fm.write('pythonver', _("checking Python version (%s)\n"),
+ ("%d.%d.%d" % sys.version_info[:3]))
+ fm.write('pythonlib', _("checking Python lib (%s)...\n"),
+ os.path.dirname(pycompat.fsencode(os.__file__)))
+
+ security = set(sslutil.supportedprotocols)
+ if sslutil.hassni:
+ security.add('sni')
+
+ fm.write('pythonsecurity', _("checking Python security support (%s)\n"),
+ fm.formatlist(sorted(security), name='protocol',
+ fmt='%s', sep=','))
+
+ # These are warnings, not errors. So don't increment problem count. This
+ # may change in the future.
+ if 'tls1.2' not in security:
+ fm.plain(_(' TLS 1.2 not supported by Python install; '
+ 'network connections lack modern security\n'))
+ if 'sni' not in security:
+ fm.plain(_(' SNI not supported by Python install; may have '
+ 'connectivity issues with some servers\n'))
+
+ # TODO print CA cert info
+
+ # hg version
+ hgver = util.version()
+ fm.write('hgver', _("checking Mercurial version (%s)\n"),
+ hgver.split('+')[0])
+ fm.write('hgverextra', _("checking Mercurial custom build (%s)\n"),
+ '+'.join(hgver.split('+')[1:]))
+
+ # compiled modules
+ fm.write('hgmodulepolicy', _("checking module policy (%s)\n"),
+ policy.policy)
+ fm.write('hgmodules', _("checking installed modules (%s)...\n"),
+ os.path.dirname(pycompat.fsencode(__file__)))
+
+ err = None
+ try:
+ from . import (
+ base85,
+ bdiff,
+ mpatch,
+ osutil,
+ )
+ dir(bdiff), dir(mpatch), dir(base85), dir(osutil) # quiet pyflakes
+ except Exception as inst:
+ err = inst
+ problems += 1
+ fm.condwrite(err, 'extensionserror', " %s\n", err)
+
+ compengines = util.compengines._engines.values()
+ fm.write('compengines', _('checking registered compression engines (%s)\n'),
+ fm.formatlist(sorted(e.name() for e in compengines),
+ name='compengine', fmt='%s', sep=', '))
+ fm.write('compenginesavail', _('checking available compression engines '
+ '(%s)\n'),
+ fm.formatlist(sorted(e.name() for e in compengines
+ if e.available()),
+ name='compengine', fmt='%s', sep=', '))
+ wirecompengines = util.compengines.supportedwireengines(util.SERVERROLE)
+ fm.write('compenginesserver', _('checking available compression engines '
+ 'for wire protocol (%s)\n'),
+ fm.formatlist([e.name() for e in wirecompengines
+ if e.wireprotosupport()],
+ name='compengine', fmt='%s', sep=', '))
+
+ # templates
+ p = templater.templatepaths()
+ fm.write('templatedirs', 'checking templates (%s)...\n', ' '.join(p))
+ fm.condwrite(not p, '', _(" no template directories found\n"))
+ if p:
+ m = templater.templatepath("map-cmdline.default")
+ if m:
+ # template found, check if it is working
+ err = None
+ try:
+ templater.templater.frommapfile(m)
+ except Exception as inst:
+ err = inst
+ p = None
+ fm.condwrite(err, 'defaulttemplateerror', " %s\n", err)
+ else:
+ p = None
+ fm.condwrite(p, 'defaulttemplate',
+ _("checking default template (%s)\n"), m)
+ fm.condwrite(not m, 'defaulttemplatenotfound',
+ _(" template '%s' not found\n"), "default")
+ if not p:
+ problems += 1
+ fm.condwrite(not p, '',
+ _(" (templates seem to have been installed incorrectly)\n"))
+
+ # editor
+ editor = ui.geteditor()
+ editor = util.expandpath(editor)
+ fm.write('editor', _("checking commit editor... (%s)\n"), editor)
+ cmdpath = util.findexe(pycompat.shlexsplit(editor)[0])
+ fm.condwrite(not cmdpath and editor == 'vi', 'vinotfound',
+ _(" No commit editor set and can't find %s in PATH\n"
+ " (specify a commit editor in your configuration"
+ " file)\n"), not cmdpath and editor == 'vi' and editor)
+ fm.condwrite(not cmdpath and editor != 'vi', 'editornotfound',
+ _(" Can't find editor '%s' in PATH\n"
+ " (specify a commit editor in your configuration"
+ " file)\n"), not cmdpath and editor)
+ if not cmdpath and editor != 'vi':
+ problems += 1
+
+ # check username
+ username = None
+ err = None
+ try:
+ username = ui.username()
+ except error.Abort as e:
+ err = e
+ problems += 1
+
+ fm.condwrite(username, 'username', _("checking username (%s)\n"), username)
+ fm.condwrite(err, 'usernameerror', _("checking username...\n %s\n"
+ " (specify a username in your configuration file)\n"), err)
+
+ fm.condwrite(not problems, '',
+ _("no problems detected\n"))
+ if not problems:
+ fm.data(problems=problems)
+ fm.condwrite(problems, 'problems',
+ _("%d problems detected,"
+ " please check your install!\n"), problems)
+ fm.end()
+
+ return problems
+
+@command('debugknown', [], _('REPO ID...'), norepo=True)
+def debugknown(ui, repopath, *ids, **opts):
+ """test whether node ids are known to a repo
+
+ Every ID must be a full-length hex node id string. Returns a list of 0s
+ and 1s indicating unknown/known.
+ """
+ repo = hg.peer(ui, opts, repopath)
+ if not repo.capable('known'):
+ raise error.Abort("known() not supported by target repository")
+ flags = repo.known([bin(s) for s in ids])
+ ui.write("%s\n" % ("".join([f and "1" or "0" for f in flags])))
+
+@command('debuglabelcomplete', [], _('LABEL...'))
+def debuglabelcomplete(ui, repo, *args):
+ '''backwards compatibility with old bash completion scripts (DEPRECATED)'''
+ commands.debugnamecomplete(ui, repo, *args)
+
+@command('debuglocks',
+ [('L', 'force-lock', None, _('free the store lock (DANGEROUS)')),
+ ('W', 'force-wlock', None,
+ _('free the working state lock (DANGEROUS)'))],
+ _('[OPTION]...'))
+def debuglocks(ui, repo, **opts):
+ """show or modify state of locks
+
+ By default, this command will show which locks are held. This
+ includes the user and process holding the lock, the amount of time
+ the lock has been held, and the machine name where the process is
+ running if it's not local.
+
+ Locks protect the integrity of Mercurial's data, so should be
+ treated with care. System crashes or other interruptions may cause
+ locks to not be properly released, though Mercurial will usually
+ detect and remove such stale locks automatically.
+
+ However, detecting stale locks may not always be possible (for
+ instance, on a shared filesystem). Removing locks may also be
+ blocked by filesystem permissions.
+
+ Returns 0 if no locks are held.
+
+ """
+
+ if opts.get('force_lock'):
+ repo.svfs.unlink('lock')
+ if opts.get('force_wlock'):
+ repo.vfs.unlink('wlock')
+ if opts.get('force_lock') or opts.get('force_lock'):
+ return 0
+
+ now = time.time()
+ held = 0
+
+ def report(vfs, name, method):
+ # this causes stale locks to get reaped for more accurate reporting
+ try:
+ l = method(False)
+ except error.LockHeld:
+ l = None
+
+ if l:
+ l.release()
+ else:
+ try:
+ stat = vfs.lstat(name)
+ age = now - stat.st_mtime
+ user = util.username(stat.st_uid)
+ locker = vfs.readlock(name)
+ if ":" in locker:
+ host, pid = locker.split(':')
+ if host == socket.gethostname():
+ locker = 'user %s, process %s' % (user, pid)
+ else:
+ locker = 'user %s, process %s, host %s' \
+ % (user, pid, host)
+ ui.write(("%-6s %s (%ds)\n") % (name + ":", locker, age))
+ return 1
+ except OSError as e:
+ if e.errno != errno.ENOENT:
+ raise
+
+ ui.write(("%-6s free\n") % (name + ":"))
+ return 0
+
+ held += report(repo.svfs, "lock", repo.lock)
+ held += report(repo.vfs, "wlock", repo.wlock)
+
+ return held
+
+@command('debugmergestate', [], '')
+def debugmergestate(ui, repo, *args):
+ """print merge state
+
+ Use --verbose to print out information about whether v1 or v2 merge state
+ was chosen."""
+ def _hashornull(h):
+ if h == nullhex:
+ return 'null'
+ else:
+ return h
+
+ def printrecords(version):
+ ui.write(('* version %s records\n') % version)
+ if version == 1:
+ records = v1records
+ else:
+ records = v2records
+
+ for rtype, record in records:
+ # pretty print some record types
+ if rtype == 'L':
+ ui.write(('local: %s\n') % record)
+ elif rtype == 'O':
+ ui.write(('other: %s\n') % record)
+ elif rtype == 'm':
+ driver, mdstate = record.split('\0', 1)
+ ui.write(('merge driver: %s (state "%s")\n')
+ % (driver, mdstate))
+ elif rtype in 'FDC':
+ r = record.split('\0')
+ f, state, hash, lfile, afile, anode, ofile = r[0:7]
+ if version == 1:
+ onode = 'not stored in v1 format'
+ flags = r[7]
+ else:
+ onode, flags = r[7:9]
+ ui.write(('file: %s (record type "%s", state "%s", hash %s)\n')
+ % (f, rtype, state, _hashornull(hash)))
+ ui.write((' local path: %s (flags "%s")\n') % (lfile, flags))
+ ui.write((' ancestor path: %s (node %s)\n')
+ % (afile, _hashornull(anode)))
+ ui.write((' other path: %s (node %s)\n')
+ % (ofile, _hashornull(onode)))
+ elif rtype == 'f':
+ filename, rawextras = record.split('\0', 1)
+ extras = rawextras.split('\0')
+ i = 0
+ extrastrings = []
+ while i < len(extras):
+ extrastrings.append('%s = %s' % (extras[i], extras[i + 1]))
+ i += 2
+
+ ui.write(('file extras: %s (%s)\n')
+ % (filename, ', '.join(extrastrings)))
+ elif rtype == 'l':
+ labels = record.split('\0', 2)
+ labels = [l for l in labels if len(l) > 0]
+ ui.write(('labels:\n'))
+ ui.write((' local: %s\n' % labels[0]))
+ ui.write((' other: %s\n' % labels[1]))
+ if len(labels) > 2:
+ ui.write((' base: %s\n' % labels[2]))
+ else:
+ ui.write(('unrecognized entry: %s\t%s\n')
+ % (rtype, record.replace('\0', '\t')))
+
+ # Avoid mergestate.read() since it may raise an exception for unsupported
+ # merge state records. We shouldn't be doing this, but this is OK since this
+ # command is pretty low-level.
+ ms = mergemod.mergestate(repo)
+
+ # sort so that reasonable information is on top
+ v1records = ms._readrecordsv1()
+ v2records = ms._readrecordsv2()
+ order = 'LOml'
+ def key(r):
+ idx = order.find(r[0])
+ if idx == -1:
+ return (1, r[1])
+ else:
+ return (0, idx)
+ v1records.sort(key=key)
+ v2records.sort(key=key)
+
+ if not v1records and not v2records:
+ ui.write(('no merge state found\n'))
+ elif not v2records:
+ ui.note(('no version 2 merge state\n'))
+ printrecords(1)
+ elif ms._v1v2match(v1records, v2records):
+ ui.note(('v1 and v2 states match: using v2\n'))
+ printrecords(2)
+ else:
+ ui.note(('v1 and v2 states mismatch: using v1\n'))
+ printrecords(1)
+ if ui.verbose:
+ printrecords(2)
+
+@command('debugnamecomplete', [], _('NAME...'))
+def debugnamecomplete(ui, repo, *args):
+ '''complete "names" - tags, open branch names, bookmark names'''
+
+ names = set()
+ # since we previously only listed open branches, we will handle that
+ # specially (after this for loop)
+ for name, ns in repo.names.iteritems():
+ if name != 'branches':
+ names.update(ns.listnames(repo))
+ names.update(tag for (tag, heads, tip, closed)
+ in repo.branchmap().iterbranches() if not closed)
+ completions = set()
+ if not args:
+ args = ['']
+ for a in args:
+ completions.update(n for n in names if n.startswith(a))
+ ui.write('\n'.join(sorted(completions)))
+ ui.write('\n')
+
+@command('debugobsolete',
+ [('', 'flags', 0, _('markers flag')),
+ ('', 'record-parents', False,
+ _('record parent information for the precursor')),
+ ('r', 'rev', [], _('display markers relevant to REV')),
+ ('', 'index', False, _('display index of the marker')),
+ ('', 'delete', [], _('delete markers specified by indices')),
+ ] + commands.commitopts2 + commands.formatteropts,
+ _('[OBSOLETED [REPLACEMENT ...]]'))
+def debugobsolete(ui, repo, precursor=None, *successors, **opts):
+ """create arbitrary obsolete marker
+
+ With no arguments, displays the list of obsolescence markers."""
+
+ def parsenodeid(s):
+ try:
+ # We do not use revsingle/revrange functions here to accept
+ # arbitrary node identifiers, possibly not present in the
+ # local repository.
+ n = bin(s)
+ if len(n) != len(nullid):
+ raise TypeError()
+ return n
+ except TypeError:
+ raise error.Abort('changeset references must be full hexadecimal '
+ 'node identifiers')
+
+ if opts.get('delete'):
+ indices = []
+ for v in opts.get('delete'):
+ try:
+ indices.append(int(v))
+ except ValueError:
+ raise error.Abort(_('invalid index value: %r') % v,
+ hint=_('use integers for indices'))
+
+ if repo.currenttransaction():
+ raise error.Abort(_('cannot delete obsmarkers in the middle '
+ 'of transaction.'))
+
+ with repo.lock():
+ n = repair.deleteobsmarkers(repo.obsstore, indices)
+ ui.write(_('deleted %i obsolescence markers\n') % n)
+
+ return
+
+ if precursor is not None:
+ if opts['rev']:
+ raise error.Abort('cannot select revision when creating marker')
+ metadata = {}
+ metadata['user'] = opts['user'] or ui.username()
+ succs = tuple(parsenodeid(succ) for succ in successors)
+ l = repo.lock()
+ try:
+ tr = repo.transaction('debugobsolete')
+ try:
+ date = opts.get('date')
+ if date:
+ date = util.parsedate(date)
+ else:
+ date = None
+ prec = parsenodeid(precursor)
+ parents = None
+ if opts['record_parents']:
+ if prec not in repo.unfiltered():
+ raise error.Abort('cannot used --record-parents on '
+ 'unknown changesets')
+ parents = repo.unfiltered()[prec].parents()
+ parents = tuple(p.node() for p in parents)
+ repo.obsstore.create(tr, prec, succs, opts['flags'],
+ parents=parents, date=date,
+ metadata=metadata)
+ tr.close()
+ except ValueError as exc:
+ raise error.Abort(_('bad obsmarker input: %s') % exc)
+ finally:
+ tr.release()
+ finally:
+ l.release()
+ else:
+ if opts['rev']:
+ revs = scmutil.revrange(repo, opts['rev'])
+ nodes = [repo[r].node() for r in revs]
+ markers = list(obsolete.getmarkers(repo, nodes=nodes))
+ markers.sort(key=lambda x: x._data)
+ else:
+ markers = obsolete.getmarkers(repo)
+
+ markerstoiter = markers
+ isrelevant = lambda m: True
+ if opts.get('rev') and opts.get('index'):
+ markerstoiter = obsolete.getmarkers(repo)
+ markerset = set(markers)
+ isrelevant = lambda m: m in markerset
+
+ fm = ui.formatter('debugobsolete', opts)
+ for i, m in enumerate(markerstoiter):
+ if not isrelevant(m):
+ # marker can be irrelevant when we're iterating over a set
+ # of markers (markerstoiter) which is bigger than the set
+ # of markers we want to display (markers)
+ # this can happen if both --index and --rev options are
+ # provided and thus we need to iterate over all of the markers
+ # to get the correct indices, but only display the ones that
+ # are relevant to --rev value
+ continue
+ fm.startitem()
+ ind = i if opts.get('index') else None
+ cmdutil.showmarker(fm, m, index=ind)
+ fm.end()
+
+@command('debugpathcomplete',
+ [('f', 'full', None, _('complete an entire path')),
+ ('n', 'normal', None, _('show only normal files')),
+ ('a', 'added', None, _('show only added files')),
+ ('r', 'removed', None, _('show only removed files'))],
+ _('FILESPEC...'))
+def debugpathcomplete(ui, repo, *specs, **opts):
+ '''complete part or all of a tracked path
+
+ This command supports shells that offer path name completion. It
+ currently completes only files already known to the dirstate.
+
+ Completion extends only to the next path segment unless
+ --full is specified, in which case entire paths are used.'''
+
+ def complete(path, acceptable):
+ dirstate = repo.dirstate
+ spec = os.path.normpath(os.path.join(pycompat.getcwd(), path))
+ rootdir = repo.root + pycompat.ossep
+ if spec != repo.root and not spec.startswith(rootdir):
+ return [], []
+ if os.path.isdir(spec):
+ spec += '/'
+ spec = spec[len(rootdir):]
+ fixpaths = pycompat.ossep != '/'
+ if fixpaths:
+ spec = spec.replace(pycompat.ossep, '/')
+ speclen = len(spec)
+ fullpaths = opts['full']
+ files, dirs = set(), set()
+ adddir, addfile = dirs.add, files.add
+ for f, st in dirstate.iteritems():
+ if f.startswith(spec) and st[0] in acceptable:
+ if fixpaths:
+ f = f.replace('/', pycompat.ossep)
+ if fullpaths:
+ addfile(f)
+ continue
+ s = f.find(pycompat.ossep, speclen)
+ if s >= 0:
+ adddir(f[:s])
+ else:
+ addfile(f)
+ return files, dirs
+
+ acceptable = ''
+ if opts['normal']:
+ acceptable += 'nm'
+ if opts['added']:
+ acceptable += 'a'
+ if opts['removed']:
+ acceptable += 'r'
+ cwd = repo.getcwd()
+ if not specs:
+ specs = ['.']
+
+ files, dirs = set(), set()
+ for spec in specs:
+ f, d = complete(spec, acceptable or 'nmar')
+ files.update(f)
+ dirs.update(d)
+ files.update(dirs)
+ ui.write('\n'.join(repo.pathto(p, cwd) for p in sorted(files)))
+ ui.write('\n')
+
+@command('debugpushkey', [], _('REPO NAMESPACE [KEY OLD NEW]'), norepo=True)
+def debugpushkey(ui, repopath, namespace, *keyinfo, **opts):
+ '''access the pushkey key/value protocol
+
+ With two args, list the keys in the given namespace.
+
+ With five args, set a key to new if it currently is set to old.
+ Reports success or failure.
+ '''
+
+ target = hg.peer(ui, {}, repopath)
+ if keyinfo:
+ key, old, new = keyinfo
+ r = target.pushkey(namespace, key, old, new)
+ ui.status(str(r) + '\n')
+ return not r
+ else:
+ for k, v in sorted(target.listkeys(namespace).iteritems()):
+ ui.write("%s\t%s\n" % (k.encode('string-escape'),
+ v.encode('string-escape')))
+
+@command('debugpvec', [], _('A B'))
+def debugpvec(ui, repo, a, b=None):
+ ca = scmutil.revsingle(repo, a)
+ cb = scmutil.revsingle(repo, b)
+ pa = pvec.ctxpvec(ca)
+ pb = pvec.ctxpvec(cb)
+ if pa == pb:
+ rel = "="
+ elif pa > pb:
+ rel = ">"
+ elif pa < pb:
+ rel = "<"
+ elif pa | pb:
+ rel = "|"
+ ui.write(_("a: %s\n") % pa)
+ ui.write(_("b: %s\n") % pb)
+ ui.write(_("depth(a): %d depth(b): %d\n") % (pa._depth, pb._depth))
+ ui.write(_("delta: %d hdist: %d distance: %d relation: %s\n") %
+ (abs(pa._depth - pb._depth), pvec._hamming(pa._vec, pb._vec),
+ pa.distance(pb), rel))
+
+@command('debugrebuilddirstate|debugrebuildstate',
+ [('r', 'rev', '', _('revision to rebuild to'), _('REV')),
+ ('', 'minimal', None, _('only rebuild files that are inconsistent with '
+ 'the working copy parent')),
+ ],
+ _('[-r REV]'))
+def debugrebuilddirstate(ui, repo, rev, **opts):
+ """rebuild the dirstate as it would look like for the given revision
+
+ If no revision is specified the first current parent will be used.
+
+ The dirstate will be set to the files of the given revision.
+ The actual working directory content or existing dirstate
+ information such as adds or removes is not considered.
+
+ ``minimal`` will only rebuild the dirstate status for files that claim to be
+ tracked but are not in the parent manifest, or that exist in the parent
+ manifest but are not in the dirstate. It will not change adds, removes, or
+ modified files that are in the working copy parent.
+
+ One use of this command is to make the next :hg:`status` invocation
+ check the actual file content.
+ """
+ ctx = scmutil.revsingle(repo, rev)
+ with repo.wlock():
+ dirstate = repo.dirstate
+ changedfiles = None
+ # See command doc for what minimal does.
+ if opts.get('minimal'):
+ manifestfiles = set(ctx.manifest().keys())
+ dirstatefiles = set(dirstate)
+ manifestonly = manifestfiles - dirstatefiles
+ dsonly = dirstatefiles - manifestfiles
+ dsnotadded = set(f for f in dsonly if dirstate[f] != 'a')
+ changedfiles = manifestonly | dsnotadded
+
+ dirstate.rebuild(ctx.node(), ctx.manifest(), changedfiles)
+
+@command('debugrebuildfncache', [], '')
+def debugrebuildfncache(ui, repo):
+ """rebuild the fncache file"""
+ repair.rebuildfncache(ui, repo)
+
+@command('debugrename',
+ [('r', 'rev', '', _('revision to debug'), _('REV'))],
+ _('[-r REV] FILE'))
+def debugrename(ui, repo, file1, *pats, **opts):
+ """dump rename information"""
+
+ ctx = scmutil.revsingle(repo, opts.get('rev'))
+ m = scmutil.match(ctx, (file1,) + pats, opts)
+ for abs in ctx.walk(m):
+ fctx = ctx[abs]
+ o = fctx.filelog().renamed(fctx.filenode())
+ rel = m.rel(abs)
+ if o:
+ ui.write(_("%s renamed from %s:%s\n") % (rel, o[0], hex(o[1])))
+ else:
+ ui.write(_("%s not renamed\n") % rel)
+
+@command('debugrevlog', commands.debugrevlogopts +
+ [('d', 'dump', False, _('dump index data'))],
+ _('-c|-m|FILE'),
+ optionalrepo=True)
+def debugrevlog(ui, repo, file_=None, **opts):
+ """show data and statistics about a revlog"""
+ r = cmdutil.openrevlog(repo, 'debugrevlog', file_, opts)
+
+ if opts.get("dump"):
+ numrevs = len(r)
+ ui.write(("# rev p1rev p2rev start end deltastart base p1 p2"
+ " rawsize totalsize compression heads chainlen\n"))
+ ts = 0
+ heads = set()
+
+ for rev in xrange(numrevs):
+ dbase = r.deltaparent(rev)
+ if dbase == -1:
+ dbase = rev
+ cbase = r.chainbase(rev)
+ clen = r.chainlen(rev)
+ p1, p2 = r.parentrevs(rev)
+ rs = r.rawsize(rev)
+ ts = ts + rs
+ heads -= set(r.parentrevs(rev))
+ heads.add(rev)
+ try:
+ compression = ts / r.end(rev)
+ except ZeroDivisionError:
+ compression = 0
+ ui.write("%5d %5d %5d %5d %5d %10d %4d %4d %4d %7d %9d "
+ "%11d %5d %8d\n" %
+ (rev, p1, p2, r.start(rev), r.end(rev),
+ r.start(dbase), r.start(cbase),
+ r.start(p1), r.start(p2),
+ rs, ts, compression, len(heads), clen))
+ return 0
+
+ v = r.version
+ format = v & 0xFFFF
+ flags = []
+ gdelta = False
+ if v & revlog.REVLOGNGINLINEDATA:
+ flags.append('inline')
+ if v & revlog.REVLOGGENERALDELTA:
+ gdelta = True
+ flags.append('generaldelta')
+ if not flags:
+ flags = ['(none)']
+
+ nummerges = 0
+ numfull = 0
+ numprev = 0
+ nump1 = 0
+ nump2 = 0
+ numother = 0
+ nump1prev = 0
+ nump2prev = 0
+ chainlengths = []
+
+ datasize = [None, 0, 0]
+ fullsize = [None, 0, 0]
+ deltasize = [None, 0, 0]
+ chunktypecounts = {}
+ chunktypesizes = {}
+
+ def addsize(size, l):
+ if l[0] is None or size < l[0]:
+ l[0] = size
+ if size > l[1]:
+ l[1] = size
+ l[2] += size
+
+ numrevs = len(r)
+ for rev in xrange(numrevs):
+ p1, p2 = r.parentrevs(rev)
+ delta = r.deltaparent(rev)
+ if format > 0:
+ addsize(r.rawsize(rev), datasize)
+ if p2 != nullrev:
+ nummerges += 1
+ size = r.length(rev)
+ if delta == nullrev:
+ chainlengths.append(0)
+ numfull += 1
+ addsize(size, fullsize)
+ else:
+ chainlengths.append(chainlengths[delta] + 1)
+ addsize(size, deltasize)
+ if delta == rev - 1:
+ numprev += 1
+ if delta == p1:
+ nump1prev += 1
+ elif delta == p2:
+ nump2prev += 1
+ elif delta == p1:
+ nump1 += 1
+ elif delta == p2:
+ nump2 += 1
+ elif delta != nullrev:
+ numother += 1
+
+ # Obtain data on the raw chunks in the revlog.
+ chunk = r._chunkraw(rev, rev)[1]
+ if chunk:
+ chunktype = chunk[0]
+ else:
+ chunktype = 'empty'
+
+ if chunktype not in chunktypecounts:
+ chunktypecounts[chunktype] = 0
+ chunktypesizes[chunktype] = 0
+
+ chunktypecounts[chunktype] += 1
+ chunktypesizes[chunktype] += size
+
+ # Adjust size min value for empty cases
+ for size in (datasize, fullsize, deltasize):
+ if size[0] is None:
+ size[0] = 0
+
+ numdeltas = numrevs - numfull
+ numoprev = numprev - nump1prev - nump2prev
+ totalrawsize = datasize[2]
+ datasize[2] /= numrevs
+ fulltotal = fullsize[2]
+ fullsize[2] /= numfull
+ deltatotal = deltasize[2]
+ if numrevs - numfull > 0:
+ deltasize[2] /= numrevs - numfull
+ totalsize = fulltotal + deltatotal
+ avgchainlen = sum(chainlengths) / numrevs
+ maxchainlen = max(chainlengths)
+ compratio = 1
+ if totalsize:
+ compratio = totalrawsize / totalsize
+
+ basedfmtstr = '%%%dd\n'
+ basepcfmtstr = '%%%dd %s(%%5.2f%%%%)\n'
+
+ def dfmtstr(max):
+ return basedfmtstr % len(str(max))
+ def pcfmtstr(max, padding=0):
+ return basepcfmtstr % (len(str(max)), ' ' * padding)
+
+ def pcfmt(value, total):
+ if total:
+ return (value, 100 * float(value) / total)
+ else:
+ return value, 100.0
+
+ ui.write(('format : %d\n') % format)
+ ui.write(('flags : %s\n') % ', '.join(flags))
+
+ ui.write('\n')
+ fmt = pcfmtstr(totalsize)
+ fmt2 = dfmtstr(totalsize)
+ ui.write(('revisions : ') + fmt2 % numrevs)
+ ui.write((' merges : ') + fmt % pcfmt(nummerges, numrevs))
+ ui.write((' normal : ') + fmt % pcfmt(numrevs - nummerges, numrevs))
+ ui.write(('revisions : ') + fmt2 % numrevs)
+ ui.write((' full : ') + fmt % pcfmt(numfull, numrevs))
+ ui.write((' deltas : ') + fmt % pcfmt(numdeltas, numrevs))
+ ui.write(('revision size : ') + fmt2 % totalsize)
+ ui.write((' full : ') + fmt % pcfmt(fulltotal, totalsize))
+ ui.write((' deltas : ') + fmt % pcfmt(deltatotal, totalsize))
+
+ def fmtchunktype(chunktype):
+ if chunktype == 'empty':
+ return ' %s : ' % chunktype
+ elif chunktype in string.ascii_letters:
+ return ' 0x%s (%s) : ' % (hex(chunktype), chunktype)
+ else:
+ return ' 0x%s : ' % hex(chunktype)
+
+ ui.write('\n')
+ ui.write(('chunks : ') + fmt2 % numrevs)
+ for chunktype in sorted(chunktypecounts):
+ ui.write(fmtchunktype(chunktype))
+ ui.write(fmt % pcfmt(chunktypecounts[chunktype], numrevs))
+ ui.write(('chunks size : ') + fmt2 % totalsize)
+ for chunktype in sorted(chunktypecounts):
+ ui.write(fmtchunktype(chunktype))
+ ui.write(fmt % pcfmt(chunktypesizes[chunktype], totalsize))
+
+ ui.write('\n')
+ fmt = dfmtstr(max(avgchainlen, compratio))
+ ui.write(('avg chain length : ') + fmt % avgchainlen)
+ ui.write(('max chain length : ') + fmt % maxchainlen)
+ ui.write(('compression ratio : ') + fmt % compratio)
+
+ if format > 0:
+ ui.write('\n')
+ ui.write(('uncompressed data size (min/max/avg) : %d / %d / %d\n')
+ % tuple(datasize))
+ ui.write(('full revision size (min/max/avg) : %d / %d / %d\n')
+ % tuple(fullsize))
+ ui.write(('delta size (min/max/avg) : %d / %d / %d\n')
+ % tuple(deltasize))
+
+ if numdeltas > 0:
+ ui.write('\n')
+ fmt = pcfmtstr(numdeltas)
+ fmt2 = pcfmtstr(numdeltas, 4)
+ ui.write(('deltas against prev : ') + fmt % pcfmt(numprev, numdeltas))
+ if numprev > 0:
+ ui.write((' where prev = p1 : ') + fmt2 % pcfmt(nump1prev,
+ numprev))
+ ui.write((' where prev = p2 : ') + fmt2 % pcfmt(nump2prev,
+ numprev))
+ ui.write((' other : ') + fmt2 % pcfmt(numoprev,
+ numprev))
+ if gdelta:
+ ui.write(('deltas against p1 : ')
+ + fmt % pcfmt(nump1, numdeltas))
+ ui.write(('deltas against p2 : ')
+ + fmt % pcfmt(nump2, numdeltas))
+ ui.write(('deltas against other : ') + fmt % pcfmt(numother,
+ numdeltas))
+
+@command('debugrevspec',
+ [('', 'optimize', None,
+ _('print parsed tree after optimizing (DEPRECATED)')),
+ ('p', 'show-stage', [],
+ _('print parsed tree at the given stage'), _('NAME')),
+ ('', 'no-optimized', False, _('evaluate tree without optimization')),
+ ('', 'verify-optimized', False, _('verify optimized result')),
+ ],
+ ('REVSPEC'))
+def debugrevspec(ui, repo, expr, **opts):
+ """parse and apply a revision specification
+
+ Use -p/--show-stage option to print the parsed tree at the given stages.
+ Use -p all to print tree at every stage.
+
+ Use --verify-optimized to compare the optimized result with the unoptimized
+ one. Returns 1 if the optimized result differs.
+ """
+ stages = [
+ ('parsed', lambda tree: tree),
+ ('expanded', lambda tree: revsetlang.expandaliases(ui, tree)),
+ ('concatenated', revsetlang.foldconcat),
+ ('analyzed', revsetlang.analyze),
+ ('optimized', revsetlang.optimize),
+ ]
+ if opts['no_optimized']:
+ stages = stages[:-1]
+ if opts['verify_optimized'] and opts['no_optimized']:
+ raise error.Abort(_('cannot use --verify-optimized with '
+ '--no-optimized'))
+ stagenames = set(n for n, f in stages)
+
+ showalways = set()
+ showchanged = set()
+ if ui.verbose and not opts['show_stage']:
+ # show parsed tree by --verbose (deprecated)
+ showalways.add('parsed')
+ showchanged.update(['expanded', 'concatenated'])
+ if opts['optimize']:
+ showalways.add('optimized')
+ if opts['show_stage'] and opts['optimize']:
+ raise error.Abort(_('cannot use --optimize with --show-stage'))
+ if opts['show_stage'] == ['all']:
+ showalways.update(stagenames)
+ else:
+ for n in opts['show_stage']:
+ if n not in stagenames:
+ raise error.Abort(_('invalid stage name: %s') % n)
+ showalways.update(opts['show_stage'])
+
+ treebystage = {}
+ printedtree = None
+ tree = revsetlang.parse(expr, lookup=repo.__contains__)
+ for n, f in stages:
+ treebystage[n] = tree = f(tree)
+ if n in showalways or (n in showchanged and tree != printedtree):
+ if opts['show_stage'] or n != 'parsed':
+ ui.write(("* %s:\n") % n)
+ ui.write(revsetlang.prettyformat(tree), "\n")
+ printedtree = tree
+
+ if opts['verify_optimized']:
+ arevs = revset.makematcher(treebystage['analyzed'])(repo)
+ brevs = revset.makematcher(treebystage['optimized'])(repo)
+ if ui.verbose:
+ ui.note(("* analyzed set:\n"), smartset.prettyformat(arevs), "\n")
+ ui.note(("* optimized set:\n"), smartset.prettyformat(brevs), "\n")
+ arevs = list(arevs)
+ brevs = list(brevs)
+ if arevs == brevs:
+ return 0
+ ui.write(('--- analyzed\n'), label='diff.file_a')
+ ui.write(('+++ optimized\n'), label='diff.file_b')
+ sm = difflib.SequenceMatcher(None, arevs, brevs)
+ for tag, alo, ahi, blo, bhi in sm.get_opcodes():
+ if tag in ('delete', 'replace'):
+ for c in arevs[alo:ahi]:
+ ui.write('-%s\n' % c, label='diff.deleted')
+ if tag in ('insert', 'replace'):
+ for c in brevs[blo:bhi]:
+ ui.write('+%s\n' % c, label='diff.inserted')
+ if tag == 'equal':
+ for c in arevs[alo:ahi]:
+ ui.write(' %s\n' % c)
+ return 1
+
+ func = revset.makematcher(tree)
+ revs = func(repo)
+ if ui.verbose:
+ ui.note(("* set:\n"), smartset.prettyformat(revs), "\n")
+ for c in revs:
+ ui.write("%s\n" % c)
+
+@command('debugsetparents', [], _('REV1 [REV2]'))
+def debugsetparents(ui, repo, rev1, rev2=None):
+ """manually set the parents of the current working directory
+
+ This is useful for writing repository conversion tools, but should
+ be used with care. For example, neither the working directory nor the
+ dirstate is updated, so file status may be incorrect after running this
+ command.
+
+ Returns 0 on success.
+ """
+
+ r1 = scmutil.revsingle(repo, rev1).node()
+ r2 = scmutil.revsingle(repo, rev2, 'null').node()
+
+ with repo.wlock():
+ repo.setparents(r1, r2)
+
+@command('debugsub',
+ [('r', 'rev', '',
+ _('revision to check'), _('REV'))],
+ _('[-r REV] [REV]'))
+def debugsub(ui, repo, rev=None):
+ ctx = scmutil.revsingle(repo, rev, None)
+ for k, v in sorted(ctx.substate.items()):
+ ui.write(('path %s\n') % k)
+ ui.write((' source %s\n') % v[0])
+ ui.write((' revision %s\n') % v[1])
+
+@command('debugsuccessorssets',
+ [],
+ _('[REV]'))
+def debugsuccessorssets(ui, repo, *revs):
+ """show set of successors for revision
+
+ A successors set of changeset A is a consistent group of revisions that
+ succeed A. It contains non-obsolete changesets only.
+
+ In most cases a changeset A has a single successors set containing a single
+ successor (changeset A replaced by A').
+
+ A changeset that is made obsolete with no successors are called "pruned".
+ Such changesets have no successors sets at all.
+
+ A changeset that has been "split" will have a successors set containing
+ more than one successor.
+
+ A changeset that has been rewritten in multiple different ways is called
+ "divergent". Such changesets have multiple successor sets (each of which
+ may also be split, i.e. have multiple successors).
+
+ Results are displayed as follows::
+
+ <rev1>
+ <successors-1A>
+ <rev2>
+ <successors-2A>
+ <successors-2B1> <successors-2B2> <successors-2B3>
+
+ Here rev2 has two possible (i.e. divergent) successors sets. The first
+ holds one element, whereas the second holds three (i.e. the changeset has
+ been split).
+ """
+ # passed to successorssets caching computation from one call to another
+ cache = {}
+ ctx2str = str
+ node2str = short
+ if ui.debug():
+ def ctx2str(ctx):
+ return ctx.hex()
+ node2str = hex
+ for rev in scmutil.revrange(repo, revs):
+ ctx = repo[rev]
+ ui.write('%s\n'% ctx2str(ctx))
+ for succsset in obsolete.successorssets(repo, ctx.node(), cache):
+ if succsset:
+ ui.write(' ')
+ ui.write(node2str(succsset[0]))
+ for node in succsset[1:]:
+ ui.write(' ')
+ ui.write(node2str(node))
+ ui.write('\n')
+
+@command('debugtemplate',
+ [('r', 'rev', [], _('apply template on changesets'), _('REV')),
+ ('D', 'define', [], _('define template keyword'), _('KEY=VALUE'))],
+ _('[-r REV]... [-D KEY=VALUE]... TEMPLATE'),
+ optionalrepo=True)
+def debugtemplate(ui, repo, tmpl, **opts):
+ """parse and apply a template
+
+ If -r/--rev is given, the template is processed as a log template and
+ applied to the given changesets. Otherwise, it is processed as a generic
+ template.
+
+ Use --verbose to print the parsed tree.
+ """
+ revs = None
+ if opts['rev']:
+ if repo is None:
+ raise error.RepoError(_('there is no Mercurial repository here '
+ '(.hg not found)'))
+ revs = scmutil.revrange(repo, opts['rev'])
+
+ props = {}
+ for d in opts['define']:
+ try:
+ k, v = (e.strip() for e in d.split('=', 1))
+ if not k:
+ raise ValueError
+ props[k] = v
+ except ValueError:
+ raise error.Abort(_('malformed keyword definition: %s') % d)
+
+ if ui.verbose:
+ aliases = ui.configitems('templatealias')
+ tree = templater.parse(tmpl)
+ ui.note(templater.prettyformat(tree), '\n')
+ newtree = templater.expandaliases(tree, aliases)
+ if newtree != tree:
+ ui.note(("* expanded:\n"), templater.prettyformat(newtree), '\n')
+
+ mapfile = None
+ if revs is None:
+ k = 'debugtemplate'
+ t = formatter.maketemplater(ui, k, tmpl)
+ ui.write(templater.stringify(t(k, **props)))
+ else:
+ displayer = cmdutil.changeset_templater(ui, repo, None, opts, tmpl,
+ mapfile, buffered=False)
+ for r in revs:
+ displayer.show(repo[r], **props)
+ displayer.close()
+
@command('debugupgraderepo', [
('o', 'optimize', [], _('extra optimization to perform'), _('NAME')),
('', 'run', False, _('performs an upgrade')),
@@ -875,3 +2026,43 @@
unable to access the repository should be low.
"""
return repair.upgraderepo(ui, repo, run=run, optimize=optimize)
+
+@command('debugwalk', commands.walkopts, _('[OPTION]... [FILE]...'),
+ inferrepo=True)
+def debugwalk(ui, repo, *pats, **opts):
+ """show how files match on given patterns"""
+ m = scmutil.match(repo[None], pats, opts)
+ items = list(repo.walk(m))
+ if not items:
+ return
+ f = lambda fn: fn
+ if ui.configbool('ui', 'slash') and pycompat.ossep != '/':
+ f = lambda fn: util.normpath(fn)
+ fmt = 'f %%-%ds %%-%ds %%s' % (
+ max([len(abs) for abs in items]),
+ max([len(m.rel(abs)) for abs in items]))
+ for abs in items:
+ line = fmt % (abs, f(m.rel(abs)), m.exact(abs) and 'exact' or '')
+ ui.write("%s\n" % line.rstrip())
+
+@command('debugwireargs',
+ [('', 'three', '', 'three'),
+ ('', 'four', '', 'four'),
+ ('', 'five', '', 'five'),
+ ] + commands.remoteopts,
+ _('REPO [OPTIONS]... [ONE [TWO]]'),
+ norepo=True)
+def debugwireargs(ui, repopath, *vals, **opts):
+ repo = hg.peer(ui, opts, repopath)
+ for opt in commands.remoteopts:
+ del opts[opt[1]]
+ args = {}
+ for k, v in opts.iteritems():
+ if v:
+ args[k] = v
+ # run twice to check that we don't mess up the stream for the next command
+ res1 = repo.debugwireargs(*vals, **args)
+ res2 = repo.debugwireargs(*vals, **args)
+ ui.write("%s\n" % res1)
+ if res1 != res2:
+ ui.warn("%s\n" % res2)
--- a/mercurial/destutil.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/destutil.py Tue Feb 28 11:13:25 2017 -0800
@@ -12,37 +12,10 @@
bookmarks,
error,
obsolete,
+ scmutil,
)
-def _destupdatevalidate(repo, rev, clean, check):
- """validate that the destination comply to various rules
-
- This exists as its own function to help wrapping from extensions."""
- wc = repo[None]
- p1 = wc.p1()
- if not clean:
- # Check that the update is linear.
- #
- # Mercurial do not allow update-merge for non linear pattern
- # (that would be technically possible but was considered too confusing
- # for user a long time ago)
- #
- # See mercurial.merge.update for details
- if p1.rev() not in repo.changelog.ancestors([rev], inclusive=True):
- dirty = wc.dirty(missing=True)
- foreground = obsolete.foreground(repo, [p1.node()])
- if not repo[rev].node() in foreground:
- if dirty:
- msg = _("uncommitted changes")
- hint = _("commit and merge, or update --clean to"
- " discard changes")
- raise error.UpdateAbort(msg, hint=hint)
- elif not check: # destination is not a descendant.
- msg = _("not a linear update")
- hint = _("merge or update --check to force update")
- raise error.UpdateAbort(msg, hint=hint)
-
-def _destupdateobs(repo, clean, check):
+def _destupdateobs(repo, clean):
"""decide of an update destination from obsolescence markers"""
node = None
wc = repo[None]
@@ -78,7 +51,7 @@
movemark = repo['.'].node()
return node, movemark, None
-def _destupdatebook(repo, clean, check):
+def _destupdatebook(repo, clean):
"""decide on an update destination from active bookmark"""
# we also move the active bookmark, if any
activemark = None
@@ -87,7 +60,7 @@
activemark = node
return node, movemark, activemark
-def _destupdatebranch(repo, clean, check):
+def _destupdatebranch(repo, clean):
"""decide on an update destination from current branch
This ignores closed branch heads.
@@ -113,7 +86,7 @@
node = repo['.'].node()
return node, movemark, None
-def _destupdatebranchfallback(repo, clean, check):
+def _destupdatebranchfallback(repo, clean):
"""decide on an update destination from closed heads in current branch"""
wc = repo[None]
currentbranch = wc.branch()
@@ -143,7 +116,7 @@
'branchfallback': _destupdatebranchfallback,
}
-def destupdate(repo, clean=False, check=False):
+def destupdate(repo, clean=False):
"""destination for bare update operation
return (rev, movemark, activemark)
@@ -156,13 +129,11 @@
node = movemark = activemark = None
for step in destupdatesteps:
- node, movemark, activemark = destupdatestepmap[step](repo, clean, check)
+ node, movemark, activemark = destupdatestepmap[step](repo, clean)
if node is not None:
break
rev = repo[node].rev()
- _destupdatevalidate(repo, rev, clean, check)
-
return rev, movemark, activemark
msgdestmerge = {
@@ -372,9 +343,6 @@
def desthistedit(ui, repo):
"""Default base revision to edit for `hg histedit`."""
- # Avoid cycle: scmutil -> revset -> destutil
- from . import scmutil
-
default = ui.config('histedit', 'defaultrev', histeditdefaultrevset)
if default:
revs = scmutil.revrange(repo, [default])
--- a/mercurial/dirstate.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/dirstate.py Tue Feb 28 11:13:25 2017 -0800
@@ -23,6 +23,7 @@
pathutil,
pycompat,
scmutil,
+ txnutil,
util,
)
@@ -59,22 +60,6 @@
return set(fname for fname, e in dmap.iteritems()
if e[0] != 'n' or e[3] == -1)
-def _trypending(root, vfs, filename):
- '''Open file to be read according to HG_PENDING environment variable
-
- This opens '.pending' of specified 'filename' only when HG_PENDING
- is equal to 'root'.
-
- This returns '(fp, is_pending_opened)' tuple.
- '''
- if root == encoding.environ.get('HG_PENDING'):
- try:
- return (vfs('%s.pending' % filename), True)
- except IOError as inst:
- if inst.errno != errno.ENOENT:
- raise
- return (vfs(filename), False)
-
class dirstate(object):
def __init__(self, opener, ui, root, validate):
@@ -385,7 +370,7 @@
raise
def _opendirstatefile(self):
- fp, mode = _trypending(self._root, self._opener, self._filename)
+ fp, mode = txnutil.trypending(self._root, self._opener, self._filename)
if self._pendingmode is not None and self._pendingmode != mode:
fp.close()
raise error.Abort(_('working directory state may be '
--- a/mercurial/dispatch.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/dispatch.py Tue Feb 28 11:13:25 2017 -0800
@@ -33,6 +33,7 @@
extensions,
fancyopts,
fileset,
+ help,
hg,
hook,
profiling,
@@ -123,7 +124,7 @@
return -1
msg = ' '.join(' ' in a and repr(a) or a for a in req.args)
- starttime = time.time()
+ starttime = util.timer()
ret = None
try:
ret = _runcatch(req)
@@ -135,8 +136,11 @@
raise
ret = -1
finally:
- duration = time.time() - starttime
+ duration = util.timer() - starttime
req.ui.flush()
+ if req.ui.logblockedtimes:
+ req.ui._blockedtimes['command_duration'] = duration * 1000
+ req.ui.log('uiblocked', 'ui blocked ms', **req.ui._blockedtimes)
req.ui.log("commandfinish", "%s exited %s after %0.2f seconds\n",
msg, ret or 0, duration)
return ret
@@ -239,19 +243,24 @@
_formatparse(ui.warn, inst)
return -1
except error.UnknownCommand as inst:
- ui.warn(_("hg: unknown command '%s'\n") % inst.args[0])
+ nocmdmsg = _("hg: unknown command '%s'\n") % inst.args[0]
try:
# check if the command is in a disabled extension
# (but don't check for extensions themselves)
- commands.help_(ui, inst.args[0], unknowncmd=True)
+ formatted = help.formattedhelp(ui, inst.args[0], unknowncmd=True)
+ ui.warn(nocmdmsg)
+ ui.write(formatted)
except (error.UnknownCommand, error.Abort):
suggested = False
if len(inst.args) == 2:
sim = _getsimilar(inst.args[1], inst.args[0])
if sim:
+ ui.warn(nocmdmsg)
_reportsimilar(ui.warn, sim)
suggested = True
if not suggested:
+ ui.pager('help')
+ ui.warn(nocmdmsg)
commands.help_(ui, 'shortlist')
except IOError:
raise
@@ -655,107 +664,120 @@
rpath = _earlygetopt(["-R", "--repository", "--repo"], args)
path, lui = _getlocal(ui, rpath)
- # Configure extensions in phases: uisetup, extsetup, cmdtable, and
- # reposetup. Programs like TortoiseHg will call _dispatch several
- # times so we keep track of configured extensions in _loaded.
- extensions.loadall(lui)
- exts = [ext for ext in extensions.extensions() if ext[0] not in _loaded]
- # Propagate any changes to lui.__class__ by extensions
- ui.__class__ = lui.__class__
-
- # (uisetup and extsetup are handled in extensions.loadall)
-
- for name, module in exts:
- for objname, loadermod, loadername in extraloaders:
- extraobj = getattr(module, objname, None)
- if extraobj is not None:
- getattr(loadermod, loadername)(ui, name, extraobj)
- _loaded.add(name)
-
- # (reposetup is handled in hg.repository)
-
# Side-effect of accessing is debugcommands module is guaranteed to be
# imported and commands.table is populated.
debugcommands.command
- addaliases(lui, commands.table)
-
- # All aliases and commands are completely defined, now.
- # Check abbreviation/ambiguity of shell alias.
- shellaliasfn = _checkshellalias(lui, ui, args)
- if shellaliasfn:
- with profiling.maybeprofile(lui):
- return shellaliasfn()
-
- # check for fallback encoding
- fallback = lui.config('ui', 'fallbackencoding')
- if fallback:
- encoding.fallbackencoding = fallback
-
- fullargs = args
- cmd, func, args, options, cmdoptions = _parse(lui, args)
-
- if options["config"]:
- raise error.Abort(_("option --config may not be abbreviated!"))
- if options["cwd"]:
- raise error.Abort(_("option --cwd may not be abbreviated!"))
- if options["repository"]:
- raise error.Abort(_(
- "option -R has to be separated from other options (e.g. not -qR) "
- "and --repository may only be abbreviated as --repo!"))
-
- if options["encoding"]:
- encoding.encoding = options["encoding"]
- if options["encodingmode"]:
- encoding.encodingmode = options["encodingmode"]
- if options["time"]:
- def get_times():
- t = os.times()
- if t[4] == 0.0: # Windows leaves this as zero, so use time.clock()
- t = (t[0], t[1], t[2], t[3], time.clock())
- return t
- s = get_times()
- def print_time():
- t = get_times()
- ui.warn(_("time: real %.3f secs (user %.3f+%.3f sys %.3f+%.3f)\n") %
- (t[4]-s[4], t[0]-s[0], t[2]-s[2], t[1]-s[1], t[3]-s[3]))
- atexit.register(print_time)
-
uis = set([ui, lui])
if req.repo:
uis.add(req.repo.ui)
- if options['verbose'] or options['debug'] or options['quiet']:
- for opt in ('verbose', 'debug', 'quiet'):
- val = str(bool(options[opt]))
- for ui_ in uis:
- ui_.setconfig('ui', opt, val, '--' + opt)
-
- if options['profile']:
+ if '--profile' in args:
for ui_ in uis:
ui_.setconfig('profiling', 'enabled', 'true', '--profile')
- if options['traceback']:
- for ui_ in uis:
- ui_.setconfig('ui', 'traceback', 'on', '--traceback')
+ with profiling.maybeprofile(lui):
+ # Configure extensions in phases: uisetup, extsetup, cmdtable, and
+ # reposetup. Programs like TortoiseHg will call _dispatch several
+ # times so we keep track of configured extensions in _loaded.
+ extensions.loadall(lui)
+ exts = [ext for ext in extensions.extensions() if ext[0] not in _loaded]
+ # Propagate any changes to lui.__class__ by extensions
+ ui.__class__ = lui.__class__
+
+ # (uisetup and extsetup are handled in extensions.loadall)
+
+ for name, module in exts:
+ for objname, loadermod, loadername in extraloaders:
+ extraobj = getattr(module, objname, None)
+ if extraobj is not None:
+ getattr(loadermod, loadername)(ui, name, extraobj)
+ _loaded.add(name)
+
+ # (reposetup is handled in hg.repository)
+
+ addaliases(lui, commands.table)
+
+ # All aliases and commands are completely defined, now.
+ # Check abbreviation/ambiguity of shell alias.
+ shellaliasfn = _checkshellalias(lui, ui, args)
+ if shellaliasfn:
+ return shellaliasfn()
+
+ # check for fallback encoding
+ fallback = lui.config('ui', 'fallbackencoding')
+ if fallback:
+ encoding.fallbackencoding = fallback
+
+ fullargs = args
+ cmd, func, args, options, cmdoptions = _parse(lui, args)
+
+ if options["config"]:
+ raise error.Abort(_("option --config may not be abbreviated!"))
+ if options["cwd"]:
+ raise error.Abort(_("option --cwd may not be abbreviated!"))
+ if options["repository"]:
+ raise error.Abort(_(
+ "option -R has to be separated from other options (e.g. not "
+ "-qR) and --repository may only be abbreviated as --repo!"))
- if options['noninteractive']:
- for ui_ in uis:
- ui_.setconfig('ui', 'interactive', 'off', '-y')
+ if options["encoding"]:
+ encoding.encoding = options["encoding"]
+ if options["encodingmode"]:
+ encoding.encodingmode = options["encodingmode"]
+ if options["time"]:
+ def get_times():
+ t = os.times()
+ if t[4] == 0.0:
+ # Windows leaves this as zero, so use time.clock()
+ t = (t[0], t[1], t[2], t[3], time.clock())
+ return t
+ s = get_times()
+ def print_time():
+ t = get_times()
+ ui.warn(
+ _("time: real %.3f secs (user %.3f+%.3f sys %.3f+%.3f)\n") %
+ (t[4]-s[4], t[0]-s[0], t[2]-s[2], t[1]-s[1], t[3]-s[3]))
+ atexit.register(print_time)
- if cmdoptions.get('insecure', False):
+ if options['verbose'] or options['debug'] or options['quiet']:
+ for opt in ('verbose', 'debug', 'quiet'):
+ val = str(bool(options[opt]))
+ for ui_ in uis:
+ ui_.setconfig('ui', opt, val, '--' + opt)
+
+ if options['traceback']:
+ for ui_ in uis:
+ ui_.setconfig('ui', 'traceback', 'on', '--traceback')
+
+ if options['noninteractive']:
+ for ui_ in uis:
+ ui_.setconfig('ui', 'interactive', 'off', '-y')
+
+ if util.parsebool(options['pager']):
+ ui.pager('internal-always-' + cmd)
+ elif options['pager'] != 'auto':
+ ui.disablepager()
+
+ if cmdoptions.get('insecure', False):
+ for ui_ in uis:
+ ui_.insecureconnections = True
+
+ # setup color handling
+ coloropt = options['color']
for ui_ in uis:
- ui_.insecureconnections = True
+ if coloropt:
+ ui_.setconfig('ui', 'color', coloropt, '--color')
+ color.setup(ui_)
- if options['version']:
- return commands.version_(ui)
- if options['help']:
- return commands.help_(ui, cmd, command=cmd is not None)
- elif not cmd:
- return commands.help_(ui, 'shortlist')
+ if options['version']:
+ return commands.version_(ui)
+ if options['help']:
+ return commands.help_(ui, cmd, command=cmd is not None)
+ elif not cmd:
+ return commands.help_(ui, 'shortlist')
- with profiling.maybeprofile(lui):
repo = None
cmdpats = args[:]
if not func.norepo:
--- a/mercurial/exchange.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/exchange.py Tue Feb 28 11:13:25 2017 -0800
@@ -1724,9 +1724,15 @@
if url.startswith('remote:http:') or url.startswith('remote:https:'):
captureoutput = True
try:
+ # note: outside bundle1, 'heads' is expected to be empty and this
+ # 'check_heads' call wil be a no-op
check_heads(repo, heads, 'uploading changes')
# push can proceed
- if util.safehasattr(cg, 'params'):
+ if not util.safehasattr(cg, 'params'):
+ # legacy case: bundle1 (changegroup 01)
+ lockandtr[1] = repo.lock()
+ r = cg.apply(repo, source, url)
+ else:
r = None
try:
def gettransaction():
@@ -1765,9 +1771,6 @@
mandatory=False)
parts.append(part)
raise
- else:
- lockandtr[1] = repo.lock()
- r = cg.apply(repo, source, url)
finally:
lockmod.release(lockandtr[2], lockandtr[1], lockandtr[0])
if recordout is not None:
--- a/mercurial/extensions.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/extensions.py Tue Feb 28 11:13:25 2017 -0800
@@ -362,7 +362,8 @@
'''find paths of disabled extensions. returns a dict of {name: path}
removes /__init__.py from packages if strip_init is True'''
import hgext
- extpath = os.path.dirname(os.path.abspath(hgext.__file__))
+ extpath = os.path.dirname(
+ os.path.abspath(pycompat.fsencode(hgext.__file__)))
try: # might not be a filesystem path
files = os.listdir(extpath)
except OSError:
--- a/mercurial/filemerge.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/filemerge.py Tue Feb 28 11:13:25 2017 -0800
@@ -489,6 +489,9 @@
args = util.interpolate(r'\$', replace, args,
lambda s: util.shellquote(util.localpath(s)))
cmd = toolpath + ' ' + args
+ if _toolbool(ui, tool, "gui"):
+ repo.ui.status(_('running merge tool %s for file %s\n') %
+ (tool, fcd.path()))
repo.ui.debug('launching merge tool: %s\n' % cmd)
r = ui.system(cmd, cwd=repo.root, environ=env)
repo.ui.debug('merge tool returned: %s\n' % r)
@@ -582,7 +585,7 @@
pre = "%s~%s." % (os.path.basename(fullbase), prefix)
(fd, name) = tempfile.mkstemp(prefix=pre, suffix=ext)
data = repo.wwritedata(ctx.path(), ctx.data())
- f = os.fdopen(fd, "wb")
+ f = os.fdopen(fd, pycompat.sysstr("wb"))
f.write(data)
f.close()
return name
--- a/mercurial/graphmod.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/graphmod.py Tue Feb 28 11:13:25 2017 -0800
@@ -22,6 +22,7 @@
from .node import nullrev
from . import (
revset,
+ smartset,
util,
)
@@ -67,8 +68,8 @@
if gp is None:
# precompute slow query as we know reachableroots() goes
# through all revs (issue4782)
- if not isinstance(revs, revset.baseset):
- revs = revset.baseset(revs)
+ if not isinstance(revs, smartset.baseset):
+ revs = smartset.baseset(revs)
gp = gpcache[mpar] = sorted(set(revset.reachableroots(
repo, revs, [mpar])))
if not gp:
--- a/mercurial/help.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/help.py Tue Feb 28 11:13:25 2017 -0800
@@ -33,14 +33,17 @@
webcommands,
)
-_exclkeywords = [
+_exclkeywords = set([
+ "(ADVANCED)",
"(DEPRECATED)",
"(EXPERIMENTAL)",
+ # i18n: "(ADVANCED)" is a keyword, must be translated consistently
+ _("(ADVANCED)"),
# i18n: "(DEPRECATED)" is a keyword, must be translated consistently
_("(DEPRECATED)"),
# i18n: "(EXPERIMENTAL)" is a keyword, must be translated consistently
_("(EXPERIMENTAL)"),
- ]
+ ])
def listexts(header, exts, indent=1, showdeprecated=False):
'''return a text listing of the given extensions'''
@@ -230,6 +233,7 @@
loaddoc('scripting')),
(['internals'], _("Technical implementation topics"),
internalshelp),
+ (['pager'], _("Pager Support"), loaddoc('pager')),
])
# Maps topics with sub-topics to a list of their sub-topics.
@@ -605,3 +609,47 @@
rst.extend(helplist(None, **opts))
return ''.join(rst)
+
+def formattedhelp(ui, name, keep=None, unknowncmd=False, full=True, **opts):
+ """get help for a given topic (as a dotted name) as rendered rst
+
+ Either returns the rendered help text or raises an exception.
+ """
+ if keep is None:
+ keep = []
+ fullname = name
+ section = None
+ subtopic = None
+ if name and '.' in name:
+ name, remaining = name.split('.', 1)
+ remaining = encoding.lower(remaining)
+ if '.' in remaining:
+ subtopic, section = remaining.split('.', 1)
+ else:
+ if name in subtopics:
+ subtopic = remaining
+ else:
+ section = remaining
+ textwidth = ui.configint('ui', 'textwidth', 78)
+ termwidth = ui.termwidth() - 2
+ if textwidth <= 0 or termwidth < textwidth:
+ textwidth = termwidth
+ text = help_(ui, name,
+ subtopic=subtopic, unknowncmd=unknowncmd, full=full, **opts)
+
+ formatted, pruned = minirst.format(text, textwidth, keep=keep,
+ section=section)
+
+ # We could have been given a weird ".foo" section without a name
+ # to look for, or we could have simply failed to found "foo.bar"
+ # because bar isn't a section of foo
+ if section and not (formatted and name):
+ raise error.Abort(_("help section not found: %s") % fullname)
+
+ if 'verbose' in pruned:
+ keep.append('omitted')
+ else:
+ keep.append('notomitted')
+ formatted, pruned = minirst.format(text, textwidth, keep=keep,
+ section=section)
+ return formatted
--- a/mercurial/help/config.txt Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/help/config.txt Tue Feb 28 11:13:25 2017 -0800
@@ -56,6 +56,7 @@
- ``<repo>/.hg/hgrc`` (per-repository)
- ``$HOME/.hgrc`` (per-user)
+ - ``${XDG_CONFIG_HOME:-$HOME/.config}/hg/hgrc`` (per-user)
- ``<install-root>/etc/mercurial/hgrc`` (per-installation)
- ``<install-root>/etc/mercurial/hgrc.d/*.rc`` (per-installation)
- ``/etc/mercurial/hgrc`` (per-system)
@@ -276,7 +277,7 @@
will let you do ``hg echo foo`` to have ``foo`` printed in your
terminal. A better example might be::
- purge = !$HG status --no-status --unknown -0 re: | xargs -0 rm
+ purge = !$HG status --no-status --unknown -0 re: | xargs -0 rm -f
which will make ``hg purge`` delete all unknown files in the
repository in the same manner as the purge extension.
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/mercurial/help/pager.txt Tue Feb 28 11:13:25 2017 -0800
@@ -0,0 +1,35 @@
+Some Mercurial commands produce a lot of output, and Mercurial will
+attempt to use a pager to make those commands more pleasant.
+
+To set the pager that should be used, set the application variable::
+
+ [pager]
+ pager = less -FRX
+
+If no pager is set, the pager extensions uses the environment variable
+$PAGER. If neither pager.pager, nor $PAGER is set, a default pager
+will be used, typically `more`.
+
+You can disable the pager for certain commands by adding them to the
+pager.ignore list::
+
+ [pager]
+ ignore = version, help, update
+
+To ignore global commands like :hg:`version` or :hg:`help`, you have
+to specify them in your user configuration file.
+
+To control whether the pager is used at all for an individual command,
+you can use --pager=<value>::
+
+ - use as needed: `auto`.
+ - require the pager: `yes` or `on`.
+ - suppress the pager: `no` or `off` (any unrecognized value
+ will also work).
+
+To globally turn off all attempts to use a pager, set::
+
+ [pager]
+ enable = false
+
+which will prevent the pager from running.
--- a/mercurial/help/patterns.txt Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/help/patterns.txt Tue Feb 28 11:13:25 2017 -0800
@@ -13,7 +13,10 @@
To use a plain path name without any pattern matching, start it with
``path:``. These path names must completely match starting at the
-current repository root.
+current repository root, and when the path points to a directory, it is matched
+recursively. To match all files in a directory non-recursively (not including
+any files in subdirectories), ``rootfilesin:`` can be used, specifying an
+absolute path (relative to the repository root).
To use an extended glob, start a name with ``glob:``. Globs are rooted
at the current directory; a glob such as ``*.c`` will only match files
@@ -39,12 +42,15 @@
All patterns, except for ``glob:`` specified in command line (not for
``-I`` or ``-X`` options), can match also against directories: files
under matched directories are treated as matched.
+For ``-I`` and ``-X`` options, ``glob:`` will match directories recursively.
Plain examples::
- path:foo/bar a name bar in a directory named foo in the root
- of the repository
- path:path:name a file or directory named "path:name"
+ path:foo/bar a name bar in a directory named foo in the root
+ of the repository
+ path:path:name a file or directory named "path:name"
+ rootfilesin:foo/bar the files in a directory called foo/bar, but not any files
+ in its subdirectories and not a file bar in directory foo
Glob examples::
@@ -52,6 +58,8 @@
*.c any name ending in ".c" in the current directory
**.c any name ending in ".c" in any subdirectory of the
current directory including itself.
+ foo/* any file in directory foo plus all its subdirectories,
+ recursively
foo/*.c any name ending in ".c" in the directory foo
foo/**.c any name ending in ".c" in any subdirectory of foo
including itself.
--- a/mercurial/hg.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/hg.py Tue Feb 28 11:13:25 2017 -0800
@@ -732,13 +732,15 @@
movemarkfrom = None
warndest = False
if checkout is None:
- updata = destutil.destupdate(repo, clean=clean, check=check)
+ updata = destutil.destupdate(repo, clean=clean)
checkout, movemarkfrom, brev = updata
warndest = True
if clean:
ret = _clean(repo, checkout)
else:
+ if check:
+ cmdutil.bailifchanged(repo, merge=False)
ret = _update(repo, checkout)
if not ret and movemarkfrom:
@@ -802,7 +804,7 @@
if not chlist:
ui.status(_("no changes found\n"))
return subreporecurse()
-
+ ui.pager('incoming')
displayer = cmdutil.show_changeset(ui, other, opts, buffered)
displaychlist(other, chlist, displayer)
displayer.close()
@@ -870,6 +872,7 @@
if opts.get('newest_first'):
o.reverse()
+ ui.pager('outgoing')
displayer = cmdutil.show_changeset(ui, repo, opts)
count = 0
for n in o:
--- a/mercurial/hgweb/webcommands.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/hgweb/webcommands.py Tue Feb 28 11:13:25 2017 -0800
@@ -32,7 +32,9 @@
error,
graphmod,
revset,
+ revsetlang,
scmutil,
+ smartset,
templatefilters,
templater,
util,
@@ -238,20 +240,20 @@
revdef = 'reverse(%s)' % query
try:
- tree = revset.parse(revdef)
+ tree = revsetlang.parse(revdef)
except error.ParseError:
# can't parse to a revset tree
return MODE_KEYWORD, query
- if revset.depth(tree) <= 2:
+ if revsetlang.depth(tree) <= 2:
# no revset syntax used
return MODE_KEYWORD, query
if any((token, (value or '')[:3]) == ('string', 're:')
- for token, value, pos in revset.tokenize(revdef)):
+ for token, value, pos in revsetlang.tokenize(revdef)):
return MODE_KEYWORD, query
- funcsused = revset.funcsused(tree)
+ funcsused = revsetlang.funcsused(tree)
if not funcsused.issubset(revset.safesymbols):
return MODE_KEYWORD, query
@@ -752,13 +754,14 @@
if fctx is not None:
path = fctx.path()
ctx = fctx.changectx()
+ basectx = ctx.p1()
parity = paritygen(web.stripecount)
style = web.config('web', 'style', 'paper')
if 'style' in req.form:
style = req.form['style'][0]
- diffs = webutil.diffs(web.repo, tmpl, ctx, None, [path], parity, style)
+ diffs = webutil.diffs(web.repo, tmpl, ctx, basectx, [path], parity, style)
if fctx is not None:
rename = webutil.renamelink(fctx)
ctx = fctx
@@ -1148,7 +1151,7 @@
# We have to feed a baseset to dagwalker as it is expecting smartset
# object. This does not have a big impact on hgweb performance itself
# since hgweb graphing code is not itself lazy yet.
- dag = graphmod.dagwalker(web.repo, revset.baseset(revs))
+ dag = graphmod.dagwalker(web.repo, smartset.baseset(revs))
# As we said one line above... not lazy.
tree = list(graphmod.colored(dag, web.repo))
--- a/mercurial/hgweb/webutil.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/hgweb/webutil.py Tue Feb 28 11:13:25 2017 -0800
@@ -442,14 +442,7 @@
m = match.always(repo.root, repo.getcwd())
diffopts = patch.diffopts(repo.ui, untrusted=True)
- if basectx is None:
- parents = ctx.parents()
- if parents:
- node1 = parents[0].node()
- else:
- node1 = nullid
- else:
- node1 = basectx.node()
+ node1 = basectx.node()
node2 = ctx.node()
block = []
--- a/mercurial/hook.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/hook.py Tue Feb 28 11:13:25 2017 -0800
@@ -9,7 +9,6 @@
import os
import sys
-import time
from .i18n import _
from . import (
@@ -88,7 +87,7 @@
% (hname, funcname))
ui.note(_("calling hook %s: %s\n") % (hname, funcname))
- starttime = time.time()
+ starttime = util.timer()
try:
r = obj(ui=ui, repo=repo, hooktype=name, **args)
@@ -106,7 +105,7 @@
ui.traceback()
return True, True
finally:
- duration = time.time() - starttime
+ duration = util.timer() - starttime
ui.log('pythonhook', 'pythonhook-%s: %s finished in %0.2f seconds\n',
name, funcname, duration)
if r:
@@ -118,7 +117,7 @@
def _exthook(ui, repo, name, cmd, args, throw):
ui.note(_("running hook %s: %s\n") % (name, cmd))
- starttime = time.time()
+ starttime = util.timer()
env = {}
# make in-memory changes visible to external process
@@ -145,7 +144,7 @@
cwd = pycompat.getcwd()
r = ui.system(cmd, environ=env, cwd=cwd)
- duration = time.time() - starttime
+ duration = util.timer() - starttime
ui.log('exthook', 'exthook-%s: %s finished in %0.2f seconds\n',
name, cmd, duration)
if r:
--- a/mercurial/httppeer.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/httppeer.py Tue Feb 28 11:13:25 2017 -0800
@@ -20,6 +20,7 @@
bundle2,
error,
httpconnection,
+ pycompat,
statichttprepo,
url,
util,
@@ -327,7 +328,7 @@
try:
# dump bundle to disk
fd, filename = tempfile.mkstemp(prefix="hg-bundle-", suffix=".hg")
- fh = os.fdopen(fd, "wb")
+ fh = os.fdopen(fd, pycompat.sysstr("wb"))
d = fp.read(4096)
while d:
fh.write(d)
--- a/mercurial/i18n.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/i18n.py Tue Feb 28 11:13:25 2017 -0800
@@ -21,7 +21,7 @@
if getattr(sys, 'frozen', None) is not None:
module = pycompat.sysexecutable
else:
- module = __file__
+ module = pycompat.fsencode(__file__)
try:
unicode
--- a/mercurial/keepalive.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/keepalive.py Tue Feb 28 11:13:25 2017 -0800
@@ -310,14 +310,16 @@
try:
if req.has_data():
data = req.get_data()
- h.putrequest('POST', req.get_selector(), **skipheaders)
+ h.putrequest(
+ req.get_method(), req.get_selector(), **skipheaders)
if 'content-type' not in headers:
h.putheader('Content-type',
'application/x-www-form-urlencoded')
if 'content-length' not in headers:
h.putheader('Content-length', '%d' % len(data))
else:
- h.putrequest('GET', req.get_selector(), **skipheaders)
+ h.putrequest(
+ req.get_method(), req.get_selector(), **skipheaders)
except socket.error as err:
raise urlerr.urlerror(err)
for k, v in headers.items():
--- a/mercurial/localrepo.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/localrepo.py Tue Feb 28 11:13:25 2017 -0800
@@ -28,6 +28,7 @@
bundle2,
changegroup,
changelog,
+ color,
context,
dirstate,
dirstateguard,
@@ -50,11 +51,13 @@
pushkey,
repoview,
revset,
+ revsetlang,
scmutil,
store,
subrepo,
tags as tagsmod,
transaction,
+ txnutil,
util,
)
@@ -270,7 +273,7 @@
self._phasedefaults = []
try:
self.ui.readconfig(self.join("hgrc"), self.root)
- extensions.loadall(self.ui)
+ self._loadextensions()
except IOError:
pass
@@ -283,6 +286,7 @@
setupfunc(self.ui, self.supported)
else:
self.supported = self._basesupported
+ color.setup(self.ui)
# Add compression engines.
for name in util.compengines:
@@ -371,6 +375,9 @@
def close(self):
self._writecaches()
+ def _loadextensions(self):
+ extensions.loadall(self.ui)
+
def _writecaches(self):
if self._revbranchcache:
self._revbranchcache.write()
@@ -509,10 +516,8 @@
@storecache('00changelog.i')
def changelog(self):
c = changelog.changelog(self.svfs)
- if 'HG_PENDING' in encoding.environ:
- p = encoding.environ['HG_PENDING']
- if p.startswith(self.root):
- c.readpending('00changelog.i.a')
+ if txnutil.mayhavepending(self.root):
+ c.readpending('00changelog.i.a')
return c
def _constructmanifest(self):
@@ -570,15 +575,16 @@
'''Find revisions matching a revset.
The revset is specified as a string ``expr`` that may contain
- %-formatting to escape certain types. See ``revset.formatspec``.
+ %-formatting to escape certain types. See ``revsetlang.formatspec``.
Revset aliases from the configuration are not expanded. To expand
- user aliases, consider calling ``scmutil.revrange()``.
+ user aliases, consider calling ``scmutil.revrange()`` or
+ ``repo.anyrevs([expr], user=True)``.
Returns a revset.abstractsmartset, which is a list-like interface
that contains integer revisions.
'''
- expr = revset.formatspec(expr, *args)
+ expr = revsetlang.formatspec(expr, *args)
m = revset.match(None, expr)
return m(self)
@@ -594,6 +600,18 @@
for r in self.revs(expr, *args):
yield self[r]
+ def anyrevs(self, specs, user=False):
+ '''Find revisions matching one of the given revsets.
+
+ Revset aliases from the configuration are not expanded by default. To
+ expand user aliases, specify ``user=True``.
+ '''
+ if user:
+ m = revset.matchany(self.ui, specs, repo=self)
+ else:
+ m = revset.matchany(None, specs)
+ return m(self)
+
def url(self):
return 'file:' + self.root
@@ -1852,6 +1870,11 @@
listsubrepos)
def heads(self, start=None):
+ if start is None:
+ cl = self.changelog
+ headrevs = reversed(cl.headrevs())
+ return [cl.node(rev) for rev in headrevs]
+
heads = self.changelog.heads(start)
# sort the output in rev descending order
return sorted(heads, key=self.changelog.rev, reverse=True)
--- a/mercurial/lock.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/lock.py Tue Feb 28 11:13:25 2017 -0800
@@ -9,15 +9,33 @@
import contextlib
import errno
+import os
import socket
import time
import warnings
from . import (
error,
+ pycompat,
util,
)
+def _getlockprefix():
+ """Return a string which is used to differentiate pid namespaces
+
+ It's useful to detect "dead" processes and remove stale locks with
+ confidence. Typically it's just hostname. On modern linux, we include an
+ extra Linux-specific pid namespace identifier.
+ """
+ result = socket.gethostname()
+ if pycompat.sysplatform.startswith('linux'):
+ try:
+ result += '/%x' % os.stat('/proc/self/ns/pid').st_ino
+ except OSError as ex:
+ if ex.errno not in (errno.ENOENT, errno.EACCES, errno.ENOTDIR):
+ raise
+ return result
+
class lock(object):
'''An advisory lock held by one process to control access to a set
of files. Non-cooperating processes or incorrectly written scripts
@@ -99,7 +117,7 @@
self.held += 1
return
if lock._host is None:
- lock._host = socket.gethostname()
+ lock._host = _getlockprefix()
lockname = '%s:%s' % (lock._host, self.pid)
retry = 5
while not self.held and retry:
--- a/mercurial/manifest.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/manifest.py Tue Feb 28 11:13:25 2017 -0800
@@ -1386,7 +1386,7 @@
return self._revlog().parents(self._node)
def read(self):
- if not self._data:
+ if self._data is None:
if self._node == revlog.nullid:
self._data = manifestdict()
else:
@@ -1484,7 +1484,7 @@
return self._repo.manifestlog._revlog.dirlog(self._dir)
def read(self):
- if not self._data:
+ if self._data is None:
rl = self._revlog()
if self._node == revlog.nullid:
self._data = treemanifest()
--- a/mercurial/match.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/match.py Tue Feb 28 11:13:25 2017 -0800
@@ -104,7 +104,10 @@
a pattern is one of:
'glob:<glob>' - a glob relative to cwd
're:<regexp>' - a regular expression
- 'path:<path>' - a path relative to repository root
+ 'path:<path>' - a path relative to repository root, which is matched
+ recursively
+ 'rootfilesin:<path>' - a path relative to repository root, which is
+ matched non-recursively (will not match subdirectories)
'relglob:<glob>' - an unrooted glob (*.c matches C files in all dirs)
'relpath:<path>' - a path relative to cwd
'relre:<regexp>' - a regexp that needn't match the start of a name
@@ -122,9 +125,12 @@
self._always = False
self._pathrestricted = bool(include or exclude or patterns)
self._warn = warn
+
+ # roots are directories which are recursively included/excluded.
self._includeroots = set()
+ self._excluderoots = set()
+ # dirs are directories which are non-recursively included.
self._includedirs = set(['.'])
- self._excluderoots = set()
if badfn is not None:
self.bad = badfn
@@ -134,14 +140,20 @@
kindpats = self._normalize(include, 'glob', root, cwd, auditor)
self.includepat, im = _buildmatch(ctx, kindpats, '(?:/|$)',
listsubrepos, root)
- self._includeroots.update(_roots(kindpats))
- self._includedirs.update(util.dirs(self._includeroots))
+ roots, dirs = _rootsanddirs(kindpats)
+ self._includeroots.update(roots)
+ self._includedirs.update(dirs)
matchfns.append(im)
if exclude:
kindpats = self._normalize(exclude, 'glob', root, cwd, auditor)
self.excludepat, em = _buildmatch(ctx, kindpats, '(?:/|$)',
listsubrepos, root)
if not _anypats(kindpats):
+ # Only consider recursive excludes as such - if a non-recursive
+ # exclude is used, we must still recurse into the excluded
+ # directory, at least to find subdirectories. In such a case,
+ # the regex still won't match the non-recursively-excluded
+ # files.
self._excluderoots.update(_roots(kindpats))
matchfns.append(lambda f: not em(f))
if exact:
@@ -153,7 +165,7 @@
elif patterns:
kindpats = self._normalize(patterns, default, root, cwd, auditor)
if not _kindpatsalwaysmatch(kindpats):
- self._files = _roots(kindpats)
+ self._files = _explicitfiles(kindpats)
self._anypats = self._anypats or _anypats(kindpats)
self.patternspat, pm = _buildmatch(ctx, kindpats, '$',
listsubrepos, root)
@@ -238,7 +250,7 @@
return 'all'
if dir in self._excluderoots:
return False
- if (self._includeroots and
+ if ((self._includeroots or self._includedirs != set(['.'])) and
'.' not in self._includeroots and
dir not in self._includeroots and
dir not in self._includedirs and
@@ -286,7 +298,7 @@
for kind, pat in [_patsplit(p, default) for p in patterns]:
if kind in ('glob', 'relpath'):
pat = pathutil.canonpath(root, cwd, pat, auditor)
- elif kind in ('relglob', 'path'):
+ elif kind in ('relglob', 'path', 'rootfilesin'):
pat = util.normpath(pat)
elif kind in ('listfile', 'listfile0'):
try:
@@ -419,7 +431,9 @@
# m.exact(file) must be based off of the actual user input, otherwise
# inexact case matches are treated as exact, and not noted without -v.
if self._files:
- self._fileroots = set(_roots(self._kp))
+ roots, dirs = _rootsanddirs(self._kp)
+ self._fileroots = set(roots)
+ self._fileroots.update(dirs)
def _normalize(self, patterns, default, root, cwd, auditor):
self._kp = super(icasefsmatcher, self)._normalize(patterns, default,
@@ -447,7 +461,8 @@
if ':' in pattern:
kind, pat = pattern.split(':', 1)
if kind in ('re', 'glob', 'path', 'relglob', 'relpath', 'relre',
- 'listfile', 'listfile0', 'set', 'include', 'subinclude'):
+ 'listfile', 'listfile0', 'set', 'include', 'subinclude',
+ 'rootfilesin'):
return kind, pat
return default, pattern
@@ -540,6 +555,14 @@
if pat == '.':
return ''
return '^' + util.re.escape(pat) + '(?:/|$)'
+ if kind == 'rootfilesin':
+ if pat == '.':
+ escaped = ''
+ else:
+ # Pattern is a directory name.
+ escaped = util.re.escape(pat) + '/'
+ # Anything after the pattern must be a non-directory.
+ return '^' + escaped + '[^/]+$'
if kind == 'relglob':
return '(?:|.*/)' + _globre(pat) + globsuffix
if kind == 'relpath':
@@ -609,17 +632,16 @@
raise error.Abort(_("invalid pattern (%s): %s") % (k, p))
raise error.Abort(_("invalid pattern"))
-def _roots(kindpats):
- '''return roots and exact explicitly listed files from patterns
+def _patternrootsanddirs(kindpats):
+ '''Returns roots and directories corresponding to each pattern.
- >>> _roots([('glob', 'g/*', ''), ('glob', 'g', ''), ('glob', 'g*', '')])
- ['g', 'g', '.']
- >>> _roots([('relpath', 'r', ''), ('path', 'p/p', ''), ('path', '', '')])
- ['r', 'p/p', '.']
- >>> _roots([('relglob', 'rg*', ''), ('re', 're/', ''), ('relre', 'rr', '')])
- ['.', '.', '.']
+ This calculates the roots and directories exactly matching the patterns and
+ returns a tuple of (roots, dirs) for each. It does not return other
+ directories which may also need to be considered, like the parent
+ directories.
'''
r = []
+ d = []
for kind, pat, source in kindpats:
if kind == 'glob': # find the non-glob prefix
root = []
@@ -630,13 +652,63 @@
r.append('/'.join(root) or '.')
elif kind in ('relpath', 'path'):
r.append(pat or '.')
+ elif kind in ('rootfilesin',):
+ d.append(pat or '.')
else: # relglob, re, relre
r.append('.')
- return r
+ return r, d
+
+def _roots(kindpats):
+ '''Returns root directories to match recursively from the given patterns.'''
+ roots, dirs = _patternrootsanddirs(kindpats)
+ return roots
+
+def _rootsanddirs(kindpats):
+ '''Returns roots and exact directories from patterns.
+
+ roots are directories to match recursively, whereas exact directories should
+ be matched non-recursively. The returned (roots, dirs) tuple will also
+ include directories that need to be implicitly considered as either, such as
+ parent directories.
+
+ >>> _rootsanddirs(\
+ [('glob', 'g/h/*', ''), ('glob', 'g/h', ''), ('glob', 'g*', '')])
+ (['g/h', 'g/h', '.'], ['g'])
+ >>> _rootsanddirs(\
+ [('rootfilesin', 'g/h', ''), ('rootfilesin', '', '')])
+ ([], ['g/h', '.', 'g'])
+ >>> _rootsanddirs(\
+ [('relpath', 'r', ''), ('path', 'p/p', ''), ('path', '', '')])
+ (['r', 'p/p', '.'], ['p'])
+ >>> _rootsanddirs(\
+ [('relglob', 'rg*', ''), ('re', 're/', ''), ('relre', 'rr', '')])
+ (['.', '.', '.'], [])
+ '''
+ r, d = _patternrootsanddirs(kindpats)
+
+ # Append the parents as non-recursive/exact directories, since they must be
+ # scanned to get to either the roots or the other exact directories.
+ d.extend(util.dirs(d))
+ d.extend(util.dirs(r))
+
+ return r, d
+
+def _explicitfiles(kindpats):
+ '''Returns the potential explicit filenames from the patterns.
+
+ >>> _explicitfiles([('path', 'foo/bar', '')])
+ ['foo/bar']
+ >>> _explicitfiles([('rootfilesin', 'foo/bar', '')])
+ []
+ '''
+ # Keep only the pattern kinds where one can specify filenames (vs only
+ # directory names).
+ filable = [kp for kp in kindpats if kp[0] not in ('rootfilesin',)]
+ return _roots(filable)
def _anypats(kindpats):
for kind, pat, source in kindpats:
- if kind in ('glob', 're', 'relglob', 'relre', 'set'):
+ if kind in ('glob', 're', 'relglob', 'relre', 'set', 'rootfilesin'):
return True
_commentre = None
--- a/mercurial/merge.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/merge.py Tue Feb 28 11:13:25 2017 -0800
@@ -1448,7 +1448,7 @@
"""
Perform a merge between the working directory and the given node
- node = the node to update to, or None if unspecified
+ node = the node to update to
branchmerge = whether to merge between branches
force = whether to force branch merging or file overwriting
matcher = a matcher to filter file lists (dirstate not updated)
@@ -1491,7 +1491,9 @@
Return the same tuple as applyupdates().
"""
- onode = node
+ # This functon used to find the default destination if node was None, but
+ # that's now in destutil.py.
+ assert node is not None
# If we're doing a partial update, we need to skip updating
# the dirstate, so make a note of any partial-ness to the
# update here.
@@ -1550,37 +1552,30 @@
if pas not in ([p1], [p2]): # nonlinear
dirty = wc.dirty(missing=True)
- if dirty or onode is None:
+ if dirty:
# Branching is a bit strange to ensure we do the minimal
- # amount of call to obsolete.background.
+ # amount of call to obsolete.foreground.
foreground = obsolete.foreground(repo, [p1.node()])
# note: the <node> variable contains a random identifier
if repo[node].node() in foreground:
- pas = [p1] # allow updating to successors
- elif dirty:
+ pass # allow updating to successors
+ else:
msg = _("uncommitted changes")
- if onode is None:
- hint = _("commit and merge, or update --clean to"
- " discard changes")
- else:
- hint = _("commit or update --clean to discard"
- " changes")
- raise error.Abort(msg, hint=hint)
- else: # node is none
- msg = _("not a linear update")
- hint = _("merge or update --check to force update")
- raise error.Abort(msg, hint=hint)
+ hint = _("commit or update --clean to discard changes")
+ raise error.UpdateAbort(msg, hint=hint)
else:
# Allow jumping branches if clean and specific rev given
- pas = [p1]
+ pass
+
+ if overwrite:
+ pas = [wc]
+ elif not branchmerge:
+ pas = [p1]
# deprecated config: merge.followcopies
followcopies = repo.ui.configbool('merge', 'followcopies', True)
if overwrite:
- pas = [wc]
followcopies = False
- elif pas == [p2]: # backwards
- pas = [p1]
elif not pas[0]:
followcopies = False
if not branchmerge and not wc.dirty(missing=True):
--- a/mercurial/obsolete.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/obsolete.py Tue Feb 28 11:13:25 2017 -0800
@@ -1120,7 +1120,7 @@
"""the set of obsolete revisions"""
obs = set()
getnode = repo.changelog.node
- notpublic = repo.revs("not public()")
+ notpublic = repo._phasecache.getrevset(repo, (phases.draft, phases.secret))
for r in notpublic:
if getnode(r) in repo.obsstore.successors:
obs.add(r)
--- a/mercurial/patch.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/patch.py Tue Feb 28 11:13:25 2017 -0800
@@ -34,6 +34,7 @@
mail,
mdiff,
pathutil,
+ pycompat,
scmutil,
similar,
util,
@@ -209,7 +210,7 @@
data = {}
fd, tmpname = tempfile.mkstemp(prefix='hg-patch-')
- tmpfp = os.fdopen(fd, 'w')
+ tmpfp = os.fdopen(fd, pycompat.sysstr('w'))
try:
msg = email.Parser.Parser().parse(fileobj)
@@ -1055,7 +1056,7 @@
ncpatchfp = None
try:
# Write the initial patch
- f = os.fdopen(patchfd, "w")
+ f = os.fdopen(patchfd, pycompat.sysstr("w"))
chunk.header.write(f)
chunk.write(f)
f.write('\n'.join(['# ' + i for i in phelp.splitlines()]))
--- a/mercurial/phases.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/phases.py Tue Feb 28 11:13:25 2017 -0800
@@ -113,8 +113,9 @@
short,
)
from . import (
- encoding,
error,
+ smartset,
+ txnutil,
)
allphases = public, draft, secret = range(3)
@@ -136,15 +137,7 @@
dirty = False
roots = [set() for i in allphases]
try:
- f = None
- if 'HG_PENDING' in encoding.environ:
- try:
- f = repo.svfs('phaseroots.pending')
- except IOError as inst:
- if inst.errno != errno.ENOENT:
- raise
- if f is None:
- f = repo.svfs('phaseroots')
+ f, pending = txnutil.trypending(repo.root, repo.svfs, 'phaseroots')
try:
for line in f:
phase, nh = line.split()
@@ -170,6 +163,27 @@
self.filterunknown(repo)
self.opener = repo.svfs
+ def getrevset(self, repo, phases):
+ """return a smartset for the given phases"""
+ self.loadphaserevs(repo) # ensure phase's sets are loaded
+
+ if self._phasesets and all(self._phasesets[p] is not None
+ for p in phases):
+ # fast path - use _phasesets
+ revs = self._phasesets[phases[0]]
+ if len(phases) > 1:
+ revs = revs.copy() # only copy when needed
+ for p in phases[1:]:
+ revs.update(self._phasesets[p])
+ if repo.changelog.filteredrevs:
+ revs = revs - repo.changelog.filteredrevs
+ return smartset.baseset(revs)
+ else:
+ # slow path - enumerate all revisions
+ phase = self.phase
+ revs = (r for r in repo if phase(repo, r) in phases)
+ return smartset.generatorset(revs, iterasc=True)
+
def copy(self):
# Shallow copy meant to ensure isolation in
# advance/retractboundary(), nothing more.
--- a/mercurial/profiling.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/profiling.py Tue Feb 28 11:13:25 2017 -0800
@@ -8,7 +8,6 @@
from __future__ import absolute_import, print_function
import contextlib
-import time
from .i18n import _
from . import (
@@ -66,7 +65,7 @@
collapse_recursion = True
thread = flamegraph.ProfileThread(fp, 1.0 / freq,
filter_, collapse_recursion)
- start_time = time.clock()
+ start_time = util.timer()
try:
thread.start()
yield
@@ -74,7 +73,7 @@
thread.stop()
thread.join()
print('Collected %d stack frames (%d unique) in %2.2f seconds.' % (
- time.clock() - start_time, thread.num_frames(),
+ util.timer() - start_time, thread.num_frames(),
thread.num_frames(unique=True)))
@contextlib.contextmanager
@@ -103,6 +102,7 @@
'bymethod': statprof.DisplayFormats.ByMethod,
'hotpath': statprof.DisplayFormats.Hotpath,
'json': statprof.DisplayFormats.Json,
+ 'chrome': statprof.DisplayFormats.Chrome,
}
if profformat in formats:
@@ -111,7 +111,23 @@
ui.warn(_('unknown profiler output format: %s\n') % profformat)
displayformat = statprof.DisplayFormats.Hotpath
- statprof.display(fp, data=data, format=displayformat)
+ kwargs = {}
+
+ def fraction(s):
+ if s.endswith('%'):
+ v = float(s[:-1]) / 100
+ else:
+ v = float(s)
+ if 0 <= v <= 1:
+ return v
+ raise ValueError(s)
+
+ if profformat == 'chrome':
+ showmin = ui.configwith(fraction, 'profiling', 'showmin', 0.005)
+ showmax = ui.configwith(fraction, 'profiling', 'showmax', 0.999)
+ kwargs.update(minthreshold=showmin, maxthreshold=showmax)
+
+ statprof.display(fp, data=data, format=displayformat, **kwargs)
@contextlib.contextmanager
def profile(ui):
--- a/mercurial/pure/osutil.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/pure/osutil.py Tue Feb 28 11:13:25 2017 -0800
@@ -338,7 +338,7 @@
_kernel32.CloseHandle(fh)
_raiseioerror(name)
- f = os.fdopen(fd, mode, bufsize)
+ f = os.fdopen(fd, pycompat.sysstr(mode), bufsize)
# unfortunately, f.name is '<fdopen>' at this point -- so we store
# the name on this wrapper. We cannot just assign to f.name,
# because that attribute is read-only.
--- a/mercurial/repair.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/repair.py Tue Feb 28 11:13:25 2017 -0800
@@ -12,7 +12,6 @@
import hashlib
import stat
import tempfile
-import time
from .i18n import _
from .node import short
@@ -905,10 +904,10 @@
# the operation nearly instantaneous and atomic (at least in well-behaved
# environments).
ui.write(_('replacing store...\n'))
- tstart = time.time()
+ tstart = util.timer()
util.rename(srcrepo.spath, backupvfs.join('store'))
util.rename(dstrepo.spath, srcrepo.spath)
- elapsed = time.time() - tstart
+ elapsed = util.timer() - tstart
ui.write(_('store replacement complete; repository was inconsistent for '
'%0.1fs\n') % elapsed)
--- a/mercurial/repoview.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/repoview.py Tue Feb 28 11:13:25 2017 -0800
@@ -139,15 +139,13 @@
if wlock:
wlock.release()
-def tryreadcache(repo, hideable):
- """read a cache if the cache exists and is valid, otherwise returns None."""
+def _readhiddencache(repo, cachefilename, newhash):
hidden = fh = None
try:
if repo.vfs.exists(cachefile):
fh = repo.vfs.open(cachefile, 'rb')
version, = struct.unpack(">H", fh.read(2))
oldhash = fh.read(20)
- newhash = cachehash(repo, hideable)
if (cacheversion, oldhash) == (version, newhash):
# cache is valid, so we can start reading the hidden revs
data = fh.read()
@@ -165,6 +163,11 @@
if fh:
fh.close()
+def tryreadcache(repo, hideable):
+ """read a cache if the cache exists and is valid, otherwise returns None."""
+ newhash = cachehash(repo, hideable)
+ return _readhiddencache(repo, cachefile, newhash)
+
def computehidden(repo):
"""compute the set of hidden revision to filter
--- a/mercurial/revset.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/revset.py Tue Feb 28 11:13:25 2017 -0800
@@ -9,7 +9,6 @@
import heapq
import re
-import string
from .i18n import _
from . import (
@@ -20,15 +19,34 @@
match as matchmod,
node,
obsolete as obsmod,
- parser,
pathutil,
phases,
- pycompat,
registrar,
repoview,
+ revsetlang,
+ smartset,
util,
)
+# helpers for processing parsed tree
+getsymbol = revsetlang.getsymbol
+getstring = revsetlang.getstring
+getinteger = revsetlang.getinteger
+getlist = revsetlang.getlist
+getrange = revsetlang.getrange
+getargs = revsetlang.getargs
+getargsdict = revsetlang.getargsdict
+
+# constants used as an argument of match() and matchany()
+anyorder = revsetlang.anyorder
+defineorder = revsetlang.defineorder
+followorder = revsetlang.followorder
+
+baseset = smartset.baseset
+generatorset = smartset.generatorset
+spanset = smartset.spanset
+fullreposet = smartset.fullreposet
+
def _revancestors(repo, revs, followfirst):
"""Like revlog.ancestors(), but supports followfirst."""
if followfirst:
@@ -146,213 +164,8 @@
revs.sort()
return revs
-elements = {
- # token-type: binding-strength, primary, prefix, infix, suffix
- "(": (21, None, ("group", 1, ")"), ("func", 1, ")"), None),
- "##": (20, None, None, ("_concat", 20), None),
- "~": (18, None, None, ("ancestor", 18), None),
- "^": (18, None, None, ("parent", 18), "parentpost"),
- "-": (5, None, ("negate", 19), ("minus", 5), None),
- "::": (17, None, ("dagrangepre", 17), ("dagrange", 17), "dagrangepost"),
- "..": (17, None, ("dagrangepre", 17), ("dagrange", 17), "dagrangepost"),
- ":": (15, "rangeall", ("rangepre", 15), ("range", 15), "rangepost"),
- "not": (10, None, ("not", 10), None, None),
- "!": (10, None, ("not", 10), None, None),
- "and": (5, None, None, ("and", 5), None),
- "&": (5, None, None, ("and", 5), None),
- "%": (5, None, None, ("only", 5), "onlypost"),
- "or": (4, None, None, ("or", 4), None),
- "|": (4, None, None, ("or", 4), None),
- "+": (4, None, None, ("or", 4), None),
- "=": (3, None, None, ("keyvalue", 3), None),
- ",": (2, None, None, ("list", 2), None),
- ")": (0, None, None, None, None),
- "symbol": (0, "symbol", None, None, None),
- "string": (0, "string", None, None, None),
- "end": (0, None, None, None, None),
-}
-
-keywords = set(['and', 'or', 'not'])
-
-# default set of valid characters for the initial letter of symbols
-_syminitletters = set(
- string.ascii_letters +
- string.digits + pycompat.sysstr('._@')) | set(map(chr, xrange(128, 256)))
-
-# default set of valid characters for non-initial letters of symbols
-_symletters = _syminitletters | set(pycompat.sysstr('-/'))
-
-def tokenize(program, lookup=None, syminitletters=None, symletters=None):
- '''
- Parse a revset statement into a stream of tokens
-
- ``syminitletters`` is the set of valid characters for the initial
- letter of symbols.
-
- By default, character ``c`` is recognized as valid for initial
- letter of symbols, if ``c.isalnum() or c in '._@' or ord(c) > 127``.
-
- ``symletters`` is the set of valid characters for non-initial
- letters of symbols.
-
- By default, character ``c`` is recognized as valid for non-initial
- letters of symbols, if ``c.isalnum() or c in '-._/@' or ord(c) > 127``.
-
- Check that @ is a valid unquoted token character (issue3686):
- >>> list(tokenize("@::"))
- [('symbol', '@', 0), ('::', None, 1), ('end', None, 3)]
-
- '''
- if syminitletters is None:
- syminitletters = _syminitletters
- if symletters is None:
- symletters = _symletters
-
- if program and lookup:
- # attempt to parse old-style ranges first to deal with
- # things like old-tag which contain query metacharacters
- parts = program.split(':', 1)
- if all(lookup(sym) for sym in parts if sym):
- if parts[0]:
- yield ('symbol', parts[0], 0)
- if len(parts) > 1:
- s = len(parts[0])
- yield (':', None, s)
- if parts[1]:
- yield ('symbol', parts[1], s + 1)
- yield ('end', None, len(program))
- return
-
- pos, l = 0, len(program)
- while pos < l:
- c = program[pos]
- if c.isspace(): # skip inter-token whitespace
- pass
- elif c == ':' and program[pos:pos + 2] == '::': # look ahead carefully
- yield ('::', None, pos)
- pos += 1 # skip ahead
- elif c == '.' and program[pos:pos + 2] == '..': # look ahead carefully
- yield ('..', None, pos)
- pos += 1 # skip ahead
- elif c == '#' and program[pos:pos + 2] == '##': # look ahead carefully
- yield ('##', None, pos)
- pos += 1 # skip ahead
- elif c in "():=,-|&+!~^%": # handle simple operators
- yield (c, None, pos)
- elif (c in '"\'' or c == 'r' and
- program[pos:pos + 2] in ("r'", 'r"')): # handle quoted strings
- if c == 'r':
- pos += 1
- c = program[pos]
- decode = lambda x: x
- else:
- decode = parser.unescapestr
- pos += 1
- s = pos
- while pos < l: # find closing quote
- d = program[pos]
- if d == '\\': # skip over escaped characters
- pos += 2
- continue
- if d == c:
- yield ('string', decode(program[s:pos]), s)
- break
- pos += 1
- else:
- raise error.ParseError(_("unterminated string"), s)
- # gather up a symbol/keyword
- elif c in syminitletters:
- s = pos
- pos += 1
- while pos < l: # find end of symbol
- d = program[pos]
- if d not in symletters:
- break
- if d == '.' and program[pos - 1] == '.': # special case for ..
- pos -= 1
- break
- pos += 1
- sym = program[s:pos]
- if sym in keywords: # operator keywords
- yield (sym, None, s)
- elif '-' in sym:
- # some jerk gave us foo-bar-baz, try to check if it's a symbol
- if lookup and lookup(sym):
- # looks like a real symbol
- yield ('symbol', sym, s)
- else:
- # looks like an expression
- parts = sym.split('-')
- for p in parts[:-1]:
- if p: # possible consecutive -
- yield ('symbol', p, s)
- s += len(p)
- yield ('-', None, pos)
- s += 1
- if parts[-1]: # possible trailing -
- yield ('symbol', parts[-1], s)
- else:
- yield ('symbol', sym, s)
- pos -= 1
- else:
- raise error.ParseError(_("syntax error in revset '%s'") %
- program, pos)
- pos += 1
- yield ('end', None, pos)
-
# helpers
-_notset = object()
-
-def getsymbol(x):
- if x and x[0] == 'symbol':
- return x[1]
- raise error.ParseError(_('not a symbol'))
-
-def getstring(x, err):
- if x and (x[0] == 'string' or x[0] == 'symbol'):
- return x[1]
- raise error.ParseError(err)
-
-def getinteger(x, err, default=_notset):
- if not x and default is not _notset:
- return default
- try:
- return int(getstring(x, err))
- except ValueError:
- raise error.ParseError(err)
-
-def getlist(x):
- if not x:
- return []
- if x[0] == 'list':
- return list(x[1:])
- return [x]
-
-def getrange(x, err):
- if not x:
- raise error.ParseError(err)
- op = x[0]
- if op == 'range':
- return x[1], x[2]
- elif op == 'rangepre':
- return None, x[1]
- elif op == 'rangepost':
- return x[1], None
- elif op == 'rangeall':
- return None, None
- raise error.ParseError(err)
-
-def getargs(x, min, max, err):
- l = getlist(x)
- if len(l) < min or (max >= 0 and len(l) > max):
- raise error.ParseError(err)
- return l
-
-def getargsdict(x, funcname, keys):
- return parser.buildargsdict(getlist(x), funcname, parser.splitargspec(keys),
- keyvaluenode='keyvalue', keynode='symbol')
-
def getset(repo, subset, x):
if not x:
raise error.ParseError(_("missing argument"))
@@ -501,7 +314,7 @@
@predicate('_destupdate')
def _destupdate(repo, subset, x):
# experimental revset for update destination
- args = getargsdict(x, 'limit', 'clean check')
+ args = getargsdict(x, 'limit', 'clean')
return subset & baseset([destutil.destupdate(repo, **args)[0]])
@predicate('_destmerge')
@@ -1139,7 +952,8 @@
fromline -= 1
fctx = repo[rev].filectx(fname)
- revs = (c.rev() for c in context.blockancestors(fctx, fromline, toline))
+ revs = (c.rev() for c, _linerange
+ in context.blockancestors(fctx, fromline, toline))
return subset & generatorset(revs, iterasc=False)
@predicate('all()', safe=True)
@@ -1638,19 +1452,10 @@
ps -= set([node.nullrev])
return subset & ps
-def _phase(repo, subset, target):
- """helper to select all rev in phase <target>"""
- repo._phasecache.loadphaserevs(repo) # ensure phase's sets are loaded
- if repo._phasecache._phasesets:
- s = repo._phasecache._phasesets[target] - repo.changelog.filteredrevs
- s = baseset(s)
- s.sort() # set are non ordered, so we enforce ascending
- return subset & s
- else:
- phase = repo._phasecache.phase
- condition = lambda r: phase(repo, r) == target
- return subset.filter(condition, condrepr=('<phase %r>', target),
- cache=False)
+def _phase(repo, subset, *targets):
+ """helper to select all rev in <targets> phases"""
+ s = repo._phasecache.getrevset(repo, targets)
+ return subset & s
@predicate('draft()', safe=True)
def draft(repo, subset, x):
@@ -1711,20 +1516,7 @@
@predicate('_notpublic', safe=True)
def _notpublic(repo, subset, x):
getargs(x, 0, 0, "_notpublic takes no arguments")
- repo._phasecache.loadphaserevs(repo) # ensure phase's sets are loaded
- if repo._phasecache._phasesets:
- s = set()
- for u in repo._phasecache._phasesets[1:]:
- s.update(u)
- s = baseset(s - repo.changelog.filteredrevs)
- s.sort()
- return subset & s
- else:
- phase = repo._phasecache.phase
- target = phases.public
- condition = lambda r: phase(repo, r) != target
- return subset.filter(condition, condrepr=('<phase %r>', target),
- cache=False)
+ return _phase(repo, subset, phases.draft, phases.secret)
@predicate('public()', safe=True)
def public(repo, subset, x):
@@ -2428,350 +2220,6 @@
"parentpost": parentpost,
}
-# Constants for ordering requirement, used in _analyze():
-#
-# If 'define', any nested functions and operations can change the ordering of
-# the entries in the set. If 'follow', any nested functions and operations
-# should take the ordering specified by the first operand to the '&' operator.
-#
-# For instance,
-#
-# X & (Y | Z)
-# ^ ^^^^^^^
-# | follow
-# define
-#
-# will be evaluated as 'or(y(x()), z(x()))', where 'x()' can change the order
-# of the entries in the set, but 'y()', 'z()' and 'or()' shouldn't.
-#
-# 'any' means the order doesn't matter. For instance,
-#
-# X & !Y
-# ^
-# any
-#
-# 'y()' can either enforce its ordering requirement or take the ordering
-# specified by 'x()' because 'not()' doesn't care the order.
-#
-# Transition of ordering requirement:
-#
-# 1. starts with 'define'
-# 2. shifts to 'follow' by 'x & y'
-# 3. changes back to 'define' on function call 'f(x)' or function-like
-# operation 'x (f) y' because 'f' may have its own ordering requirement
-# for 'x' and 'y' (e.g. 'first(x)')
-#
-anyorder = 'any' # don't care the order
-defineorder = 'define' # should define the order
-followorder = 'follow' # must follow the current order
-
-# transition table for 'x & y', from the current expression 'x' to 'y'
-_tofolloworder = {
- anyorder: anyorder,
- defineorder: followorder,
- followorder: followorder,
-}
-
-def _matchonly(revs, bases):
- """
- >>> f = lambda *args: _matchonly(*map(parse, args))
- >>> f('ancestors(A)', 'not ancestors(B)')
- ('list', ('symbol', 'A'), ('symbol', 'B'))
- """
- if (revs is not None
- and revs[0] == 'func'
- and getsymbol(revs[1]) == 'ancestors'
- and bases is not None
- and bases[0] == 'not'
- and bases[1][0] == 'func'
- and getsymbol(bases[1][1]) == 'ancestors'):
- return ('list', revs[2], bases[1][2])
-
-def _fixops(x):
- """Rewrite raw parsed tree to resolve ambiguous syntax which cannot be
- handled well by our simple top-down parser"""
- if not isinstance(x, tuple):
- return x
-
- op = x[0]
- if op == 'parent':
- # x^:y means (x^) : y, not x ^ (:y)
- # x^: means (x^) :, not x ^ (:)
- post = ('parentpost', x[1])
- if x[2][0] == 'dagrangepre':
- return _fixops(('dagrange', post, x[2][1]))
- elif x[2][0] == 'rangepre':
- return _fixops(('range', post, x[2][1]))
- elif x[2][0] == 'rangeall':
- return _fixops(('rangepost', post))
- elif op == 'or':
- # make number of arguments deterministic:
- # x + y + z -> (or x y z) -> (or (list x y z))
- return (op, _fixops(('list',) + x[1:]))
-
- return (op,) + tuple(_fixops(y) for y in x[1:])
-
-def _analyze(x, order):
- if x is None:
- return x
-
- op = x[0]
- if op == 'minus':
- return _analyze(('and', x[1], ('not', x[2])), order)
- elif op == 'only':
- t = ('func', ('symbol', 'only'), ('list', x[1], x[2]))
- return _analyze(t, order)
- elif op == 'onlypost':
- return _analyze(('func', ('symbol', 'only'), x[1]), order)
- elif op == 'dagrangepre':
- return _analyze(('func', ('symbol', 'ancestors'), x[1]), order)
- elif op == 'dagrangepost':
- return _analyze(('func', ('symbol', 'descendants'), x[1]), order)
- elif op == 'negate':
- s = getstring(x[1], _("can't negate that"))
- return _analyze(('string', '-' + s), order)
- elif op in ('string', 'symbol'):
- return x
- elif op == 'and':
- ta = _analyze(x[1], order)
- tb = _analyze(x[2], _tofolloworder[order])
- return (op, ta, tb, order)
- elif op == 'or':
- return (op, _analyze(x[1], order), order)
- elif op == 'not':
- return (op, _analyze(x[1], anyorder), order)
- elif op == 'rangeall':
- return (op, None, order)
- elif op in ('rangepre', 'rangepost', 'parentpost'):
- return (op, _analyze(x[1], defineorder), order)
- elif op == 'group':
- return _analyze(x[1], order)
- elif op in ('dagrange', 'range', 'parent', 'ancestor'):
- ta = _analyze(x[1], defineorder)
- tb = _analyze(x[2], defineorder)
- return (op, ta, tb, order)
- elif op == 'list':
- return (op,) + tuple(_analyze(y, order) for y in x[1:])
- elif op == 'keyvalue':
- return (op, x[1], _analyze(x[2], order))
- elif op == 'func':
- f = getsymbol(x[1])
- d = defineorder
- if f == 'present':
- # 'present(set)' is known to return the argument set with no
- # modification, so forward the current order to its argument
- d = order
- return (op, x[1], _analyze(x[2], d), order)
- raise ValueError('invalid operator %r' % op)
-
-def analyze(x, order=defineorder):
- """Transform raw parsed tree to evaluatable tree which can be fed to
- optimize() or getset()
-
- All pseudo operations should be mapped to real operations or functions
- defined in methods or symbols table respectively.
-
- 'order' specifies how the current expression 'x' is ordered (see the
- constants defined above.)
- """
- return _analyze(x, order)
-
-def _optimize(x, small):
- if x is None:
- return 0, x
-
- smallbonus = 1
- if small:
- smallbonus = .5
-
- op = x[0]
- if op in ('string', 'symbol'):
- return smallbonus, x # single revisions are small
- elif op == 'and':
- wa, ta = _optimize(x[1], True)
- wb, tb = _optimize(x[2], True)
- order = x[3]
- w = min(wa, wb)
-
- # (::x and not ::y)/(not ::y and ::x) have a fast path
- tm = _matchonly(ta, tb) or _matchonly(tb, ta)
- if tm:
- return w, ('func', ('symbol', 'only'), tm, order)
-
- if tb is not None and tb[0] == 'not':
- return wa, ('difference', ta, tb[1], order)
-
- if wa > wb:
- return w, (op, tb, ta, order)
- return w, (op, ta, tb, order)
- elif op == 'or':
- # fast path for machine-generated expression, that is likely to have
- # lots of trivial revisions: 'a + b + c()' to '_list(a b) + c()'
- order = x[2]
- ws, ts, ss = [], [], []
- def flushss():
- if not ss:
- return
- if len(ss) == 1:
- w, t = ss[0]
- else:
- s = '\0'.join(t[1] for w, t in ss)
- y = ('func', ('symbol', '_list'), ('string', s), order)
- w, t = _optimize(y, False)
- ws.append(w)
- ts.append(t)
- del ss[:]
- for y in getlist(x[1]):
- w, t = _optimize(y, False)
- if t is not None and (t[0] == 'string' or t[0] == 'symbol'):
- ss.append((w, t))
- continue
- flushss()
- ws.append(w)
- ts.append(t)
- flushss()
- if len(ts) == 1:
- return ws[0], ts[0] # 'or' operation is fully optimized out
- # we can't reorder trees by weight because it would change the order.
- # ("sort(a + b)" == "sort(b + a)", but "a + b" != "b + a")
- # ts = tuple(t for w, t in sorted(zip(ws, ts), key=lambda wt: wt[0]))
- return max(ws), (op, ('list',) + tuple(ts), order)
- elif op == 'not':
- # Optimize not public() to _notpublic() because we have a fast version
- if x[1][:3] == ('func', ('symbol', 'public'), None):
- order = x[1][3]
- newsym = ('func', ('symbol', '_notpublic'), None, order)
- o = _optimize(newsym, not small)
- return o[0], o[1]
- else:
- o = _optimize(x[1], not small)
- order = x[2]
- return o[0], (op, o[1], order)
- elif op == 'rangeall':
- return smallbonus, x
- elif op in ('rangepre', 'rangepost', 'parentpost'):
- o = _optimize(x[1], small)
- order = x[2]
- return o[0], (op, o[1], order)
- elif op in ('dagrange', 'range', 'parent', 'ancestor'):
- wa, ta = _optimize(x[1], small)
- wb, tb = _optimize(x[2], small)
- order = x[3]
- return wa + wb, (op, ta, tb, order)
- elif op == 'list':
- ws, ts = zip(*(_optimize(y, small) for y in x[1:]))
- return sum(ws), (op,) + ts
- elif op == 'keyvalue':
- w, t = _optimize(x[2], small)
- return w, (op, x[1], t)
- elif op == 'func':
- f = getsymbol(x[1])
- wa, ta = _optimize(x[2], small)
- if f in ('author', 'branch', 'closed', 'date', 'desc', 'file', 'grep',
- 'keyword', 'outgoing', 'user', 'destination'):
- w = 10 # slow
- elif f in ('modifies', 'adds', 'removes'):
- w = 30 # slower
- elif f == "contains":
- w = 100 # very slow
- elif f == "ancestor":
- w = 1 * smallbonus
- elif f in ('reverse', 'limit', 'first', 'wdir', '_intlist'):
- w = 0
- elif f == "sort":
- w = 10 # assume most sorts look at changelog
- else:
- w = 1
- order = x[3]
- return w + wa, (op, x[1], ta, order)
- raise ValueError('invalid operator %r' % op)
-
-def optimize(tree):
- """Optimize evaluatable tree
-
- All pseudo operations should be transformed beforehand.
- """
- _weight, newtree = _optimize(tree, small=True)
- return newtree
-
-# the set of valid characters for the initial letter of symbols in
-# alias declarations and definitions
-_aliassyminitletters = _syminitletters | set(pycompat.sysstr('$'))
-
-def _parsewith(spec, lookup=None, syminitletters=None):
- """Generate a parse tree of given spec with given tokenizing options
-
- >>> _parsewith('foo($1)', syminitletters=_aliassyminitletters)
- ('func', ('symbol', 'foo'), ('symbol', '$1'))
- >>> _parsewith('$1')
- Traceback (most recent call last):
- ...
- ParseError: ("syntax error in revset '$1'", 0)
- >>> _parsewith('foo bar')
- Traceback (most recent call last):
- ...
- ParseError: ('invalid token', 4)
- """
- p = parser.parser(elements)
- tree, pos = p.parse(tokenize(spec, lookup=lookup,
- syminitletters=syminitletters))
- if pos != len(spec):
- raise error.ParseError(_('invalid token'), pos)
- return _fixops(parser.simplifyinfixops(tree, ('list', 'or')))
-
-class _aliasrules(parser.basealiasrules):
- """Parsing and expansion rule set of revset aliases"""
- _section = _('revset alias')
-
- @staticmethod
- def _parse(spec):
- """Parse alias declaration/definition ``spec``
-
- This allows symbol names to use also ``$`` as an initial letter
- (for backward compatibility), and callers of this function should
- examine whether ``$`` is used also for unexpected symbols or not.
- """
- return _parsewith(spec, syminitletters=_aliassyminitletters)
-
- @staticmethod
- def _trygetfunc(tree):
- if tree[0] == 'func' and tree[1][0] == 'symbol':
- return tree[1][1], getlist(tree[2])
-
-def expandaliases(ui, tree):
- aliases = _aliasrules.buildmap(ui.configitems('revsetalias'))
- tree = _aliasrules.expand(aliases, tree)
- # warn about problematic (but not referred) aliases
- for name, alias in sorted(aliases.iteritems()):
- if alias.error and not alias.warned:
- ui.warn(_('warning: %s\n') % (alias.error))
- alias.warned = True
- return tree
-
-def foldconcat(tree):
- """Fold elements to be concatenated by `##`
- """
- if not isinstance(tree, tuple) or tree[0] in ('string', 'symbol'):
- return tree
- if tree[0] == '_concat':
- pending = [tree]
- l = []
- while pending:
- e = pending.pop()
- if e[0] == '_concat':
- pending.extend(reversed(e[1:]))
- elif e[0] in ('string', 'symbol'):
- l.append(e[1])
- else:
- msg = _("\"##\" can't concatenate \"%s\" element") % (e[0])
- raise error.ParseError(msg)
- return ('string', ''.join(l))
- else:
- return tuple(foldconcat(t) for t in tree)
-
-def parse(spec, lookup=None):
- return _parsewith(spec, lookup=lookup)
-
def posttreebuilthook(tree, repo):
# hook for extensions to execute code on the optimized tree
pass
@@ -2801,15 +2249,16 @@
if repo:
lookup = repo.__contains__
if len(specs) == 1:
- tree = parse(specs[0], lookup)
+ tree = revsetlang.parse(specs[0], lookup)
else:
- tree = ('or', ('list',) + tuple(parse(s, lookup) for s in specs))
+ tree = ('or',
+ ('list',) + tuple(revsetlang.parse(s, lookup) for s in specs))
if ui:
- tree = expandaliases(ui, tree)
- tree = foldconcat(tree)
- tree = analyze(tree, order)
- tree = optimize(tree)
+ tree = revsetlang.expandaliases(ui, tree)
+ tree = revsetlang.foldconcat(tree)
+ tree = revsetlang.analyze(tree, order)
+ tree = revsetlang.optimize(tree)
posttreebuilthook(tree, repo)
return makematcher(tree)
@@ -2825,1082 +2274,6 @@
return result
return mfunc
-def formatspec(expr, *args):
- '''
- This is a convenience function for using revsets internally, and
- escapes arguments appropriately. Aliases are intentionally ignored
- so that intended expression behavior isn't accidentally subverted.
-
- Supported arguments:
-
- %r = revset expression, parenthesized
- %d = int(arg), no quoting
- %s = string(arg), escaped and single-quoted
- %b = arg.branch(), escaped and single-quoted
- %n = hex(arg), single-quoted
- %% = a literal '%'
-
- Prefixing the type with 'l' specifies a parenthesized list of that type.
-
- >>> formatspec('%r:: and %lr', '10 or 11', ("this()", "that()"))
- '(10 or 11):: and ((this()) or (that()))'
- >>> formatspec('%d:: and not %d::', 10, 20)
- '10:: and not 20::'
- >>> formatspec('%ld or %ld', [], [1])
- "_list('') or 1"
- >>> formatspec('keyword(%s)', 'foo\\xe9')
- "keyword('foo\\\\xe9')"
- >>> b = lambda: 'default'
- >>> b.branch = b
- >>> formatspec('branch(%b)', b)
- "branch('default')"
- >>> formatspec('root(%ls)', ['a', 'b', 'c', 'd'])
- "root(_list('a\\x00b\\x00c\\x00d'))"
- '''
-
- def quote(s):
- return repr(str(s))
-
- def argtype(c, arg):
- if c == 'd':
- return str(int(arg))
- elif c == 's':
- return quote(arg)
- elif c == 'r':
- parse(arg) # make sure syntax errors are confined
- return '(%s)' % arg
- elif c == 'n':
- return quote(node.hex(arg))
- elif c == 'b':
- return quote(arg.branch())
-
- def listexp(s, t):
- l = len(s)
- if l == 0:
- return "_list('')"
- elif l == 1:
- return argtype(t, s[0])
- elif t == 'd':
- return "_intlist('%s')" % "\0".join(str(int(a)) for a in s)
- elif t == 's':
- return "_list('%s')" % "\0".join(s)
- elif t == 'n':
- return "_hexlist('%s')" % "\0".join(node.hex(a) for a in s)
- elif t == 'b':
- return "_list('%s')" % "\0".join(a.branch() for a in s)
-
- m = l // 2
- return '(%s or %s)' % (listexp(s[:m], t), listexp(s[m:], t))
-
- ret = ''
- pos = 0
- arg = 0
- while pos < len(expr):
- c = expr[pos]
- if c == '%':
- pos += 1
- d = expr[pos]
- if d == '%':
- ret += d
- elif d in 'dsnbr':
- ret += argtype(d, args[arg])
- arg += 1
- elif d == 'l':
- # a list of some type
- pos += 1
- d = expr[pos]
- ret += listexp(list(args[arg]), d)
- arg += 1
- else:
- raise error.Abort(_('unexpected revspec format character %s')
- % d)
- else:
- ret += c
- pos += 1
-
- return ret
-
-def prettyformat(tree):
- return parser.prettyformat(tree, ('string', 'symbol'))
-
-def depth(tree):
- if isinstance(tree, tuple):
- return max(map(depth, tree)) + 1
- else:
- return 0
-
-def funcsused(tree):
- if not isinstance(tree, tuple) or tree[0] in ('string', 'symbol'):
- return set()
- else:
- funcs = set()
- for s in tree[1:]:
- funcs |= funcsused(s)
- if tree[0] == 'func':
- funcs.add(tree[1][1])
- return funcs
-
-def _formatsetrepr(r):
- """Format an optional printable representation of a set
-
- ======== =================================
- type(r) example
- ======== =================================
- tuple ('<not %r>', other)
- str '<branch closed>'
- callable lambda: '<branch %r>' % sorted(b)
- object other
- ======== =================================
- """
- if r is None:
- return ''
- elif isinstance(r, tuple):
- return r[0] % r[1:]
- elif isinstance(r, str):
- return r
- elif callable(r):
- return r()
- else:
- return repr(r)
-
-class abstractsmartset(object):
-
- def __nonzero__(self):
- """True if the smartset is not empty"""
- raise NotImplementedError()
-
- def __contains__(self, rev):
- """provide fast membership testing"""
- raise NotImplementedError()
-
- def __iter__(self):
- """iterate the set in the order it is supposed to be iterated"""
- raise NotImplementedError()
-
- # Attributes containing a function to perform a fast iteration in a given
- # direction. A smartset can have none, one, or both defined.
- #
- # Default value is None instead of a function returning None to avoid
- # initializing an iterator just for testing if a fast method exists.
- fastasc = None
- fastdesc = None
-
- def isascending(self):
- """True if the set will iterate in ascending order"""
- raise NotImplementedError()
-
- def isdescending(self):
- """True if the set will iterate in descending order"""
- raise NotImplementedError()
-
- def istopo(self):
- """True if the set will iterate in topographical order"""
- raise NotImplementedError()
-
- def min(self):
- """return the minimum element in the set"""
- if self.fastasc is None:
- v = min(self)
- else:
- for v in self.fastasc():
- break
- else:
- raise ValueError('arg is an empty sequence')
- self.min = lambda: v
- return v
-
- def max(self):
- """return the maximum element in the set"""
- if self.fastdesc is None:
- return max(self)
- else:
- for v in self.fastdesc():
- break
- else:
- raise ValueError('arg is an empty sequence')
- self.max = lambda: v
- return v
-
- def first(self):
- """return the first element in the set (user iteration perspective)
-
- Return None if the set is empty"""
- raise NotImplementedError()
-
- def last(self):
- """return the last element in the set (user iteration perspective)
-
- Return None if the set is empty"""
- raise NotImplementedError()
-
- def __len__(self):
- """return the length of the smartsets
-
- This can be expensive on smartset that could be lazy otherwise."""
- raise NotImplementedError()
-
- def reverse(self):
- """reverse the expected iteration order"""
- raise NotImplementedError()
-
- def sort(self, reverse=True):
- """get the set to iterate in an ascending or descending order"""
- raise NotImplementedError()
-
- def __and__(self, other):
- """Returns a new object with the intersection of the two collections.
-
- This is part of the mandatory API for smartset."""
- if isinstance(other, fullreposet):
- return self
- return self.filter(other.__contains__, condrepr=other, cache=False)
-
- def __add__(self, other):
- """Returns a new object with the union of the two collections.
-
- This is part of the mandatory API for smartset."""
- return addset(self, other)
-
- def __sub__(self, other):
- """Returns a new object with the substraction of the two collections.
-
- This is part of the mandatory API for smartset."""
- c = other.__contains__
- return self.filter(lambda r: not c(r), condrepr=('<not %r>', other),
- cache=False)
-
- def filter(self, condition, condrepr=None, cache=True):
- """Returns this smartset filtered by condition as a new smartset.
-
- `condition` is a callable which takes a revision number and returns a
- boolean. Optional `condrepr` provides a printable representation of
- the given `condition`.
-
- This is part of the mandatory API for smartset."""
- # builtin cannot be cached. but do not needs to
- if cache and util.safehasattr(condition, 'func_code'):
- condition = util.cachefunc(condition)
- return filteredset(self, condition, condrepr)
-
-class baseset(abstractsmartset):
- """Basic data structure that represents a revset and contains the basic
- operation that it should be able to perform.
-
- Every method in this class should be implemented by any smartset class.
- """
- def __init__(self, data=(), datarepr=None, istopo=False):
- """
- datarepr: a tuple of (format, obj, ...), a function or an object that
- provides a printable representation of the given data.
- """
- self._ascending = None
- self._istopo = istopo
- if not isinstance(data, list):
- if isinstance(data, set):
- self._set = data
- # set has no order we pick one for stability purpose
- self._ascending = True
- data = list(data)
- self._list = data
- self._datarepr = datarepr
-
- @util.propertycache
- def _set(self):
- return set(self._list)
-
- @util.propertycache
- def _asclist(self):
- asclist = self._list[:]
- asclist.sort()
- return asclist
-
- def __iter__(self):
- if self._ascending is None:
- return iter(self._list)
- elif self._ascending:
- return iter(self._asclist)
- else:
- return reversed(self._asclist)
-
- def fastasc(self):
- return iter(self._asclist)
-
- def fastdesc(self):
- return reversed(self._asclist)
-
- @util.propertycache
- def __contains__(self):
- return self._set.__contains__
-
- def __nonzero__(self):
- return bool(self._list)
-
- def sort(self, reverse=False):
- self._ascending = not bool(reverse)
- self._istopo = False
-
- def reverse(self):
- if self._ascending is None:
- self._list.reverse()
- else:
- self._ascending = not self._ascending
- self._istopo = False
-
- def __len__(self):
- return len(self._list)
-
- def isascending(self):
- """Returns True if the collection is ascending order, False if not.
-
- This is part of the mandatory API for smartset."""
- if len(self) <= 1:
- return True
- return self._ascending is not None and self._ascending
-
- def isdescending(self):
- """Returns True if the collection is descending order, False if not.
-
- This is part of the mandatory API for smartset."""
- if len(self) <= 1:
- return True
- return self._ascending is not None and not self._ascending
-
- def istopo(self):
- """Is the collection is in topographical order or not.
-
- This is part of the mandatory API for smartset."""
- if len(self) <= 1:
- return True
- return self._istopo
-
- def first(self):
- if self:
- if self._ascending is None:
- return self._list[0]
- elif self._ascending:
- return self._asclist[0]
- else:
- return self._asclist[-1]
- return None
-
- def last(self):
- if self:
- if self._ascending is None:
- return self._list[-1]
- elif self._ascending:
- return self._asclist[-1]
- else:
- return self._asclist[0]
- return None
-
- def __repr__(self):
- d = {None: '', False: '-', True: '+'}[self._ascending]
- s = _formatsetrepr(self._datarepr)
- if not s:
- l = self._list
- # if _list has been built from a set, it might have a different
- # order from one python implementation to another.
- # We fallback to the sorted version for a stable output.
- if self._ascending is not None:
- l = self._asclist
- s = repr(l)
- return '<%s%s %s>' % (type(self).__name__, d, s)
-
-class filteredset(abstractsmartset):
- """Duck type for baseset class which iterates lazily over the revisions in
- the subset and contains a function which tests for membership in the
- revset
- """
- def __init__(self, subset, condition=lambda x: True, condrepr=None):
- """
- condition: a function that decide whether a revision in the subset
- belongs to the revset or not.
- condrepr: a tuple of (format, obj, ...), a function or an object that
- provides a printable representation of the given condition.
- """
- self._subset = subset
- self._condition = condition
- self._condrepr = condrepr
-
- def __contains__(self, x):
- return x in self._subset and self._condition(x)
-
- def __iter__(self):
- return self._iterfilter(self._subset)
-
- def _iterfilter(self, it):
- cond = self._condition
- for x in it:
- if cond(x):
- yield x
-
- @property
- def fastasc(self):
- it = self._subset.fastasc
- if it is None:
- return None
- return lambda: self._iterfilter(it())
-
- @property
- def fastdesc(self):
- it = self._subset.fastdesc
- if it is None:
- return None
- return lambda: self._iterfilter(it())
-
- def __nonzero__(self):
- fast = None
- candidates = [self.fastasc if self.isascending() else None,
- self.fastdesc if self.isdescending() else None,
- self.fastasc,
- self.fastdesc]
- for candidate in candidates:
- if candidate is not None:
- fast = candidate
- break
-
- if fast is not None:
- it = fast()
- else:
- it = self
-
- for r in it:
- return True
- return False
-
- def __len__(self):
- # Basic implementation to be changed in future patches.
- # until this gets improved, we use generator expression
- # here, since list comprehensions are free to call __len__ again
- # causing infinite recursion
- l = baseset(r for r in self)
- return len(l)
-
- def sort(self, reverse=False):
- self._subset.sort(reverse=reverse)
-
- def reverse(self):
- self._subset.reverse()
-
- def isascending(self):
- return self._subset.isascending()
-
- def isdescending(self):
- return self._subset.isdescending()
-
- def istopo(self):
- return self._subset.istopo()
-
- def first(self):
- for x in self:
- return x
- return None
-
- def last(self):
- it = None
- if self.isascending():
- it = self.fastdesc
- elif self.isdescending():
- it = self.fastasc
- if it is not None:
- for x in it():
- return x
- return None #empty case
- else:
- x = None
- for x in self:
- pass
- return x
-
- def __repr__(self):
- xs = [repr(self._subset)]
- s = _formatsetrepr(self._condrepr)
- if s:
- xs.append(s)
- return '<%s %s>' % (type(self).__name__, ', '.join(xs))
-
-def _iterordered(ascending, iter1, iter2):
- """produce an ordered iteration from two iterators with the same order
-
- The ascending is used to indicated the iteration direction.
- """
- choice = max
- if ascending:
- choice = min
-
- val1 = None
- val2 = None
- try:
- # Consume both iterators in an ordered way until one is empty
- while True:
- if val1 is None:
- val1 = next(iter1)
- if val2 is None:
- val2 = next(iter2)
- n = choice(val1, val2)
- yield n
- if val1 == n:
- val1 = None
- if val2 == n:
- val2 = None
- except StopIteration:
- # Flush any remaining values and consume the other one
- it = iter2
- if val1 is not None:
- yield val1
- it = iter1
- elif val2 is not None:
- # might have been equality and both are empty
- yield val2
- for val in it:
- yield val
-
-class addset(abstractsmartset):
- """Represent the addition of two sets
-
- Wrapper structure for lazily adding two structures without losing much
- performance on the __contains__ method
-
- If the ascending attribute is set, that means the two structures are
- ordered in either an ascending or descending way. Therefore, we can add
- them maintaining the order by iterating over both at the same time
-
- >>> xs = baseset([0, 3, 2])
- >>> ys = baseset([5, 2, 4])
-
- >>> rs = addset(xs, ys)
- >>> bool(rs), 0 in rs, 1 in rs, 5 in rs, rs.first(), rs.last()
- (True, True, False, True, 0, 4)
- >>> rs = addset(xs, baseset([]))
- >>> bool(rs), 0 in rs, 1 in rs, rs.first(), rs.last()
- (True, True, False, 0, 2)
- >>> rs = addset(baseset([]), baseset([]))
- >>> bool(rs), 0 in rs, rs.first(), rs.last()
- (False, False, None, None)
-
- iterate unsorted:
- >>> rs = addset(xs, ys)
- >>> # (use generator because pypy could call len())
- >>> list(x for x in rs) # without _genlist
- [0, 3, 2, 5, 4]
- >>> assert not rs._genlist
- >>> len(rs)
- 5
- >>> [x for x in rs] # with _genlist
- [0, 3, 2, 5, 4]
- >>> assert rs._genlist
-
- iterate ascending:
- >>> rs = addset(xs, ys, ascending=True)
- >>> # (use generator because pypy could call len())
- >>> list(x for x in rs), list(x for x in rs.fastasc()) # without _asclist
- ([0, 2, 3, 4, 5], [0, 2, 3, 4, 5])
- >>> assert not rs._asclist
- >>> len(rs)
- 5
- >>> [x for x in rs], [x for x in rs.fastasc()]
- ([0, 2, 3, 4, 5], [0, 2, 3, 4, 5])
- >>> assert rs._asclist
-
- iterate descending:
- >>> rs = addset(xs, ys, ascending=False)
- >>> # (use generator because pypy could call len())
- >>> list(x for x in rs), list(x for x in rs.fastdesc()) # without _asclist
- ([5, 4, 3, 2, 0], [5, 4, 3, 2, 0])
- >>> assert not rs._asclist
- >>> len(rs)
- 5
- >>> [x for x in rs], [x for x in rs.fastdesc()]
- ([5, 4, 3, 2, 0], [5, 4, 3, 2, 0])
- >>> assert rs._asclist
-
- iterate ascending without fastasc:
- >>> rs = addset(xs, generatorset(ys), ascending=True)
- >>> assert rs.fastasc is None
- >>> [x for x in rs]
- [0, 2, 3, 4, 5]
-
- iterate descending without fastdesc:
- >>> rs = addset(generatorset(xs), ys, ascending=False)
- >>> assert rs.fastdesc is None
- >>> [x for x in rs]
- [5, 4, 3, 2, 0]
- """
- def __init__(self, revs1, revs2, ascending=None):
- self._r1 = revs1
- self._r2 = revs2
- self._iter = None
- self._ascending = ascending
- self._genlist = None
- self._asclist = None
-
- def __len__(self):
- return len(self._list)
-
- def __nonzero__(self):
- return bool(self._r1) or bool(self._r2)
-
- @util.propertycache
- def _list(self):
- if not self._genlist:
- self._genlist = baseset(iter(self))
- return self._genlist
-
- def __iter__(self):
- """Iterate over both collections without repeating elements
-
- If the ascending attribute is not set, iterate over the first one and
- then over the second one checking for membership on the first one so we
- dont yield any duplicates.
-
- If the ascending attribute is set, iterate over both collections at the
- same time, yielding only one value at a time in the given order.
- """
- if self._ascending is None:
- if self._genlist:
- return iter(self._genlist)
- def arbitraryordergen():
- for r in self._r1:
- yield r
- inr1 = self._r1.__contains__
- for r in self._r2:
- if not inr1(r):
- yield r
- return arbitraryordergen()
- # try to use our own fast iterator if it exists
- self._trysetasclist()
- if self._ascending:
- attr = 'fastasc'
- else:
- attr = 'fastdesc'
- it = getattr(self, attr)
- if it is not None:
- return it()
- # maybe half of the component supports fast
- # get iterator for _r1
- iter1 = getattr(self._r1, attr)
- if iter1 is None:
- # let's avoid side effect (not sure it matters)
- iter1 = iter(sorted(self._r1, reverse=not self._ascending))
- else:
- iter1 = iter1()
- # get iterator for _r2
- iter2 = getattr(self._r2, attr)
- if iter2 is None:
- # let's avoid side effect (not sure it matters)
- iter2 = iter(sorted(self._r2, reverse=not self._ascending))
- else:
- iter2 = iter2()
- return _iterordered(self._ascending, iter1, iter2)
-
- def _trysetasclist(self):
- """populate the _asclist attribute if possible and necessary"""
- if self._genlist is not None and self._asclist is None:
- self._asclist = sorted(self._genlist)
-
- @property
- def fastasc(self):
- self._trysetasclist()
- if self._asclist is not None:
- return self._asclist.__iter__
- iter1 = self._r1.fastasc
- iter2 = self._r2.fastasc
- if None in (iter1, iter2):
- return None
- return lambda: _iterordered(True, iter1(), iter2())
-
- @property
- def fastdesc(self):
- self._trysetasclist()
- if self._asclist is not None:
- return self._asclist.__reversed__
- iter1 = self._r1.fastdesc
- iter2 = self._r2.fastdesc
- if None in (iter1, iter2):
- return None
- return lambda: _iterordered(False, iter1(), iter2())
-
- def __contains__(self, x):
- return x in self._r1 or x in self._r2
-
- def sort(self, reverse=False):
- """Sort the added set
-
- For this we use the cached list with all the generated values and if we
- know they are ascending or descending we can sort them in a smart way.
- """
- self._ascending = not reverse
-
- def isascending(self):
- return self._ascending is not None and self._ascending
-
- def isdescending(self):
- return self._ascending is not None and not self._ascending
-
- def istopo(self):
- # not worth the trouble asserting if the two sets combined are still
- # in topographical order. Use the sort() predicate to explicitly sort
- # again instead.
- return False
-
- def reverse(self):
- if self._ascending is None:
- self._list.reverse()
- else:
- self._ascending = not self._ascending
-
- def first(self):
- for x in self:
- return x
- return None
-
- def last(self):
- self.reverse()
- val = self.first()
- self.reverse()
- return val
-
- def __repr__(self):
- d = {None: '', False: '-', True: '+'}[self._ascending]
- return '<%s%s %r, %r>' % (type(self).__name__, d, self._r1, self._r2)
-
-class generatorset(abstractsmartset):
- """Wrap a generator for lazy iteration
-
- Wrapper structure for generators that provides lazy membership and can
- be iterated more than once.
- When asked for membership it generates values until either it finds the
- requested one or has gone through all the elements in the generator
- """
- def __init__(self, gen, iterasc=None):
- """
- gen: a generator producing the values for the generatorset.
- """
- self._gen = gen
- self._asclist = None
- self._cache = {}
- self._genlist = []
- self._finished = False
- self._ascending = True
- if iterasc is not None:
- if iterasc:
- self.fastasc = self._iterator
- self.__contains__ = self._asccontains
- else:
- self.fastdesc = self._iterator
- self.__contains__ = self._desccontains
-
- def __nonzero__(self):
- # Do not use 'for r in self' because it will enforce the iteration
- # order (default ascending), possibly unrolling a whole descending
- # iterator.
- if self._genlist:
- return True
- for r in self._consumegen():
- return True
- return False
-
- def __contains__(self, x):
- if x in self._cache:
- return self._cache[x]
-
- # Use new values only, as existing values would be cached.
- for l in self._consumegen():
- if l == x:
- return True
-
- self._cache[x] = False
- return False
-
- def _asccontains(self, x):
- """version of contains optimised for ascending generator"""
- if x in self._cache:
- return self._cache[x]
-
- # Use new values only, as existing values would be cached.
- for l in self._consumegen():
- if l == x:
- return True
- if l > x:
- break
-
- self._cache[x] = False
- return False
-
- def _desccontains(self, x):
- """version of contains optimised for descending generator"""
- if x in self._cache:
- return self._cache[x]
-
- # Use new values only, as existing values would be cached.
- for l in self._consumegen():
- if l == x:
- return True
- if l < x:
- break
-
- self._cache[x] = False
- return False
-
- def __iter__(self):
- if self._ascending:
- it = self.fastasc
- else:
- it = self.fastdesc
- if it is not None:
- return it()
- # we need to consume the iterator
- for x in self._consumegen():
- pass
- # recall the same code
- return iter(self)
-
- def _iterator(self):
- if self._finished:
- return iter(self._genlist)
-
- # We have to use this complex iteration strategy to allow multiple
- # iterations at the same time. We need to be able to catch revision
- # removed from _consumegen and added to genlist in another instance.
- #
- # Getting rid of it would provide an about 15% speed up on this
- # iteration.
- genlist = self._genlist
- nextrev = self._consumegen().next
- _len = len # cache global lookup
- def gen():
- i = 0
- while True:
- if i < _len(genlist):
- yield genlist[i]
- else:
- yield nextrev()
- i += 1
- return gen()
-
- def _consumegen(self):
- cache = self._cache
- genlist = self._genlist.append
- for item in self._gen:
- cache[item] = True
- genlist(item)
- yield item
- if not self._finished:
- self._finished = True
- asc = self._genlist[:]
- asc.sort()
- self._asclist = asc
- self.fastasc = asc.__iter__
- self.fastdesc = asc.__reversed__
-
- def __len__(self):
- for x in self._consumegen():
- pass
- return len(self._genlist)
-
- def sort(self, reverse=False):
- self._ascending = not reverse
-
- def reverse(self):
- self._ascending = not self._ascending
-
- def isascending(self):
- return self._ascending
-
- def isdescending(self):
- return not self._ascending
-
- def istopo(self):
- # not worth the trouble asserting if the two sets combined are still
- # in topographical order. Use the sort() predicate to explicitly sort
- # again instead.
- return False
-
- def first(self):
- if self._ascending:
- it = self.fastasc
- else:
- it = self.fastdesc
- if it is None:
- # we need to consume all and try again
- for x in self._consumegen():
- pass
- return self.first()
- return next(it(), None)
-
- def last(self):
- if self._ascending:
- it = self.fastdesc
- else:
- it = self.fastasc
- if it is None:
- # we need to consume all and try again
- for x in self._consumegen():
- pass
- return self.first()
- return next(it(), None)
-
- def __repr__(self):
- d = {False: '-', True: '+'}[self._ascending]
- return '<%s%s>' % (type(self).__name__, d)
-
-class spanset(abstractsmartset):
- """Duck type for baseset class which represents a range of revisions and
- can work lazily and without having all the range in memory
-
- Note that spanset(x, y) behave almost like xrange(x, y) except for two
- notable points:
- - when x < y it will be automatically descending,
- - revision filtered with this repoview will be skipped.
-
- """
- def __init__(self, repo, start=0, end=None):
- """
- start: first revision included the set
- (default to 0)
- end: first revision excluded (last+1)
- (default to len(repo)
-
- Spanset will be descending if `end` < `start`.
- """
- if end is None:
- end = len(repo)
- self._ascending = start <= end
- if not self._ascending:
- start, end = end + 1, start +1
- self._start = start
- self._end = end
- self._hiddenrevs = repo.changelog.filteredrevs
-
- def sort(self, reverse=False):
- self._ascending = not reverse
-
- def reverse(self):
- self._ascending = not self._ascending
-
- def istopo(self):
- # not worth the trouble asserting if the two sets combined are still
- # in topographical order. Use the sort() predicate to explicitly sort
- # again instead.
- return False
-
- def _iterfilter(self, iterrange):
- s = self._hiddenrevs
- for r in iterrange:
- if r not in s:
- yield r
-
- def __iter__(self):
- if self._ascending:
- return self.fastasc()
- else:
- return self.fastdesc()
-
- def fastasc(self):
- iterrange = xrange(self._start, self._end)
- if self._hiddenrevs:
- return self._iterfilter(iterrange)
- return iter(iterrange)
-
- def fastdesc(self):
- iterrange = xrange(self._end - 1, self._start - 1, -1)
- if self._hiddenrevs:
- return self._iterfilter(iterrange)
- return iter(iterrange)
-
- def __contains__(self, rev):
- hidden = self._hiddenrevs
- return ((self._start <= rev < self._end)
- and not (hidden and rev in hidden))
-
- def __nonzero__(self):
- for r in self:
- return True
- return False
-
- def __len__(self):
- if not self._hiddenrevs:
- return abs(self._end - self._start)
- else:
- count = 0
- start = self._start
- end = self._end
- for rev in self._hiddenrevs:
- if (end < rev <= start) or (start <= rev < end):
- count += 1
- return abs(self._end - self._start) - count
-
- def isascending(self):
- return self._ascending
-
- def isdescending(self):
- return not self._ascending
-
- def first(self):
- if self._ascending:
- it = self.fastasc
- else:
- it = self.fastdesc
- for x in it():
- return x
- return None
-
- def last(self):
- if self._ascending:
- it = self.fastdesc
- else:
- it = self.fastasc
- for x in it():
- return x
- return None
-
- def __repr__(self):
- d = {False: '-', True: '+'}[self._ascending]
- return '<%s%s %d:%d>' % (type(self).__name__, d,
- self._start, self._end - 1)
-
-class fullreposet(spanset):
- """a set containing all revisions in the repo
-
- This class exists to host special optimization and magic to handle virtual
- revisions such as "null".
- """
-
- def __init__(self, repo):
- super(fullreposet, self).__init__(repo)
-
- def __and__(self, other):
- """As self contains the whole repo, all of the other set should also be
- in self. Therefore `self & other = other`.
-
- This boldly assumes the other contains valid revs only.
- """
- # other not a smartset, make is so
- if not util.safehasattr(other, 'isascending'):
- # filter out hidden revision
- # (this boldly assumes all smartset are pure)
- #
- # `other` was used with "&", let's assume this is a set like
- # object.
- other = baseset(other - self._hiddenrevs)
-
- other.sort(reverse=self.isdescending())
- return other
-
-def prettyformatset(revs):
- lines = []
- rs = repr(revs)
- p = 0
- while p < len(rs):
- q = rs.find('<', p + 1)
- if q < 0:
- q = len(rs)
- l = rs.count('<', 0, p) - rs.count('>', 0, p)
- assert l >= 0
- lines.append((l, rs[p:q].rstrip()))
- p = q
- return '\n'.join(' ' * l + s for l, s in lines)
-
def loadpredicate(ui, extname, registrarobj):
"""Load revset predicates from specified registrarobj
"""
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/mercurial/revsetlang.py Tue Feb 28 11:13:25 2017 -0800
@@ -0,0 +1,684 @@
+# revsetlang.py - parser, tokenizer and utility for revision set language
+#
+# Copyright 2010 Matt Mackall <mpm@selenic.com>
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+from __future__ import absolute_import
+
+import string
+
+from .i18n import _
+from . import (
+ error,
+ node,
+ parser,
+ pycompat,
+)
+
+elements = {
+ # token-type: binding-strength, primary, prefix, infix, suffix
+ "(": (21, None, ("group", 1, ")"), ("func", 1, ")"), None),
+ "##": (20, None, None, ("_concat", 20), None),
+ "~": (18, None, None, ("ancestor", 18), None),
+ "^": (18, None, None, ("parent", 18), "parentpost"),
+ "-": (5, None, ("negate", 19), ("minus", 5), None),
+ "::": (17, None, ("dagrangepre", 17), ("dagrange", 17), "dagrangepost"),
+ "..": (17, None, ("dagrangepre", 17), ("dagrange", 17), "dagrangepost"),
+ ":": (15, "rangeall", ("rangepre", 15), ("range", 15), "rangepost"),
+ "not": (10, None, ("not", 10), None, None),
+ "!": (10, None, ("not", 10), None, None),
+ "and": (5, None, None, ("and", 5), None),
+ "&": (5, None, None, ("and", 5), None),
+ "%": (5, None, None, ("only", 5), "onlypost"),
+ "or": (4, None, None, ("or", 4), None),
+ "|": (4, None, None, ("or", 4), None),
+ "+": (4, None, None, ("or", 4), None),
+ "=": (3, None, None, ("keyvalue", 3), None),
+ ",": (2, None, None, ("list", 2), None),
+ ")": (0, None, None, None, None),
+ "symbol": (0, "symbol", None, None, None),
+ "string": (0, "string", None, None, None),
+ "end": (0, None, None, None, None),
+}
+
+keywords = set(['and', 'or', 'not'])
+
+# default set of valid characters for the initial letter of symbols
+_syminitletters = set(
+ string.ascii_letters +
+ string.digits + pycompat.sysstr('._@')) | set(map(chr, xrange(128, 256)))
+
+# default set of valid characters for non-initial letters of symbols
+_symletters = _syminitletters | set(pycompat.sysstr('-/'))
+
+def tokenize(program, lookup=None, syminitletters=None, symletters=None):
+ '''
+ Parse a revset statement into a stream of tokens
+
+ ``syminitletters`` is the set of valid characters for the initial
+ letter of symbols.
+
+ By default, character ``c`` is recognized as valid for initial
+ letter of symbols, if ``c.isalnum() or c in '._@' or ord(c) > 127``.
+
+ ``symletters`` is the set of valid characters for non-initial
+ letters of symbols.
+
+ By default, character ``c`` is recognized as valid for non-initial
+ letters of symbols, if ``c.isalnum() or c in '-._/@' or ord(c) > 127``.
+
+ Check that @ is a valid unquoted token character (issue3686):
+ >>> list(tokenize("@::"))
+ [('symbol', '@', 0), ('::', None, 1), ('end', None, 3)]
+
+ '''
+ if syminitletters is None:
+ syminitletters = _syminitletters
+ if symletters is None:
+ symletters = _symletters
+
+ if program and lookup:
+ # attempt to parse old-style ranges first to deal with
+ # things like old-tag which contain query metacharacters
+ parts = program.split(':', 1)
+ if all(lookup(sym) for sym in parts if sym):
+ if parts[0]:
+ yield ('symbol', parts[0], 0)
+ if len(parts) > 1:
+ s = len(parts[0])
+ yield (':', None, s)
+ if parts[1]:
+ yield ('symbol', parts[1], s + 1)
+ yield ('end', None, len(program))
+ return
+
+ pos, l = 0, len(program)
+ while pos < l:
+ c = program[pos]
+ if c.isspace(): # skip inter-token whitespace
+ pass
+ elif c == ':' and program[pos:pos + 2] == '::': # look ahead carefully
+ yield ('::', None, pos)
+ pos += 1 # skip ahead
+ elif c == '.' and program[pos:pos + 2] == '..': # look ahead carefully
+ yield ('..', None, pos)
+ pos += 1 # skip ahead
+ elif c == '#' and program[pos:pos + 2] == '##': # look ahead carefully
+ yield ('##', None, pos)
+ pos += 1 # skip ahead
+ elif c in "():=,-|&+!~^%": # handle simple operators
+ yield (c, None, pos)
+ elif (c in '"\'' or c == 'r' and
+ program[pos:pos + 2] in ("r'", 'r"')): # handle quoted strings
+ if c == 'r':
+ pos += 1
+ c = program[pos]
+ decode = lambda x: x
+ else:
+ decode = parser.unescapestr
+ pos += 1
+ s = pos
+ while pos < l: # find closing quote
+ d = program[pos]
+ if d == '\\': # skip over escaped characters
+ pos += 2
+ continue
+ if d == c:
+ yield ('string', decode(program[s:pos]), s)
+ break
+ pos += 1
+ else:
+ raise error.ParseError(_("unterminated string"), s)
+ # gather up a symbol/keyword
+ elif c in syminitletters:
+ s = pos
+ pos += 1
+ while pos < l: # find end of symbol
+ d = program[pos]
+ if d not in symletters:
+ break
+ if d == '.' and program[pos - 1] == '.': # special case for ..
+ pos -= 1
+ break
+ pos += 1
+ sym = program[s:pos]
+ if sym in keywords: # operator keywords
+ yield (sym, None, s)
+ elif '-' in sym:
+ # some jerk gave us foo-bar-baz, try to check if it's a symbol
+ if lookup and lookup(sym):
+ # looks like a real symbol
+ yield ('symbol', sym, s)
+ else:
+ # looks like an expression
+ parts = sym.split('-')
+ for p in parts[:-1]:
+ if p: # possible consecutive -
+ yield ('symbol', p, s)
+ s += len(p)
+ yield ('-', None, pos)
+ s += 1
+ if parts[-1]: # possible trailing -
+ yield ('symbol', parts[-1], s)
+ else:
+ yield ('symbol', sym, s)
+ pos -= 1
+ else:
+ raise error.ParseError(_("syntax error in revset '%s'") %
+ program, pos)
+ pos += 1
+ yield ('end', None, pos)
+
+# helpers
+
+_notset = object()
+
+def getsymbol(x):
+ if x and x[0] == 'symbol':
+ return x[1]
+ raise error.ParseError(_('not a symbol'))
+
+def getstring(x, err):
+ if x and (x[0] == 'string' or x[0] == 'symbol'):
+ return x[1]
+ raise error.ParseError(err)
+
+def getinteger(x, err, default=_notset):
+ if not x and default is not _notset:
+ return default
+ try:
+ return int(getstring(x, err))
+ except ValueError:
+ raise error.ParseError(err)
+
+def getlist(x):
+ if not x:
+ return []
+ if x[0] == 'list':
+ return list(x[1:])
+ return [x]
+
+def getrange(x, err):
+ if not x:
+ raise error.ParseError(err)
+ op = x[0]
+ if op == 'range':
+ return x[1], x[2]
+ elif op == 'rangepre':
+ return None, x[1]
+ elif op == 'rangepost':
+ return x[1], None
+ elif op == 'rangeall':
+ return None, None
+ raise error.ParseError(err)
+
+def getargs(x, min, max, err):
+ l = getlist(x)
+ if len(l) < min or (max >= 0 and len(l) > max):
+ raise error.ParseError(err)
+ return l
+
+def getargsdict(x, funcname, keys):
+ return parser.buildargsdict(getlist(x), funcname, parser.splitargspec(keys),
+ keyvaluenode='keyvalue', keynode='symbol')
+
+# Constants for ordering requirement, used in _analyze():
+#
+# If 'define', any nested functions and operations can change the ordering of
+# the entries in the set. If 'follow', any nested functions and operations
+# should take the ordering specified by the first operand to the '&' operator.
+#
+# For instance,
+#
+# X & (Y | Z)
+# ^ ^^^^^^^
+# | follow
+# define
+#
+# will be evaluated as 'or(y(x()), z(x()))', where 'x()' can change the order
+# of the entries in the set, but 'y()', 'z()' and 'or()' shouldn't.
+#
+# 'any' means the order doesn't matter. For instance,
+#
+# X & !Y
+# ^
+# any
+#
+# 'y()' can either enforce its ordering requirement or take the ordering
+# specified by 'x()' because 'not()' doesn't care the order.
+#
+# Transition of ordering requirement:
+#
+# 1. starts with 'define'
+# 2. shifts to 'follow' by 'x & y'
+# 3. changes back to 'define' on function call 'f(x)' or function-like
+# operation 'x (f) y' because 'f' may have its own ordering requirement
+# for 'x' and 'y' (e.g. 'first(x)')
+#
+anyorder = 'any' # don't care the order
+defineorder = 'define' # should define the order
+followorder = 'follow' # must follow the current order
+
+# transition table for 'x & y', from the current expression 'x' to 'y'
+_tofolloworder = {
+ anyorder: anyorder,
+ defineorder: followorder,
+ followorder: followorder,
+}
+
+def _matchonly(revs, bases):
+ """
+ >>> f = lambda *args: _matchonly(*map(parse, args))
+ >>> f('ancestors(A)', 'not ancestors(B)')
+ ('list', ('symbol', 'A'), ('symbol', 'B'))
+ """
+ if (revs is not None
+ and revs[0] == 'func'
+ and getsymbol(revs[1]) == 'ancestors'
+ and bases is not None
+ and bases[0] == 'not'
+ and bases[1][0] == 'func'
+ and getsymbol(bases[1][1]) == 'ancestors'):
+ return ('list', revs[2], bases[1][2])
+
+def _fixops(x):
+ """Rewrite raw parsed tree to resolve ambiguous syntax which cannot be
+ handled well by our simple top-down parser"""
+ if not isinstance(x, tuple):
+ return x
+
+ op = x[0]
+ if op == 'parent':
+ # x^:y means (x^) : y, not x ^ (:y)
+ # x^: means (x^) :, not x ^ (:)
+ post = ('parentpost', x[1])
+ if x[2][0] == 'dagrangepre':
+ return _fixops(('dagrange', post, x[2][1]))
+ elif x[2][0] == 'rangepre':
+ return _fixops(('range', post, x[2][1]))
+ elif x[2][0] == 'rangeall':
+ return _fixops(('rangepost', post))
+ elif op == 'or':
+ # make number of arguments deterministic:
+ # x + y + z -> (or x y z) -> (or (list x y z))
+ return (op, _fixops(('list',) + x[1:]))
+
+ return (op,) + tuple(_fixops(y) for y in x[1:])
+
+def _analyze(x, order):
+ if x is None:
+ return x
+
+ op = x[0]
+ if op == 'minus':
+ return _analyze(('and', x[1], ('not', x[2])), order)
+ elif op == 'only':
+ t = ('func', ('symbol', 'only'), ('list', x[1], x[2]))
+ return _analyze(t, order)
+ elif op == 'onlypost':
+ return _analyze(('func', ('symbol', 'only'), x[1]), order)
+ elif op == 'dagrangepre':
+ return _analyze(('func', ('symbol', 'ancestors'), x[1]), order)
+ elif op == 'dagrangepost':
+ return _analyze(('func', ('symbol', 'descendants'), x[1]), order)
+ elif op == 'negate':
+ s = getstring(x[1], _("can't negate that"))
+ return _analyze(('string', '-' + s), order)
+ elif op in ('string', 'symbol'):
+ return x
+ elif op == 'and':
+ ta = _analyze(x[1], order)
+ tb = _analyze(x[2], _tofolloworder[order])
+ return (op, ta, tb, order)
+ elif op == 'or':
+ return (op, _analyze(x[1], order), order)
+ elif op == 'not':
+ return (op, _analyze(x[1], anyorder), order)
+ elif op == 'rangeall':
+ return (op, None, order)
+ elif op in ('rangepre', 'rangepost', 'parentpost'):
+ return (op, _analyze(x[1], defineorder), order)
+ elif op == 'group':
+ return _analyze(x[1], order)
+ elif op in ('dagrange', 'range', 'parent', 'ancestor'):
+ ta = _analyze(x[1], defineorder)
+ tb = _analyze(x[2], defineorder)
+ return (op, ta, tb, order)
+ elif op == 'list':
+ return (op,) + tuple(_analyze(y, order) for y in x[1:])
+ elif op == 'keyvalue':
+ return (op, x[1], _analyze(x[2], order))
+ elif op == 'func':
+ f = getsymbol(x[1])
+ d = defineorder
+ if f == 'present':
+ # 'present(set)' is known to return the argument set with no
+ # modification, so forward the current order to its argument
+ d = order
+ return (op, x[1], _analyze(x[2], d), order)
+ raise ValueError('invalid operator %r' % op)
+
+def analyze(x, order=defineorder):
+ """Transform raw parsed tree to evaluatable tree which can be fed to
+ optimize() or getset()
+
+ All pseudo operations should be mapped to real operations or functions
+ defined in methods or symbols table respectively.
+
+ 'order' specifies how the current expression 'x' is ordered (see the
+ constants defined above.)
+ """
+ return _analyze(x, order)
+
+def _optimize(x, small):
+ if x is None:
+ return 0, x
+
+ smallbonus = 1
+ if small:
+ smallbonus = .5
+
+ op = x[0]
+ if op in ('string', 'symbol'):
+ return smallbonus, x # single revisions are small
+ elif op == 'and':
+ wa, ta = _optimize(x[1], True)
+ wb, tb = _optimize(x[2], True)
+ order = x[3]
+ w = min(wa, wb)
+
+ # (::x and not ::y)/(not ::y and ::x) have a fast path
+ tm = _matchonly(ta, tb) or _matchonly(tb, ta)
+ if tm:
+ return w, ('func', ('symbol', 'only'), tm, order)
+
+ if tb is not None and tb[0] == 'not':
+ return wa, ('difference', ta, tb[1], order)
+
+ if wa > wb:
+ return w, (op, tb, ta, order)
+ return w, (op, ta, tb, order)
+ elif op == 'or':
+ # fast path for machine-generated expression, that is likely to have
+ # lots of trivial revisions: 'a + b + c()' to '_list(a b) + c()'
+ order = x[2]
+ ws, ts, ss = [], [], []
+ def flushss():
+ if not ss:
+ return
+ if len(ss) == 1:
+ w, t = ss[0]
+ else:
+ s = '\0'.join(t[1] for w, t in ss)
+ y = ('func', ('symbol', '_list'), ('string', s), order)
+ w, t = _optimize(y, False)
+ ws.append(w)
+ ts.append(t)
+ del ss[:]
+ for y in getlist(x[1]):
+ w, t = _optimize(y, False)
+ if t is not None and (t[0] == 'string' or t[0] == 'symbol'):
+ ss.append((w, t))
+ continue
+ flushss()
+ ws.append(w)
+ ts.append(t)
+ flushss()
+ if len(ts) == 1:
+ return ws[0], ts[0] # 'or' operation is fully optimized out
+ # we can't reorder trees by weight because it would change the order.
+ # ("sort(a + b)" == "sort(b + a)", but "a + b" != "b + a")
+ # ts = tuple(t for w, t in sorted(zip(ws, ts), key=lambda wt: wt[0]))
+ return max(ws), (op, ('list',) + tuple(ts), order)
+ elif op == 'not':
+ # Optimize not public() to _notpublic() because we have a fast version
+ if x[1][:3] == ('func', ('symbol', 'public'), None):
+ order = x[1][3]
+ newsym = ('func', ('symbol', '_notpublic'), None, order)
+ o = _optimize(newsym, not small)
+ return o[0], o[1]
+ else:
+ o = _optimize(x[1], not small)
+ order = x[2]
+ return o[0], (op, o[1], order)
+ elif op == 'rangeall':
+ return smallbonus, x
+ elif op in ('rangepre', 'rangepost', 'parentpost'):
+ o = _optimize(x[1], small)
+ order = x[2]
+ return o[0], (op, o[1], order)
+ elif op in ('dagrange', 'range', 'parent', 'ancestor'):
+ wa, ta = _optimize(x[1], small)
+ wb, tb = _optimize(x[2], small)
+ order = x[3]
+ return wa + wb, (op, ta, tb, order)
+ elif op == 'list':
+ ws, ts = zip(*(_optimize(y, small) for y in x[1:]))
+ return sum(ws), (op,) + ts
+ elif op == 'keyvalue':
+ w, t = _optimize(x[2], small)
+ return w, (op, x[1], t)
+ elif op == 'func':
+ f = getsymbol(x[1])
+ wa, ta = _optimize(x[2], small)
+ if f in ('author', 'branch', 'closed', 'date', 'desc', 'file', 'grep',
+ 'keyword', 'outgoing', 'user', 'destination'):
+ w = 10 # slow
+ elif f in ('modifies', 'adds', 'removes'):
+ w = 30 # slower
+ elif f == "contains":
+ w = 100 # very slow
+ elif f == "ancestor":
+ w = 1 * smallbonus
+ elif f in ('reverse', 'limit', 'first', 'wdir', '_intlist'):
+ w = 0
+ elif f == "sort":
+ w = 10 # assume most sorts look at changelog
+ else:
+ w = 1
+ order = x[3]
+ return w + wa, (op, x[1], ta, order)
+ raise ValueError('invalid operator %r' % op)
+
+def optimize(tree):
+ """Optimize evaluatable tree
+
+ All pseudo operations should be transformed beforehand.
+ """
+ _weight, newtree = _optimize(tree, small=True)
+ return newtree
+
+# the set of valid characters for the initial letter of symbols in
+# alias declarations and definitions
+_aliassyminitletters = _syminitletters | set(pycompat.sysstr('$'))
+
+def _parsewith(spec, lookup=None, syminitletters=None):
+ """Generate a parse tree of given spec with given tokenizing options
+
+ >>> _parsewith('foo($1)', syminitletters=_aliassyminitletters)
+ ('func', ('symbol', 'foo'), ('symbol', '$1'))
+ >>> _parsewith('$1')
+ Traceback (most recent call last):
+ ...
+ ParseError: ("syntax error in revset '$1'", 0)
+ >>> _parsewith('foo bar')
+ Traceback (most recent call last):
+ ...
+ ParseError: ('invalid token', 4)
+ """
+ p = parser.parser(elements)
+ tree, pos = p.parse(tokenize(spec, lookup=lookup,
+ syminitletters=syminitletters))
+ if pos != len(spec):
+ raise error.ParseError(_('invalid token'), pos)
+ return _fixops(parser.simplifyinfixops(tree, ('list', 'or')))
+
+class _aliasrules(parser.basealiasrules):
+ """Parsing and expansion rule set of revset aliases"""
+ _section = _('revset alias')
+
+ @staticmethod
+ def _parse(spec):
+ """Parse alias declaration/definition ``spec``
+
+ This allows symbol names to use also ``$`` as an initial letter
+ (for backward compatibility), and callers of this function should
+ examine whether ``$`` is used also for unexpected symbols or not.
+ """
+ return _parsewith(spec, syminitletters=_aliassyminitletters)
+
+ @staticmethod
+ def _trygetfunc(tree):
+ if tree[0] == 'func' and tree[1][0] == 'symbol':
+ return tree[1][1], getlist(tree[2])
+
+def expandaliases(ui, tree):
+ aliases = _aliasrules.buildmap(ui.configitems('revsetalias'))
+ tree = _aliasrules.expand(aliases, tree)
+ # warn about problematic (but not referred) aliases
+ for name, alias in sorted(aliases.iteritems()):
+ if alias.error and not alias.warned:
+ ui.warn(_('warning: %s\n') % (alias.error))
+ alias.warned = True
+ return tree
+
+def foldconcat(tree):
+ """Fold elements to be concatenated by `##`
+ """
+ if not isinstance(tree, tuple) or tree[0] in ('string', 'symbol'):
+ return tree
+ if tree[0] == '_concat':
+ pending = [tree]
+ l = []
+ while pending:
+ e = pending.pop()
+ if e[0] == '_concat':
+ pending.extend(reversed(e[1:]))
+ elif e[0] in ('string', 'symbol'):
+ l.append(e[1])
+ else:
+ msg = _("\"##\" can't concatenate \"%s\" element") % (e[0])
+ raise error.ParseError(msg)
+ return ('string', ''.join(l))
+ else:
+ return tuple(foldconcat(t) for t in tree)
+
+def parse(spec, lookup=None):
+ return _parsewith(spec, lookup=lookup)
+
+def formatspec(expr, *args):
+ '''
+ This is a convenience function for using revsets internally, and
+ escapes arguments appropriately. Aliases are intentionally ignored
+ so that intended expression behavior isn't accidentally subverted.
+
+ Supported arguments:
+
+ %r = revset expression, parenthesized
+ %d = int(arg), no quoting
+ %s = string(arg), escaped and single-quoted
+ %b = arg.branch(), escaped and single-quoted
+ %n = hex(arg), single-quoted
+ %% = a literal '%'
+
+ Prefixing the type with 'l' specifies a parenthesized list of that type.
+
+ >>> formatspec('%r:: and %lr', '10 or 11', ("this()", "that()"))
+ '(10 or 11):: and ((this()) or (that()))'
+ >>> formatspec('%d:: and not %d::', 10, 20)
+ '10:: and not 20::'
+ >>> formatspec('%ld or %ld', [], [1])
+ "_list('') or 1"
+ >>> formatspec('keyword(%s)', 'foo\\xe9')
+ "keyword('foo\\\\xe9')"
+ >>> b = lambda: 'default'
+ >>> b.branch = b
+ >>> formatspec('branch(%b)', b)
+ "branch('default')"
+ >>> formatspec('root(%ls)', ['a', 'b', 'c', 'd'])
+ "root(_list('a\\x00b\\x00c\\x00d'))"
+ '''
+
+ def quote(s):
+ return repr(str(s))
+
+ def argtype(c, arg):
+ if c == 'd':
+ return str(int(arg))
+ elif c == 's':
+ return quote(arg)
+ elif c == 'r':
+ parse(arg) # make sure syntax errors are confined
+ return '(%s)' % arg
+ elif c == 'n':
+ return quote(node.hex(arg))
+ elif c == 'b':
+ return quote(arg.branch())
+
+ def listexp(s, t):
+ l = len(s)
+ if l == 0:
+ return "_list('')"
+ elif l == 1:
+ return argtype(t, s[0])
+ elif t == 'd':
+ return "_intlist('%s')" % "\0".join(str(int(a)) for a in s)
+ elif t == 's':
+ return "_list('%s')" % "\0".join(s)
+ elif t == 'n':
+ return "_hexlist('%s')" % "\0".join(node.hex(a) for a in s)
+ elif t == 'b':
+ return "_list('%s')" % "\0".join(a.branch() for a in s)
+
+ m = l // 2
+ return '(%s or %s)' % (listexp(s[:m], t), listexp(s[m:], t))
+
+ ret = ''
+ pos = 0
+ arg = 0
+ while pos < len(expr):
+ c = expr[pos]
+ if c == '%':
+ pos += 1
+ d = expr[pos]
+ if d == '%':
+ ret += d
+ elif d in 'dsnbr':
+ ret += argtype(d, args[arg])
+ arg += 1
+ elif d == 'l':
+ # a list of some type
+ pos += 1
+ d = expr[pos]
+ ret += listexp(list(args[arg]), d)
+ arg += 1
+ else:
+ raise error.Abort(_('unexpected revspec format character %s')
+ % d)
+ else:
+ ret += c
+ pos += 1
+
+ return ret
+
+def prettyformat(tree):
+ return parser.prettyformat(tree, ('string', 'symbol'))
+
+def depth(tree):
+ if isinstance(tree, tuple):
+ return max(map(depth, tree)) + 1
+ else:
+ return 0
+
+def funcsused(tree):
+ if not isinstance(tree, tuple) or tree[0] in ('string', 'symbol'):
+ return set()
+ else:
+ funcs = set()
+ for s in tree[1:]:
+ funcs |= funcsused(s)
+ if tree[0] == 'func':
+ funcs.add(tree[1][1])
+ return funcs
--- a/mercurial/scmposix.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/scmposix.py Tue Feb 28 11:13:25 2017 -0800
@@ -40,8 +40,15 @@
def userrcpath():
if pycompat.sysplatform == 'plan9':
return [encoding.environ['home'] + '/lib/hgrc']
+ elif pycompat.sysplatform == 'darwin':
+ return [os.path.expanduser('~/.hgrc')]
else:
- return [os.path.expanduser('~/.hgrc')]
+ confighome = encoding.environ.get('XDG_CONFIG_HOME')
+ if confighome is None or not os.path.isabs(confighome):
+ confighome = os.path.expanduser('~/.config')
+
+ return [os.path.expanduser('~/.hgrc'),
+ os.path.join(confighome, 'hg', 'hgrc')]
def termsize(ui):
try:
--- a/mercurial/scmutil.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/scmutil.py Tue Feb 28 11:13:25 2017 -0800
@@ -29,7 +29,7 @@
pathutil,
phases,
pycompat,
- revset,
+ revsetlang,
similar,
util,
)
@@ -890,7 +890,7 @@
return repo[l.last()]
def _pairspec(revspec):
- tree = revset.parse(revspec)
+ tree = revsetlang.parse(revspec)
return tree and tree[0] in ('range', 'rangepre', 'rangepost', 'rangeall')
def revpair(repo, revs):
@@ -936,7 +936,7 @@
revision numbers.
It is assumed the revsets are already formatted. If you have arguments
- that need to be expanded in the revset, call ``revset.formatspec()``
+ that need to be expanded in the revset, call ``revsetlang.formatspec()``
and pass the result as an element of ``specs``.
Specifying a single revset is allowed.
@@ -947,10 +947,9 @@
allspecs = []
for spec in specs:
if isinstance(spec, int):
- spec = revset.formatspec('rev(%d)', spec)
+ spec = revsetlang.formatspec('rev(%d)', spec)
allspecs.append(spec)
- m = revset.matchany(repo.ui, allspecs, repo)
- return m(repo)
+ return repo.anyrevs(allspecs, user=True)
def meaningfulparents(repo, ctx):
"""Return list of meaningful (or all if debug) parentrevs for rev.
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/mercurial/smartset.py Tue Feb 28 11:13:25 2017 -0800
@@ -0,0 +1,1056 @@
+# smartset.py - data structure for revision set
+#
+# Copyright 2010 Matt Mackall <mpm@selenic.com>
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+from __future__ import absolute_import
+
+from . import (
+ util,
+)
+
+def _formatsetrepr(r):
+ """Format an optional printable representation of a set
+
+ ======== =================================
+ type(r) example
+ ======== =================================
+ tuple ('<not %r>', other)
+ str '<branch closed>'
+ callable lambda: '<branch %r>' % sorted(b)
+ object other
+ ======== =================================
+ """
+ if r is None:
+ return ''
+ elif isinstance(r, tuple):
+ return r[0] % r[1:]
+ elif isinstance(r, str):
+ return r
+ elif callable(r):
+ return r()
+ else:
+ return repr(r)
+
+class abstractsmartset(object):
+
+ def __nonzero__(self):
+ """True if the smartset is not empty"""
+ raise NotImplementedError()
+
+ def __contains__(self, rev):
+ """provide fast membership testing"""
+ raise NotImplementedError()
+
+ def __iter__(self):
+ """iterate the set in the order it is supposed to be iterated"""
+ raise NotImplementedError()
+
+ # Attributes containing a function to perform a fast iteration in a given
+ # direction. A smartset can have none, one, or both defined.
+ #
+ # Default value is None instead of a function returning None to avoid
+ # initializing an iterator just for testing if a fast method exists.
+ fastasc = None
+ fastdesc = None
+
+ def isascending(self):
+ """True if the set will iterate in ascending order"""
+ raise NotImplementedError()
+
+ def isdescending(self):
+ """True if the set will iterate in descending order"""
+ raise NotImplementedError()
+
+ def istopo(self):
+ """True if the set will iterate in topographical order"""
+ raise NotImplementedError()
+
+ def min(self):
+ """return the minimum element in the set"""
+ if self.fastasc is None:
+ v = min(self)
+ else:
+ for v in self.fastasc():
+ break
+ else:
+ raise ValueError('arg is an empty sequence')
+ self.min = lambda: v
+ return v
+
+ def max(self):
+ """return the maximum element in the set"""
+ if self.fastdesc is None:
+ return max(self)
+ else:
+ for v in self.fastdesc():
+ break
+ else:
+ raise ValueError('arg is an empty sequence')
+ self.max = lambda: v
+ return v
+
+ def first(self):
+ """return the first element in the set (user iteration perspective)
+
+ Return None if the set is empty"""
+ raise NotImplementedError()
+
+ def last(self):
+ """return the last element in the set (user iteration perspective)
+
+ Return None if the set is empty"""
+ raise NotImplementedError()
+
+ def __len__(self):
+ """return the length of the smartsets
+
+ This can be expensive on smartset that could be lazy otherwise."""
+ raise NotImplementedError()
+
+ def reverse(self):
+ """reverse the expected iteration order"""
+ raise NotImplementedError()
+
+ def sort(self, reverse=True):
+ """get the set to iterate in an ascending or descending order"""
+ raise NotImplementedError()
+
+ def __and__(self, other):
+ """Returns a new object with the intersection of the two collections.
+
+ This is part of the mandatory API for smartset."""
+ if isinstance(other, fullreposet):
+ return self
+ return self.filter(other.__contains__, condrepr=other, cache=False)
+
+ def __add__(self, other):
+ """Returns a new object with the union of the two collections.
+
+ This is part of the mandatory API for smartset."""
+ return addset(self, other)
+
+ def __sub__(self, other):
+ """Returns a new object with the substraction of the two collections.
+
+ This is part of the mandatory API for smartset."""
+ c = other.__contains__
+ return self.filter(lambda r: not c(r), condrepr=('<not %r>', other),
+ cache=False)
+
+ def filter(self, condition, condrepr=None, cache=True):
+ """Returns this smartset filtered by condition as a new smartset.
+
+ `condition` is a callable which takes a revision number and returns a
+ boolean. Optional `condrepr` provides a printable representation of
+ the given `condition`.
+
+ This is part of the mandatory API for smartset."""
+ # builtin cannot be cached. but do not needs to
+ if cache and util.safehasattr(condition, 'func_code'):
+ condition = util.cachefunc(condition)
+ return filteredset(self, condition, condrepr)
+
+class baseset(abstractsmartset):
+ """Basic data structure that represents a revset and contains the basic
+ operation that it should be able to perform.
+
+ Every method in this class should be implemented by any smartset class.
+
+ This class could be constructed by an (unordered) set, or an (ordered)
+ list-like object. If a set is provided, it'll be sorted lazily.
+
+ >>> x = [4, 0, 7, 6]
+ >>> y = [5, 6, 7, 3]
+
+ Construct by a set:
+ >>> xs = baseset(set(x))
+ >>> ys = baseset(set(y))
+ >>> [list(i) for i in [xs + ys, xs & ys, xs - ys]]
+ [[0, 4, 6, 7, 3, 5], [6, 7], [0, 4]]
+ >>> [type(i).__name__ for i in [xs + ys, xs & ys, xs - ys]]
+ ['addset', 'baseset', 'baseset']
+
+ Construct by a list-like:
+ >>> xs = baseset(x)
+ >>> ys = baseset(i for i in y)
+ >>> [list(i) for i in [xs + ys, xs & ys, xs - ys]]
+ [[4, 0, 7, 6, 5, 3], [7, 6], [4, 0]]
+ >>> [type(i).__name__ for i in [xs + ys, xs & ys, xs - ys]]
+ ['addset', 'filteredset', 'filteredset']
+
+ Populate "_set" fields in the lists so set optimization may be used:
+ >>> [1 in xs, 3 in ys]
+ [False, True]
+
+ Without sort(), results won't be changed:
+ >>> [list(i) for i in [xs + ys, xs & ys, xs - ys]]
+ [[4, 0, 7, 6, 5, 3], [7, 6], [4, 0]]
+ >>> [type(i).__name__ for i in [xs + ys, xs & ys, xs - ys]]
+ ['addset', 'filteredset', 'filteredset']
+
+ With sort(), set optimization could be used:
+ >>> xs.sort(reverse=True)
+ >>> [list(i) for i in [xs + ys, xs & ys, xs - ys]]
+ [[7, 6, 4, 0, 5, 3], [7, 6], [4, 0]]
+ >>> [type(i).__name__ for i in [xs + ys, xs & ys, xs - ys]]
+ ['addset', 'baseset', 'baseset']
+
+ >>> ys.sort()
+ >>> [list(i) for i in [xs + ys, xs & ys, xs - ys]]
+ [[7, 6, 4, 0, 3, 5], [7, 6], [4, 0]]
+ >>> [type(i).__name__ for i in [xs + ys, xs & ys, xs - ys]]
+ ['addset', 'baseset', 'baseset']
+
+ istopo is preserved across set operations
+ >>> xs = baseset(set(x), istopo=True)
+ >>> rs = xs & ys
+ >>> type(rs).__name__
+ 'baseset'
+ >>> rs._istopo
+ True
+ """
+ def __init__(self, data=(), datarepr=None, istopo=False):
+ """
+ datarepr: a tuple of (format, obj, ...), a function or an object that
+ provides a printable representation of the given data.
+ """
+ self._ascending = None
+ self._istopo = istopo
+ if not isinstance(data, list):
+ if isinstance(data, set):
+ self._set = data
+ # set has no order we pick one for stability purpose
+ self._ascending = True
+ # converting set to list has a cost, do it lazily
+ data = None
+ else:
+ data = list(data)
+ if data is not None:
+ self._list = data
+ self._datarepr = datarepr
+
+ @util.propertycache
+ def _set(self):
+ return set(self._list)
+
+ @util.propertycache
+ def _asclist(self):
+ asclist = self._list[:]
+ asclist.sort()
+ return asclist
+
+ @util.propertycache
+ def _list(self):
+ # _list is only lazily constructed if we have _set
+ assert '_set' in self.__dict__
+ return list(self._set)
+
+ def __iter__(self):
+ if self._ascending is None:
+ return iter(self._list)
+ elif self._ascending:
+ return iter(self._asclist)
+ else:
+ return reversed(self._asclist)
+
+ def fastasc(self):
+ return iter(self._asclist)
+
+ def fastdesc(self):
+ return reversed(self._asclist)
+
+ @util.propertycache
+ def __contains__(self):
+ return self._set.__contains__
+
+ def __nonzero__(self):
+ return bool(len(self))
+
+ def sort(self, reverse=False):
+ self._ascending = not bool(reverse)
+ self._istopo = False
+
+ def reverse(self):
+ if self._ascending is None:
+ self._list.reverse()
+ else:
+ self._ascending = not self._ascending
+ self._istopo = False
+
+ def __len__(self):
+ if '_list' in self.__dict__:
+ return len(self._list)
+ else:
+ return len(self._set)
+
+ def isascending(self):
+ """Returns True if the collection is ascending order, False if not.
+
+ This is part of the mandatory API for smartset."""
+ if len(self) <= 1:
+ return True
+ return self._ascending is not None and self._ascending
+
+ def isdescending(self):
+ """Returns True if the collection is descending order, False if not.
+
+ This is part of the mandatory API for smartset."""
+ if len(self) <= 1:
+ return True
+ return self._ascending is not None and not self._ascending
+
+ def istopo(self):
+ """Is the collection is in topographical order or not.
+
+ This is part of the mandatory API for smartset."""
+ if len(self) <= 1:
+ return True
+ return self._istopo
+
+ def first(self):
+ if self:
+ if self._ascending is None:
+ return self._list[0]
+ elif self._ascending:
+ return self._asclist[0]
+ else:
+ return self._asclist[-1]
+ return None
+
+ def last(self):
+ if self:
+ if self._ascending is None:
+ return self._list[-1]
+ elif self._ascending:
+ return self._asclist[-1]
+ else:
+ return self._asclist[0]
+ return None
+
+ def _fastsetop(self, other, op):
+ # try to use native set operations as fast paths
+ if (type(other) is baseset and '_set' in other.__dict__ and '_set' in
+ self.__dict__ and self._ascending is not None):
+ s = baseset(data=getattr(self._set, op)(other._set),
+ istopo=self._istopo)
+ s._ascending = self._ascending
+ else:
+ s = getattr(super(baseset, self), op)(other)
+ return s
+
+ def __and__(self, other):
+ return self._fastsetop(other, '__and__')
+
+ def __sub__(self, other):
+ return self._fastsetop(other, '__sub__')
+
+ def __repr__(self):
+ d = {None: '', False: '-', True: '+'}[self._ascending]
+ s = _formatsetrepr(self._datarepr)
+ if not s:
+ l = self._list
+ # if _list has been built from a set, it might have a different
+ # order from one python implementation to another.
+ # We fallback to the sorted version for a stable output.
+ if self._ascending is not None:
+ l = self._asclist
+ s = repr(l)
+ return '<%s%s %s>' % (type(self).__name__, d, s)
+
+class filteredset(abstractsmartset):
+ """Duck type for baseset class which iterates lazily over the revisions in
+ the subset and contains a function which tests for membership in the
+ revset
+ """
+ def __init__(self, subset, condition=lambda x: True, condrepr=None):
+ """
+ condition: a function that decide whether a revision in the subset
+ belongs to the revset or not.
+ condrepr: a tuple of (format, obj, ...), a function or an object that
+ provides a printable representation of the given condition.
+ """
+ self._subset = subset
+ self._condition = condition
+ self._condrepr = condrepr
+
+ def __contains__(self, x):
+ return x in self._subset and self._condition(x)
+
+ def __iter__(self):
+ return self._iterfilter(self._subset)
+
+ def _iterfilter(self, it):
+ cond = self._condition
+ for x in it:
+ if cond(x):
+ yield x
+
+ @property
+ def fastasc(self):
+ it = self._subset.fastasc
+ if it is None:
+ return None
+ return lambda: self._iterfilter(it())
+
+ @property
+ def fastdesc(self):
+ it = self._subset.fastdesc
+ if it is None:
+ return None
+ return lambda: self._iterfilter(it())
+
+ def __nonzero__(self):
+ fast = None
+ candidates = [self.fastasc if self.isascending() else None,
+ self.fastdesc if self.isdescending() else None,
+ self.fastasc,
+ self.fastdesc]
+ for candidate in candidates:
+ if candidate is not None:
+ fast = candidate
+ break
+
+ if fast is not None:
+ it = fast()
+ else:
+ it = self
+
+ for r in it:
+ return True
+ return False
+
+ def __len__(self):
+ # Basic implementation to be changed in future patches.
+ # until this gets improved, we use generator expression
+ # here, since list comprehensions are free to call __len__ again
+ # causing infinite recursion
+ l = baseset(r for r in self)
+ return len(l)
+
+ def sort(self, reverse=False):
+ self._subset.sort(reverse=reverse)
+
+ def reverse(self):
+ self._subset.reverse()
+
+ def isascending(self):
+ return self._subset.isascending()
+
+ def isdescending(self):
+ return self._subset.isdescending()
+
+ def istopo(self):
+ return self._subset.istopo()
+
+ def first(self):
+ for x in self:
+ return x
+ return None
+
+ def last(self):
+ it = None
+ if self.isascending():
+ it = self.fastdesc
+ elif self.isdescending():
+ it = self.fastasc
+ if it is not None:
+ for x in it():
+ return x
+ return None #empty case
+ else:
+ x = None
+ for x in self:
+ pass
+ return x
+
+ def __repr__(self):
+ xs = [repr(self._subset)]
+ s = _formatsetrepr(self._condrepr)
+ if s:
+ xs.append(s)
+ return '<%s %s>' % (type(self).__name__, ', '.join(xs))
+
+def _iterordered(ascending, iter1, iter2):
+ """produce an ordered iteration from two iterators with the same order
+
+ The ascending is used to indicated the iteration direction.
+ """
+ choice = max
+ if ascending:
+ choice = min
+
+ val1 = None
+ val2 = None
+ try:
+ # Consume both iterators in an ordered way until one is empty
+ while True:
+ if val1 is None:
+ val1 = next(iter1)
+ if val2 is None:
+ val2 = next(iter2)
+ n = choice(val1, val2)
+ yield n
+ if val1 == n:
+ val1 = None
+ if val2 == n:
+ val2 = None
+ except StopIteration:
+ # Flush any remaining values and consume the other one
+ it = iter2
+ if val1 is not None:
+ yield val1
+ it = iter1
+ elif val2 is not None:
+ # might have been equality and both are empty
+ yield val2
+ for val in it:
+ yield val
+
+class addset(abstractsmartset):
+ """Represent the addition of two sets
+
+ Wrapper structure for lazily adding two structures without losing much
+ performance on the __contains__ method
+
+ If the ascending attribute is set, that means the two structures are
+ ordered in either an ascending or descending way. Therefore, we can add
+ them maintaining the order by iterating over both at the same time
+
+ >>> xs = baseset([0, 3, 2])
+ >>> ys = baseset([5, 2, 4])
+
+ >>> rs = addset(xs, ys)
+ >>> bool(rs), 0 in rs, 1 in rs, 5 in rs, rs.first(), rs.last()
+ (True, True, False, True, 0, 4)
+ >>> rs = addset(xs, baseset([]))
+ >>> bool(rs), 0 in rs, 1 in rs, rs.first(), rs.last()
+ (True, True, False, 0, 2)
+ >>> rs = addset(baseset([]), baseset([]))
+ >>> bool(rs), 0 in rs, rs.first(), rs.last()
+ (False, False, None, None)
+
+ iterate unsorted:
+ >>> rs = addset(xs, ys)
+ >>> # (use generator because pypy could call len())
+ >>> list(x for x in rs) # without _genlist
+ [0, 3, 2, 5, 4]
+ >>> assert not rs._genlist
+ >>> len(rs)
+ 5
+ >>> [x for x in rs] # with _genlist
+ [0, 3, 2, 5, 4]
+ >>> assert rs._genlist
+
+ iterate ascending:
+ >>> rs = addset(xs, ys, ascending=True)
+ >>> # (use generator because pypy could call len())
+ >>> list(x for x in rs), list(x for x in rs.fastasc()) # without _asclist
+ ([0, 2, 3, 4, 5], [0, 2, 3, 4, 5])
+ >>> assert not rs._asclist
+ >>> len(rs)
+ 5
+ >>> [x for x in rs], [x for x in rs.fastasc()]
+ ([0, 2, 3, 4, 5], [0, 2, 3, 4, 5])
+ >>> assert rs._asclist
+
+ iterate descending:
+ >>> rs = addset(xs, ys, ascending=False)
+ >>> # (use generator because pypy could call len())
+ >>> list(x for x in rs), list(x for x in rs.fastdesc()) # without _asclist
+ ([5, 4, 3, 2, 0], [5, 4, 3, 2, 0])
+ >>> assert not rs._asclist
+ >>> len(rs)
+ 5
+ >>> [x for x in rs], [x for x in rs.fastdesc()]
+ ([5, 4, 3, 2, 0], [5, 4, 3, 2, 0])
+ >>> assert rs._asclist
+
+ iterate ascending without fastasc:
+ >>> rs = addset(xs, generatorset(ys), ascending=True)
+ >>> assert rs.fastasc is None
+ >>> [x for x in rs]
+ [0, 2, 3, 4, 5]
+
+ iterate descending without fastdesc:
+ >>> rs = addset(generatorset(xs), ys, ascending=False)
+ >>> assert rs.fastdesc is None
+ >>> [x for x in rs]
+ [5, 4, 3, 2, 0]
+ """
+ def __init__(self, revs1, revs2, ascending=None):
+ self._r1 = revs1
+ self._r2 = revs2
+ self._iter = None
+ self._ascending = ascending
+ self._genlist = None
+ self._asclist = None
+
+ def __len__(self):
+ return len(self._list)
+
+ def __nonzero__(self):
+ return bool(self._r1) or bool(self._r2)
+
+ @util.propertycache
+ def _list(self):
+ if not self._genlist:
+ self._genlist = baseset(iter(self))
+ return self._genlist
+
+ def __iter__(self):
+ """Iterate over both collections without repeating elements
+
+ If the ascending attribute is not set, iterate over the first one and
+ then over the second one checking for membership on the first one so we
+ dont yield any duplicates.
+
+ If the ascending attribute is set, iterate over both collections at the
+ same time, yielding only one value at a time in the given order.
+ """
+ if self._ascending is None:
+ if self._genlist:
+ return iter(self._genlist)
+ def arbitraryordergen():
+ for r in self._r1:
+ yield r
+ inr1 = self._r1.__contains__
+ for r in self._r2:
+ if not inr1(r):
+ yield r
+ return arbitraryordergen()
+ # try to use our own fast iterator if it exists
+ self._trysetasclist()
+ if self._ascending:
+ attr = 'fastasc'
+ else:
+ attr = 'fastdesc'
+ it = getattr(self, attr)
+ if it is not None:
+ return it()
+ # maybe half of the component supports fast
+ # get iterator for _r1
+ iter1 = getattr(self._r1, attr)
+ if iter1 is None:
+ # let's avoid side effect (not sure it matters)
+ iter1 = iter(sorted(self._r1, reverse=not self._ascending))
+ else:
+ iter1 = iter1()
+ # get iterator for _r2
+ iter2 = getattr(self._r2, attr)
+ if iter2 is None:
+ # let's avoid side effect (not sure it matters)
+ iter2 = iter(sorted(self._r2, reverse=not self._ascending))
+ else:
+ iter2 = iter2()
+ return _iterordered(self._ascending, iter1, iter2)
+
+ def _trysetasclist(self):
+ """populate the _asclist attribute if possible and necessary"""
+ if self._genlist is not None and self._asclist is None:
+ self._asclist = sorted(self._genlist)
+
+ @property
+ def fastasc(self):
+ self._trysetasclist()
+ if self._asclist is not None:
+ return self._asclist.__iter__
+ iter1 = self._r1.fastasc
+ iter2 = self._r2.fastasc
+ if None in (iter1, iter2):
+ return None
+ return lambda: _iterordered(True, iter1(), iter2())
+
+ @property
+ def fastdesc(self):
+ self._trysetasclist()
+ if self._asclist is not None:
+ return self._asclist.__reversed__
+ iter1 = self._r1.fastdesc
+ iter2 = self._r2.fastdesc
+ if None in (iter1, iter2):
+ return None
+ return lambda: _iterordered(False, iter1(), iter2())
+
+ def __contains__(self, x):
+ return x in self._r1 or x in self._r2
+
+ def sort(self, reverse=False):
+ """Sort the added set
+
+ For this we use the cached list with all the generated values and if we
+ know they are ascending or descending we can sort them in a smart way.
+ """
+ self._ascending = not reverse
+
+ def isascending(self):
+ return self._ascending is not None and self._ascending
+
+ def isdescending(self):
+ return self._ascending is not None and not self._ascending
+
+ def istopo(self):
+ # not worth the trouble asserting if the two sets combined are still
+ # in topographical order. Use the sort() predicate to explicitly sort
+ # again instead.
+ return False
+
+ def reverse(self):
+ if self._ascending is None:
+ self._list.reverse()
+ else:
+ self._ascending = not self._ascending
+
+ def first(self):
+ for x in self:
+ return x
+ return None
+
+ def last(self):
+ self.reverse()
+ val = self.first()
+ self.reverse()
+ return val
+
+ def __repr__(self):
+ d = {None: '', False: '-', True: '+'}[self._ascending]
+ return '<%s%s %r, %r>' % (type(self).__name__, d, self._r1, self._r2)
+
+class generatorset(abstractsmartset):
+ """Wrap a generator for lazy iteration
+
+ Wrapper structure for generators that provides lazy membership and can
+ be iterated more than once.
+ When asked for membership it generates values until either it finds the
+ requested one or has gone through all the elements in the generator
+ """
+ def __init__(self, gen, iterasc=None):
+ """
+ gen: a generator producing the values for the generatorset.
+ """
+ self._gen = gen
+ self._asclist = None
+ self._cache = {}
+ self._genlist = []
+ self._finished = False
+ self._ascending = True
+ if iterasc is not None:
+ if iterasc:
+ self.fastasc = self._iterator
+ self.__contains__ = self._asccontains
+ else:
+ self.fastdesc = self._iterator
+ self.__contains__ = self._desccontains
+
+ def __nonzero__(self):
+ # Do not use 'for r in self' because it will enforce the iteration
+ # order (default ascending), possibly unrolling a whole descending
+ # iterator.
+ if self._genlist:
+ return True
+ for r in self._consumegen():
+ return True
+ return False
+
+ def __contains__(self, x):
+ if x in self._cache:
+ return self._cache[x]
+
+ # Use new values only, as existing values would be cached.
+ for l in self._consumegen():
+ if l == x:
+ return True
+
+ self._cache[x] = False
+ return False
+
+ def _asccontains(self, x):
+ """version of contains optimised for ascending generator"""
+ if x in self._cache:
+ return self._cache[x]
+
+ # Use new values only, as existing values would be cached.
+ for l in self._consumegen():
+ if l == x:
+ return True
+ if l > x:
+ break
+
+ self._cache[x] = False
+ return False
+
+ def _desccontains(self, x):
+ """version of contains optimised for descending generator"""
+ if x in self._cache:
+ return self._cache[x]
+
+ # Use new values only, as existing values would be cached.
+ for l in self._consumegen():
+ if l == x:
+ return True
+ if l < x:
+ break
+
+ self._cache[x] = False
+ return False
+
+ def __iter__(self):
+ if self._ascending:
+ it = self.fastasc
+ else:
+ it = self.fastdesc
+ if it is not None:
+ return it()
+ # we need to consume the iterator
+ for x in self._consumegen():
+ pass
+ # recall the same code
+ return iter(self)
+
+ def _iterator(self):
+ if self._finished:
+ return iter(self._genlist)
+
+ # We have to use this complex iteration strategy to allow multiple
+ # iterations at the same time. We need to be able to catch revision
+ # removed from _consumegen and added to genlist in another instance.
+ #
+ # Getting rid of it would provide an about 15% speed up on this
+ # iteration.
+ genlist = self._genlist
+ nextrev = self._consumegen().next
+ _len = len # cache global lookup
+ def gen():
+ i = 0
+ while True:
+ if i < _len(genlist):
+ yield genlist[i]
+ else:
+ yield nextrev()
+ i += 1
+ return gen()
+
+ def _consumegen(self):
+ cache = self._cache
+ genlist = self._genlist.append
+ for item in self._gen:
+ cache[item] = True
+ genlist(item)
+ yield item
+ if not self._finished:
+ self._finished = True
+ asc = self._genlist[:]
+ asc.sort()
+ self._asclist = asc
+ self.fastasc = asc.__iter__
+ self.fastdesc = asc.__reversed__
+
+ def __len__(self):
+ for x in self._consumegen():
+ pass
+ return len(self._genlist)
+
+ def sort(self, reverse=False):
+ self._ascending = not reverse
+
+ def reverse(self):
+ self._ascending = not self._ascending
+
+ def isascending(self):
+ return self._ascending
+
+ def isdescending(self):
+ return not self._ascending
+
+ def istopo(self):
+ # not worth the trouble asserting if the two sets combined are still
+ # in topographical order. Use the sort() predicate to explicitly sort
+ # again instead.
+ return False
+
+ def first(self):
+ if self._ascending:
+ it = self.fastasc
+ else:
+ it = self.fastdesc
+ if it is None:
+ # we need to consume all and try again
+ for x in self._consumegen():
+ pass
+ return self.first()
+ return next(it(), None)
+
+ def last(self):
+ if self._ascending:
+ it = self.fastdesc
+ else:
+ it = self.fastasc
+ if it is None:
+ # we need to consume all and try again
+ for x in self._consumegen():
+ pass
+ return self.first()
+ return next(it(), None)
+
+ def __repr__(self):
+ d = {False: '-', True: '+'}[self._ascending]
+ return '<%s%s>' % (type(self).__name__, d)
+
+class spanset(abstractsmartset):
+ """Duck type for baseset class which represents a range of revisions and
+ can work lazily and without having all the range in memory
+
+ Note that spanset(x, y) behave almost like xrange(x, y) except for two
+ notable points:
+ - when x < y it will be automatically descending,
+ - revision filtered with this repoview will be skipped.
+
+ """
+ def __init__(self, repo, start=0, end=None):
+ """
+ start: first revision included the set
+ (default to 0)
+ end: first revision excluded (last+1)
+ (default to len(repo)
+
+ Spanset will be descending if `end` < `start`.
+ """
+ if end is None:
+ end = len(repo)
+ self._ascending = start <= end
+ if not self._ascending:
+ start, end = end + 1, start +1
+ self._start = start
+ self._end = end
+ self._hiddenrevs = repo.changelog.filteredrevs
+
+ def sort(self, reverse=False):
+ self._ascending = not reverse
+
+ def reverse(self):
+ self._ascending = not self._ascending
+
+ def istopo(self):
+ # not worth the trouble asserting if the two sets combined are still
+ # in topographical order. Use the sort() predicate to explicitly sort
+ # again instead.
+ return False
+
+ def _iterfilter(self, iterrange):
+ s = self._hiddenrevs
+ for r in iterrange:
+ if r not in s:
+ yield r
+
+ def __iter__(self):
+ if self._ascending:
+ return self.fastasc()
+ else:
+ return self.fastdesc()
+
+ def fastasc(self):
+ iterrange = xrange(self._start, self._end)
+ if self._hiddenrevs:
+ return self._iterfilter(iterrange)
+ return iter(iterrange)
+
+ def fastdesc(self):
+ iterrange = xrange(self._end - 1, self._start - 1, -1)
+ if self._hiddenrevs:
+ return self._iterfilter(iterrange)
+ return iter(iterrange)
+
+ def __contains__(self, rev):
+ hidden = self._hiddenrevs
+ return ((self._start <= rev < self._end)
+ and not (hidden and rev in hidden))
+
+ def __nonzero__(self):
+ for r in self:
+ return True
+ return False
+
+ def __len__(self):
+ if not self._hiddenrevs:
+ return abs(self._end - self._start)
+ else:
+ count = 0
+ start = self._start
+ end = self._end
+ for rev in self._hiddenrevs:
+ if (end < rev <= start) or (start <= rev < end):
+ count += 1
+ return abs(self._end - self._start) - count
+
+ def isascending(self):
+ return self._ascending
+
+ def isdescending(self):
+ return not self._ascending
+
+ def first(self):
+ if self._ascending:
+ it = self.fastasc
+ else:
+ it = self.fastdesc
+ for x in it():
+ return x
+ return None
+
+ def last(self):
+ if self._ascending:
+ it = self.fastdesc
+ else:
+ it = self.fastasc
+ for x in it():
+ return x
+ return None
+
+ def __repr__(self):
+ d = {False: '-', True: '+'}[self._ascending]
+ return '<%s%s %d:%d>' % (type(self).__name__, d,
+ self._start, self._end - 1)
+
+class fullreposet(spanset):
+ """a set containing all revisions in the repo
+
+ This class exists to host special optimization and magic to handle virtual
+ revisions such as "null".
+ """
+
+ def __init__(self, repo):
+ super(fullreposet, self).__init__(repo)
+
+ def __and__(self, other):
+ """As self contains the whole repo, all of the other set should also be
+ in self. Therefore `self & other = other`.
+
+ This boldly assumes the other contains valid revs only.
+ """
+ # other not a smartset, make is so
+ if not util.safehasattr(other, 'isascending'):
+ # filter out hidden revision
+ # (this boldly assumes all smartset are pure)
+ #
+ # `other` was used with "&", let's assume this is a set like
+ # object.
+ other = baseset(other - self._hiddenrevs)
+
+ other.sort(reverse=self.isdescending())
+ return other
+
+def prettyformat(revs):
+ lines = []
+ rs = repr(revs)
+ p = 0
+ while p < len(rs):
+ q = rs.find('<', p + 1)
+ if q < 0:
+ q = len(rs)
+ l = rs.count('<', 0, p) - rs.count('>', 0, p)
+ assert l >= 0
+ lines.append((l, rs[p:q].rstrip()))
+ p = q
+ return '\n'.join(' ' * l + s for l, s in lines)
--- a/mercurial/sslutil.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/sslutil.py Tue Feb 28 11:13:25 2017 -0800
@@ -720,7 +720,8 @@
# to load the system CA store. If we're running on Apple Python, use this
# trick.
if _plainapplepython():
- dummycert = os.path.join(os.path.dirname(__file__), 'dummycert.pem')
+ dummycert = os.path.join(
+ os.path.dirname(pycompat.fsencode(__file__)), 'dummycert.pem')
if os.path.exists(dummycert):
return dummycert
--- a/mercurial/statprof.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/statprof.py Tue Feb 28 11:13:25 2017 -0800
@@ -433,6 +433,7 @@
Hotpath = 3
FlameGraph = 4
Json = 5
+ Chrome = 6
def display(fp=None, format=3, data=None, **kwargs):
'''Print statistics, either to stdout or the given file object.'''
@@ -457,10 +458,12 @@
write_to_flame(data, fp, **kwargs)
elif format == DisplayFormats.Json:
write_to_json(data, fp)
+ elif format == DisplayFormats.Chrome:
+ write_to_chrome(data, fp, **kwargs)
else:
raise Exception("Invalid display format")
- if format != DisplayFormats.Json:
+ if format not in (DisplayFormats.Json, DisplayFormats.Chrome):
print('---', file=fp)
print('Sample count: %d' % len(data.samples), file=fp)
print('Total time: %f seconds' % data.accumulated_time, file=fp)
@@ -713,6 +716,23 @@
os.system("perl ~/flamegraph.pl %s > %s" % (path, outputfile))
print("Written to %s" % outputfile, file=fp)
+_pathcache = {}
+def simplifypath(path):
+ '''Attempt to make the path to a Python module easier to read by
+ removing whatever part of the Python search path it was found
+ on.'''
+
+ if path in _pathcache:
+ return _pathcache[path]
+ hgpath = pycompat.fsencode(encoding.__file__).rsplit(os.sep, 2)[0]
+ for p in [hgpath] + sys.path:
+ prefix = p + os.sep
+ if path.startswith(prefix):
+ path = path[len(prefix):]
+ break
+ _pathcache[path] = path
+ return path
+
def write_to_json(data, fp):
samples = []
@@ -726,6 +746,102 @@
print(json.dumps(samples), file=fp)
+def write_to_chrome(data, fp, minthreshold=0.005, maxthreshold=0.999):
+ samples = []
+ laststack = collections.deque()
+ lastseen = collections.deque()
+
+ # The Chrome tracing format allows us to use a compact stack
+ # representation to save space. It's fiddly but worth it.
+ # We maintain a bijection between stack and ID.
+ stack2id = {}
+ id2stack = [] # will eventually be rendered
+
+ def stackid(stack):
+ if not stack:
+ return
+ if stack in stack2id:
+ return stack2id[stack]
+ parent = stackid(stack[1:])
+ myid = len(stack2id)
+ stack2id[stack] = myid
+ id2stack.append(dict(category=stack[0][0], name='%s %s' % stack[0]))
+ if parent is not None:
+ id2stack[-1].update(parent=parent)
+ return myid
+
+ def endswith(a, b):
+ return list(a)[-len(b):] == list(b)
+
+ # The sampling profiler can sample multiple times without
+ # advancing the clock, potentially causing the Chrome trace viewer
+ # to render single-pixel columns that we cannot zoom in on. We
+ # work around this by pretending that zero-duration samples are a
+ # millisecond in length.
+
+ clamp = 0.001
+
+ # We provide knobs that by default attempt to filter out stack
+ # frames that are too noisy:
+ #
+ # * A few take almost all execution time. These are usually boring
+ # setup functions, giving a stack that is deep but uninformative.
+ #
+ # * Numerous samples take almost no time, but introduce lots of
+ # noisy, oft-deep "spines" into a rendered profile.
+
+ blacklist = set()
+ totaltime = data.samples[-1].time - data.samples[0].time
+ minthreshold = totaltime * minthreshold
+ maxthreshold = max(totaltime * maxthreshold, clamp)
+
+ def poplast():
+ oldsid = stackid(tuple(laststack))
+ oldcat, oldfunc = laststack.popleft()
+ oldtime, oldidx = lastseen.popleft()
+ duration = sample.time - oldtime
+ if minthreshold <= duration <= maxthreshold:
+ # ensure no zero-duration events
+ sampletime = max(oldtime + clamp, sample.time)
+ samples.append(dict(ph='E', name=oldfunc, cat=oldcat, sf=oldsid,
+ ts=sampletime*1e6, pid=0))
+ else:
+ blacklist.add(oldidx)
+
+ # Much fiddling to synthesize correctly(ish) nested begin/end
+ # events given only stack snapshots.
+
+ for sample in data.samples:
+ tos = sample.stack[0]
+ name = tos.function
+ path = simplifypath(tos.path)
+ category = '%s:%d' % (path, tos.lineno)
+ stack = tuple((('%s:%d' % (simplifypath(frame.path), frame.lineno),
+ frame.function) for frame in sample.stack))
+ qstack = collections.deque(stack)
+ if laststack == qstack:
+ continue
+ while laststack and qstack and laststack[-1] == qstack[-1]:
+ laststack.pop()
+ qstack.pop()
+ while laststack:
+ poplast()
+ for f in reversed(qstack):
+ lastseen.appendleft((sample.time, len(samples)))
+ laststack.appendleft(f)
+ path, name = f
+ sid = stackid(tuple(laststack))
+ samples.append(dict(ph='B', name=name, cat=path, ts=sample.time*1e6,
+ sf=sid, pid=0))
+ laststack = collections.deque(stack)
+ while laststack:
+ poplast()
+ events = [s[1] for s in enumerate(samples) if s[0] not in blacklist]
+ frames = collections.OrderedDict((str(k), v)
+ for (k,v) in enumerate(id2stack))
+ json.dump(dict(traceEvents=events, stackFrames=frames), fp, indent=1)
+ fp.write('\n')
+
def printusage():
print("""
The statprof command line allows you to inspect the last profile's results in
--- a/mercurial/store.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/store.py Tue Feb 28 11:13:25 2017 -0800
@@ -101,7 +101,7 @@
e = '_'
if pycompat.ispy3:
xchr = lambda x: bytes([x])
- asciistr = bytes(xrange(127))
+ asciistr = [bytes(a) for a in range(127)]
else:
xchr = chr
asciistr = map(chr, xrange(127))
--- a/mercurial/streamclone.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/streamclone.py Tue Feb 28 11:13:25 2017 -0800
@@ -8,7 +8,6 @@
from __future__ import absolute_import
import struct
-import time
from .i18n import _
from . import (
@@ -297,7 +296,7 @@
(filecount, util.bytecount(bytecount)))
handled_bytes = 0
repo.ui.progress(_('clone'), 0, total=bytecount, unit=_('bytes'))
- start = time.time()
+ start = util.timer()
# TODO: get rid of (potential) inconsistency
#
@@ -340,7 +339,7 @@
# streamclone-ed file at next access
repo.invalidate(clearfilecache=True)
- elapsed = time.time() - start
+ elapsed = util.timer() - start
if elapsed <= 0:
elapsed = 0.001
repo.ui.progress(_('clone'), None)
--- a/mercurial/subrepo.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/subrepo.py Tue Feb 28 11:13:25 2017 -0800
@@ -542,8 +542,8 @@
"""return filename iterator"""
raise NotImplementedError
- def filedata(self, name):
- """return file data"""
+ def filedata(self, name, decode):
+ """return file data, optionally passed through repo decoders"""
raise NotImplementedError
def fileflags(self, name):
@@ -558,7 +558,7 @@
"""handle the files command for this subrepo"""
return 1
- def archive(self, archiver, prefix, match=None):
+ def archive(self, archiver, prefix, match=None, decode=True):
if match is not None:
files = [f for f in self.files() if match(f)]
else:
@@ -572,7 +572,7 @@
mode = 'x' in flags and 0o755 or 0o644
symlink = 'l' in flags
archiver.addfile(prefix + self._path + '/' + name,
- mode, symlink, self.filedata(name))
+ mode, symlink, self.filedata(name, decode))
self.ui.progress(_('archiving (%s)') % relpath, i + 1,
unit=_('files'), total=total)
self.ui.progress(_('archiving (%s)') % relpath, None)
@@ -782,7 +782,7 @@
% (inst, subrelpath(self)))
@annotatesubrepoerror
- def archive(self, archiver, prefix, match=None):
+ def archive(self, archiver, prefix, match=None, decode=True):
self._get(self._state + ('hg',))
total = abstractsubrepo.archive(self, archiver, prefix, match)
rev = self._state[1]
@@ -790,7 +790,8 @@
for subpath in ctx.substate:
s = subrepo(ctx, subpath, True)
submatch = matchmod.subdirmatcher(subpath, match)
- total += s.archive(archiver, prefix + self._path + '/', submatch)
+ total += s.archive(archiver, prefix + self._path + '/', submatch,
+ decode)
return total
@annotatesubrepoerror
@@ -956,9 +957,12 @@
ctx = self._repo[rev]
return ctx.manifest().keys()
- def filedata(self, name):
+ def filedata(self, name, decode):
rev = self._state[1]
- return self._repo[rev][name].data()
+ data = self._repo[rev][name].data()
+ if decode:
+ data = self._repo.wwritedata(name, data)
+ return data
def fileflags(self, name):
rev = self._state[1]
@@ -1292,7 +1296,7 @@
paths.append(name.encode('utf-8'))
return paths
- def filedata(self, name):
+ def filedata(self, name, decode):
return self._svncommand(['cat'], name)[0]
@@ -1410,6 +1414,10 @@
errpipe = None
if self.ui.quiet:
errpipe = open(os.devnull, 'w')
+ if self.ui._colormode and len(commands) and commands[0] == "diff":
+ # insert the argument in the front,
+ # the end of git diff arguments is used for paths
+ commands.insert(1, '--color')
p = subprocess.Popen([self._gitexecutable] + commands, bufsize=-1,
cwd=cwd, env=env, close_fds=util.closefds,
stdout=subprocess.PIPE, stderr=errpipe)
@@ -1772,7 +1780,7 @@
else:
self.wvfs.unlink(f)
- def archive(self, archiver, prefix, match=None):
+ def archive(self, archiver, prefix, match=None, decode=True):
total = 0
source, revision = self._state
if not revision:
--- a/mercurial/tags.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/tags.py Tue Feb 28 11:13:25 2017 -0800
@@ -14,7 +14,6 @@
import array
import errno
-import time
from .node import (
bin,
@@ -25,6 +24,7 @@
from . import (
encoding,
error,
+ scmutil,
util,
)
@@ -278,8 +278,6 @@
If the cache is not up to date, the caller is responsible for reading tag
info from each returned head. (See findglobaltags().)
'''
- from . import scmutil # avoid cycle
-
try:
cachefile = repo.vfs(_filename(repo), 'r')
# force reading the file for static-http
@@ -344,7 +342,7 @@
# potentially expensive search.
return ([], {}, valid, None, True)
- starttime = time.time()
+ starttime = util.timer()
# Now we have to lookup the .hgtags filenode for every new head.
# This is the most expensive part of finding tags, so performance
@@ -359,7 +357,7 @@
fnodescache.write()
- duration = time.time() - starttime
+ duration = util.timer() - starttime
ui.log('tagscache',
'%d/%d cache hits/lookups in %0.4f '
'seconds\n',
--- a/mercurial/templater.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/templater.py Tue Feb 28 11:13:25 2017 -0800
@@ -20,6 +20,7 @@
pycompat,
registrar,
revset as revsetmod,
+ revsetlang,
templatefilters,
templatekw,
util,
@@ -778,7 +779,7 @@
if len(args) > 1:
formatargs = [evalfuncarg(context, mapping, a) for a in args[1:]]
- revs = query(revsetmod.formatspec(raw, *formatargs))
+ revs = query(revsetlang.formatspec(raw, *formatargs))
revs = list(revs)
else:
revsetcache = mapping['cache'].setdefault("revsetcache", {})
--- a/mercurial/templates/gitweb/filelog.tmpl Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/templates/gitweb/filelog.tmpl Tue Feb 28 11:13:25 2017 -0800
@@ -38,6 +38,8 @@
</table>
<div class="page_nav">
+<a href="{url|urlescape}log/{symrev}/{file|urlescape}{lessvars%urlparameter}">less</a>
+<a href="{url|urlescape}log/{symrev}/{file|urlescape}{morevars%urlparameter}">more</a>
{nav%filenav}
</div>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/mercurial/txnutil.py Tue Feb 28 11:13:25 2017 -0800
@@ -0,0 +1,36 @@
+# txnutil.py - transaction related utilities
+#
+# Copyright FUJIWARA Katsunori <foozy@lares.dti.ne.jp> and others
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+from __future__ import absolute_import
+
+import errno
+
+from . import (
+ encoding,
+)
+
+def mayhavepending(root):
+ '''return whether 'root' may have pending changes, which are
+ visible to this process.
+ '''
+ return root == encoding.environ.get('HG_PENDING')
+
+def trypending(root, vfs, filename, **kwargs):
+ '''Open file to be read according to HG_PENDING environment variable
+
+ This opens '.pending' of specified 'filename' only when HG_PENDING
+ is equal to 'root'.
+
+ This returns '(fp, is_pending_opened)' tuple.
+ '''
+ if mayhavepending(root):
+ try:
+ return (vfs('%s.pending' % filename, **kwargs), True)
+ except IOError as inst:
+ if inst.errno != errno.ENOENT:
+ raise
+ return (vfs(filename, **kwargs), False)
--- a/mercurial/ui.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/ui.py Tue Feb 28 11:13:25 2017 -0800
@@ -7,13 +7,17 @@
from __future__ import absolute_import
+import atexit
+import collections
import contextlib
import errno
import getpass
import inspect
import os
import re
+import signal
import socket
+import subprocess
import sys
import tempfile
import traceback
@@ -22,6 +26,7 @@
from .node import hex
from . import (
+ color,
config,
encoding,
error,
@@ -34,6 +39,14 @@
urlreq = util.urlreq
+# for use with str.translate(None, _keepalnum), to keep just alphanumerics
+if pycompat.ispy3:
+ _bytes = [bytes([c]) for c in range(256)]
+ _notalnum = [s for s in _bytes if not s.isalnum()]
+else:
+ _notalnum = [c for c in map(chr, range(256)) if not c.isalnum()]
+_keepalnum = ''.join(_notalnum)
+
samplehgrcs = {
'user':
"""# example user config (see 'hg help config' for more info)
@@ -94,6 +107,26 @@
# pager =""",
}
+
+class httppasswordmgrdbproxy(object):
+ """Delays loading urllib2 until it's needed."""
+ def __init__(self):
+ self._mgr = None
+
+ def _get_mgr(self):
+ if self._mgr is None:
+ self._mgr = urlreq.httppasswordmgrwithdefaultrealm()
+ return self._mgr
+
+ def add_password(self, *args, **kwargs):
+ return self._get_mgr().add_password(*args, **kwargs)
+
+ def find_user_password(self, *args, **kwargs):
+ return self._get_mgr().find_user_password(*args, **kwargs)
+
+def _catchterm(*args):
+ raise error.SignalInterrupt
+
class ui(object):
def __init__(self, src=None):
"""Create a fresh new ui object if no src given
@@ -120,11 +153,19 @@
self.callhooks = True
# Insecure server connections requested.
self.insecureconnections = False
+ # Blocked time
+ self.logblockedtimes = False
+ # color mode: see mercurial/color.py for possible value
+ self._colormode = None
+ self._terminfoparams = {}
+ self._styles = {}
if src:
self.fout = src.fout
self.ferr = src.ferr
self.fin = src.fin
+ self.pageractive = src.pageractive
+ self._disablepager = src._disablepager
self._tcfg = src._tcfg.copy()
self._ucfg = src._ucfg.copy()
@@ -134,18 +175,26 @@
self.environ = src.environ
self.callhooks = src.callhooks
self.insecureconnections = src.insecureconnections
+ self._colormode = src._colormode
+ self._terminfoparams = src._terminfoparams.copy()
+ self._styles = src._styles.copy()
+
self.fixconfig()
self.httppasswordmgrdb = src.httppasswordmgrdb
+ self._blockedtimes = src._blockedtimes
else:
self.fout = util.stdout
self.ferr = util.stderr
self.fin = util.stdin
+ self.pageractive = False
+ self._disablepager = False
# shared read-only environment
self.environ = encoding.environ
- self.httppasswordmgrdb = urlreq.httppasswordmgrwithdefaultrealm()
+ self.httppasswordmgrdb = httppasswordmgrdbproxy()
+ self._blockedtimes = collections.defaultdict(int)
allowed = self.configlist('experimental', 'exportableenviron')
if '*' in allowed:
@@ -172,7 +221,17 @@
"""Clear internal state that shouldn't persist across commands"""
if self._progbar:
self._progbar.resetstate() # reset last-print time of progress bar
- self.httppasswordmgrdb = urlreq.httppasswordmgrwithdefaultrealm()
+ self.httppasswordmgrdb = httppasswordmgrdbproxy()
+
+ @contextlib.contextmanager
+ def timeblockedsection(self, key):
+ # this is open-coded below - search for timeblockedsection to find them
+ starttime = util.timer()
+ try:
+ yield
+ finally:
+ self._blockedtimes[key + '_blocked'] += \
+ (util.timer() - starttime) * 1000
def formatter(self, topic, opts):
return formatter.formatter(self, topic, opts)
@@ -277,6 +336,7 @@
self._reportuntrusted = self.debugflag or self.configbool("ui",
"report_untrusted", True)
self.tracebackflag = self.configbool('ui', 'traceback', False)
+ self.logblockedtimes = self.configbool('ui', 'logblockedtimes')
if section in (None, 'trusted'):
# update trust information
@@ -402,6 +462,41 @@
% (section, name, v))
return b
+ def configwith(self, convert, section, name, default=None,
+ desc=None, untrusted=False):
+ """parse a configuration element with a conversion function
+
+ >>> u = ui(); s = 'foo'
+ >>> u.setconfig(s, 'float1', '42')
+ >>> u.configwith(float, s, 'float1')
+ 42.0
+ >>> u.setconfig(s, 'float2', '-4.25')
+ >>> u.configwith(float, s, 'float2')
+ -4.25
+ >>> u.configwith(float, s, 'unknown', 7)
+ 7
+ >>> u.setconfig(s, 'invalid', 'somevalue')
+ >>> u.configwith(float, s, 'invalid')
+ Traceback (most recent call last):
+ ...
+ ConfigError: foo.invalid is not a valid float ('somevalue')
+ >>> u.configwith(float, s, 'invalid', desc='womble')
+ Traceback (most recent call last):
+ ...
+ ConfigError: foo.invalid is not a valid womble ('somevalue')
+ """
+
+ v = self.config(section, name, None, untrusted)
+ if v is None:
+ return default
+ try:
+ return convert(v)
+ except ValueError:
+ if desc is None:
+ desc = convert.__name__
+ raise error.ConfigError(_("%s.%s is not a valid %s ('%s')")
+ % (section, name, desc, v))
+
def configint(self, section, name, default=None, untrusted=False):
"""parse a configuration element as an integer
@@ -418,17 +513,11 @@
>>> u.configint(s, 'invalid')
Traceback (most recent call last):
...
- ConfigError: foo.invalid is not an integer ('somevalue')
+ ConfigError: foo.invalid is not a valid integer ('somevalue')
"""
- v = self.config(section, name, None, untrusted)
- if v is None:
- return default
- try:
- return int(v)
- except ValueError:
- raise error.ConfigError(_("%s.%s is not an integer ('%s')")
- % (section, name, v))
+ return self.configwith(int, section, name, default, 'integer',
+ untrusted)
def configbytes(self, section, name, default=0, untrusted=False):
"""parse a configuration element as a quantity in bytes
@@ -696,55 +785,176 @@
def write(self, *args, **opts):
'''write args to output
- By default, this method simply writes to the buffer or stdout,
- but extensions or GUI tools may override this method,
- write_err(), popbuffer(), and label() to style output from
- various parts of hg.
+ By default, this method simply writes to the buffer or stdout.
+ Color mode can be set on the UI class to have the output decorated
+ with color modifier before being written to stdout.
- An optional keyword argument, "label", can be passed in.
- This should be a string containing label names separated by
- space. Label names take the form of "topic.type". For example,
- ui.debug() issues a label of "ui.debug".
+ The color used is controlled by an optional keyword argument, "label".
+ This should be a string containing label names separated by space.
+ Label names take the form of "topic.type". For example, ui.debug()
+ issues a label of "ui.debug".
When labeling output for a specific command, a label of
"cmdname.type" is recommended. For example, status issues
a label of "status.modified" for modified files.
'''
if self._buffers and not opts.get('prompt', False):
- self._buffers[-1].extend(a for a in args)
+ if self._bufferapplylabels:
+ label = opts.get('label', '')
+ self._buffers[-1].extend(self.label(a, label) for a in args)
+ else:
+ self._buffers[-1].extend(args)
+ elif self._colormode == 'win32':
+ # windows color printing is its own can of crab, defer to
+ # the color module and that is it.
+ color.win32print(self, self._write, *args, **opts)
else:
+ msgs = args
+ if self._colormode is not None:
+ label = opts.get('label', '')
+ msgs = [self.label(a, label) for a in args]
+ self._write(*msgs, **opts)
+
+ def _write(self, *msgs, **opts):
self._progclear()
- for a in args:
- self.fout.write(a)
+ # opencode timeblockedsection because this is a critical path
+ starttime = util.timer()
+ try:
+ for a in msgs:
+ self.fout.write(a)
+ finally:
+ self._blockedtimes['stdio_blocked'] += \
+ (util.timer() - starttime) * 1000
def write_err(self, *args, **opts):
self._progclear()
+ if self._bufferstates and self._bufferstates[-1][0]:
+ self.write(*args, **opts)
+ elif self._colormode == 'win32':
+ # windows color printing is its own can of crab, defer to
+ # the color module and that is it.
+ color.win32print(self, self._write_err, *args, **opts)
+ else:
+ msgs = args
+ if self._colormode is not None:
+ label = opts.get('label', '')
+ msgs = [self.label(a, label) for a in args]
+ self._write_err(*msgs, **opts)
+
+ def _write_err(self, *msgs, **opts):
try:
- if self._bufferstates and self._bufferstates[-1][0]:
- return self.write(*args, **opts)
- if not getattr(self.fout, 'closed', False):
- self.fout.flush()
- for a in args:
- self.ferr.write(a)
- # stderr may be buffered under win32 when redirected to files,
- # including stdout.
- if not getattr(self.ferr, 'closed', False):
- self.ferr.flush()
+ with self.timeblockedsection('stdio'):
+ if not getattr(self.fout, 'closed', False):
+ self.fout.flush()
+ for a in msgs:
+ self.ferr.write(a)
+ # stderr may be buffered under win32 when redirected to files,
+ # including stdout.
+ if not getattr(self.ferr, 'closed', False):
+ self.ferr.flush()
except IOError as inst:
if inst.errno not in (errno.EPIPE, errno.EIO, errno.EBADF):
raise
def flush(self):
- try: self.fout.flush()
- except (IOError, ValueError): pass
- try: self.ferr.flush()
- except (IOError, ValueError): pass
+ # opencode timeblockedsection because this is a critical path
+ starttime = util.timer()
+ try:
+ try: self.fout.flush()
+ except (IOError, ValueError): pass
+ try: self.ferr.flush()
+ except (IOError, ValueError): pass
+ finally:
+ self._blockedtimes['stdio_blocked'] += \
+ (util.timer() - starttime) * 1000
def _isatty(self, fh):
if self.configbool('ui', 'nontty', False):
return False
return util.isatty(fh)
+ def disablepager(self):
+ self._disablepager = True
+
+ def pager(self, command):
+ """Start a pager for subsequent command output.
+
+ Commands which produce a long stream of output should call
+ this function to activate the user's preferred pagination
+ mechanism (which may be no pager). Calling this function
+ precludes any future use of interactive functionality, such as
+ prompting the user or activating curses.
+
+ Args:
+ command: The full, non-aliased name of the command. That is, "log"
+ not "history, "summary" not "summ", etc.
+ """
+ if (self._disablepager
+ or self.pageractive
+ or command in self.configlist('pager', 'ignore')
+ or not self.configbool('pager', 'enable', True)
+ or not self.configbool('pager', 'attend-' + command, True)
+ # TODO: if we want to allow HGPLAINEXCEPT=pager,
+ # formatted() will need some adjustment.
+ or not self.formatted()
+ or self.plain()
+ # TODO: expose debugger-enabled on the UI object
+ or '--debugger' in sys.argv):
+ # We only want to paginate if the ui appears to be
+ # interactive, the user didn't say HGPLAIN or
+ # HGPLAINEXCEPT=pager, and the user didn't specify --debug.
+ return
+
+ # TODO: add a "system defaults" config section so this default
+ # of more(1) can be easily replaced with a global
+ # configuration file. For example, on OS X the sane default is
+ # less(1), not more(1), and on debian it's
+ # sensible-pager(1). We should probably also give the system
+ # default editor command similar treatment.
+ envpager = encoding.environ.get('PAGER', 'more')
+ pagercmd = self.config('pager', 'pager', envpager)
+ if not pagercmd:
+ return
+
+ self.debug('starting pager for command %r\n' % command)
+ self.pageractive = True
+ # Preserve the formatted-ness of the UI. This is important
+ # because we mess with stdout, which might confuse
+ # auto-detection of things being formatted.
+ self.setconfig('ui', 'formatted', self.formatted(), 'pager')
+ self.setconfig('ui', 'interactive', False, 'pager')
+ if util.safehasattr(signal, "SIGPIPE"):
+ signal.signal(signal.SIGPIPE, _catchterm)
+ self._runpager(pagercmd)
+
+ def _runpager(self, command):
+ """Actually start the pager and set up file descriptors.
+
+ This is separate in part so that extensions (like chg) can
+ override how a pager is invoked.
+ """
+ pager = subprocess.Popen(command, shell=True, bufsize=-1,
+ close_fds=util.closefds, stdin=subprocess.PIPE,
+ stdout=util.stdout, stderr=util.stderr)
+
+ # back up original file descriptors
+ stdoutfd = os.dup(util.stdout.fileno())
+ stderrfd = os.dup(util.stderr.fileno())
+
+ os.dup2(pager.stdin.fileno(), util.stdout.fileno())
+ if self._isatty(util.stderr):
+ os.dup2(pager.stdin.fileno(), util.stderr.fileno())
+
+ @atexit.register
+ def killpager():
+ if util.safehasattr(signal, "SIGINT"):
+ signal.signal(signal.SIGINT, signal.SIG_IGN)
+ # restore original fds, closing pager.stdin copies in the process
+ os.dup2(stdoutfd, util.stdout.fileno())
+ os.dup2(stderrfd, util.stderr.fileno())
+ pager.stdin.close()
+ pager.wait()
+
def interface(self, feature):
"""what interface to use for interactive console features?
@@ -900,7 +1110,8 @@
sys.stdout = self.fout
# prompt ' ' must exist; otherwise readline may delete entire line
# - http://bugs.python.org/issue12833
- line = raw_input(' ')
+ with self.timeblockedsection('stdio'):
+ line = raw_input(' ')
sys.stdin = oldin
sys.stdout = oldout
@@ -980,13 +1191,14 @@
self.write_err(self.label(prompt or _('password: '), 'ui.prompt'))
# disable getpass() only if explicitly specified. it's still valid
# to interact with tty even if fin is not a tty.
- if self.configbool('ui', 'nontty'):
- l = self.fin.readline()
- if not l:
- raise EOFError
- return l.rstrip('\n')
- else:
- return getpass.getpass('')
+ with self.timeblockedsection('stdio'):
+ if self.configbool('ui', 'nontty'):
+ l = self.fin.readline()
+ if not l:
+ raise EOFError
+ return l.rstrip('\n')
+ else:
+ return getpass.getpass('')
except EOFError:
raise error.ResponseExpected()
def status(self, *msg, **opts):
@@ -1038,7 +1250,7 @@
suffix=extra['suffix'], text=True,
dir=rdir)
try:
- f = os.fdopen(fd, "w")
+ f = os.fdopen(fd, pycompat.sysstr("w"))
f.write(text)
f.close()
@@ -1058,7 +1270,8 @@
self.system("%s \"%s\"" % (editor, name),
environ=environ,
- onerr=error.Abort, errprefix=_("edit failed"))
+ onerr=error.Abort, errprefix=_("edit failed"),
+ blockedtag='editor')
f = open(name)
t = f.read()
@@ -1068,15 +1281,33 @@
return t
- def system(self, cmd, environ=None, cwd=None, onerr=None, errprefix=None):
+ def system(self, cmd, environ=None, cwd=None, onerr=None, errprefix=None,
+ blockedtag=None):
'''execute shell command with appropriate output stream. command
output will be redirected if fout is not stdout.
+
+ if command fails and onerr is None, return status, else raise onerr
+ object as exception.
'''
+ if blockedtag is None:
+ blockedtag = 'unknown_system_' + cmd.translate(None, _keepalnum)
out = self.fout
if any(s[1] for s in self._bufferstates):
out = self
- return util.system(cmd, environ=environ, cwd=cwd, onerr=onerr,
- errprefix=errprefix, out=out)
+ with self.timeblockedsection(blockedtag):
+ rc = self._runsystem(cmd, environ=environ, cwd=cwd, out=out)
+ if rc and onerr:
+ errmsg = '%s %s' % (os.path.basename(cmd.split(None, 1)[0]),
+ util.explainexit(rc)[0])
+ if errprefix:
+ errmsg = '%s: %s' % (errprefix, errmsg)
+ raise onerr(errmsg)
+ return rc
+
+ def _runsystem(self, cmd, environ, cwd, out):
+ """actually execute the given shell command (can be overridden by
+ extensions like chg)"""
+ return util.system(cmd, environ=environ, cwd=cwd, out=out)
def traceback(self, exc=None, force=False):
'''print exception traceback if traceback printing enabled or forced.
@@ -1180,13 +1411,15 @@
def label(self, msg, label):
'''style msg based on supplied label
- Like ui.write(), this just returns msg unchanged, but extensions
- and GUI tools can override it to allow styling output without
- writing it.
+ If some color mode is enabled, this will add the necessary control
+ characters to apply such color. In addition, 'debug' color mode adds
+ markup showing which label affects a piece of text.
ui.write(s, 'label') is equivalent to
ui.write(ui.label(s, 'label')).
'''
+ if self._colormode is not None:
+ return color.colorlabel(self, msg, label)
return msg
def develwarn(self, msg, stacklevel=1, config=None):
--- a/mercurial/util.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/util.py Tue Feb 28 11:13:25 2017 -0800
@@ -63,9 +63,21 @@
urlreq = pycompat.urlreq
xmlrpclib = pycompat.xmlrpclib
+def isatty(fp):
+ try:
+ return fp.isatty()
+ except AttributeError:
+ return False
+
+# glibc determines buffering on first write to stdout - if we replace a TTY
+# destined stdout with a pipe destined stdout (e.g. pager), we want line
+# buffering
+if isatty(stdout):
+ stdout = os.fdopen(stdout.fileno(), pycompat.sysstr('wb'), 1)
+
if pycompat.osname == 'nt':
from . import windows as platform
- stdout = platform.winstdout(pycompat.stdout)
+ stdout = platform.winstdout(stdout)
else:
from . import posix as platform
@@ -797,7 +809,7 @@
inname, outname = None, None
try:
infd, inname = tempfile.mkstemp(prefix='hg-filter-in-')
- fp = os.fdopen(infd, 'wb')
+ fp = os.fdopen(infd, pycompat.sysstr('wb'))
fp.write(s)
fp.close()
outfd, outname = tempfile.mkstemp(prefix='hg-filter-out-')
@@ -943,10 +955,7 @@
# executable version (py2exe) doesn't support __file__
datapath = os.path.dirname(pycompat.sysexecutable)
else:
- datapath = os.path.dirname(__file__)
-
-if not isinstance(datapath, bytes):
- datapath = pycompat.fsencode(datapath)
+ datapath = os.path.dirname(pycompat.fsencode(__file__))
i18n.setdatapath(datapath)
@@ -968,8 +977,9 @@
_sethgexecutable(encoding.environ['EXECUTABLEPATH'])
else:
_sethgexecutable(pycompat.sysexecutable)
- elif os.path.basename(getattr(mainmod, '__file__', '')) == 'hg':
- _sethgexecutable(mainmod.__file__)
+ elif (os.path.basename(
+ pycompat.fsencode(getattr(mainmod, '__file__', ''))) == 'hg'):
+ _sethgexecutable(pycompat.fsencode(mainmod.__file__))
else:
exe = findexe('hg') or os.path.basename(sys.argv[0])
_sethgexecutable(exe)
@@ -999,20 +1009,16 @@
env['HG'] = hgexecutable()
return env
-def system(cmd, environ=None, cwd=None, onerr=None, errprefix=None, out=None):
+def system(cmd, environ=None, cwd=None, out=None):
'''enhanced shell command execution.
run with environment maybe modified, maybe in different dir.
- if command fails and onerr is None, return status, else raise onerr
- object as exception.
-
if out is specified, it is assumed to be a file-like object that has a
write() method. stdout and stderr will be redirected to out.'''
try:
stdout.flush()
except Exception:
pass
- origcmd = cmd
cmd = quotecommand(cmd)
if pycompat.sysplatform == 'plan9' and (sys.version_info[0] == 2
and sys.version_info[1] < 7):
@@ -1036,12 +1042,6 @@
rc = proc.returncode
if pycompat.sysplatform == 'OpenVMS' and rc & 1:
rc = 0
- if rc and onerr:
- errmsg = '%s %s' % (os.path.basename(origcmd.split(None, 1)[0]),
- explainexit(rc)[0])
- if errprefix:
- errmsg = '%s: %s' % (errprefix, errmsg)
- raise onerr(errmsg)
return rc
def checksignature(func):
@@ -1191,8 +1191,13 @@
if pycompat.osname == 'nt':
checkosfilename = checkwinfilename
+ timer = time.clock
else:
checkosfilename = platform.checkosfilename
+ timer = time.time
+
+if safehasattr(time, "perf_counter"):
+ timer = time.perf_counter
def makelock(info, pathname):
try:
@@ -2750,12 +2755,6 @@
u.user = u.passwd = None
return str(u)
-def isatty(fp):
- try:
- return fp.isatty()
- except AttributeError:
- return False
-
timecount = unitcountfn(
(1, 1e3, _('%.0f s')),
(100, 1, _('%.1f s')),
@@ -2786,13 +2785,13 @@
'''
def wrapper(*args, **kwargs):
- start = time.time()
+ start = timer()
indent = 2
_timenesting[0] += indent
try:
return func(*args, **kwargs)
finally:
- elapsed = time.time() - start
+ elapsed = timer() - start
_timenesting[0] -= indent
stderr.write('%s%s: %s\n' %
(' ' * _timenesting[0], func.__name__,
--- a/mercurial/verify.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/verify.py Tue Feb 28 11:13:25 2017 -0800
@@ -18,6 +18,7 @@
from . import (
error,
revlog,
+ scmutil,
util,
)
@@ -32,21 +33,13 @@
f = f.replace('//', '/')
return f
-def _validpath(repo, path):
- """Returns False if a path should NOT be treated as part of a repo.
-
- For all in-core cases, this returns True, as we have no way for a
- path to be mentioned in the history but not actually be
- relevant. For narrow clones, this is important because many
- filelogs will be missing, and changelog entries may mention
- modified files that are outside the narrow scope.
- """
- return True
-
class verifier(object):
- def __init__(self, repo):
+ # The match argument is always None in hg core, but e.g. the narrowhg
+ # extension will pass in a matcher here.
+ def __init__(self, repo, match=None):
self.repo = repo.unfiltered()
self.ui = repo.ui
+ self.match = match or scmutil.matchall(repo)
self.badrevs = set()
self.errors = 0
self.warnings = 0
@@ -170,6 +163,7 @@
def _verifychangelog(self):
ui = self.ui
repo = self.repo
+ match = self.match
cl = repo.changelog
ui.status(_("checking changesets\n"))
@@ -189,7 +183,7 @@
mflinkrevs.setdefault(changes[0], []).append(i)
self.refersmf = True
for f in changes[3]:
- if _validpath(repo, f):
+ if match(f):
filelinkrevs.setdefault(_normpath(f), []).append(i)
except Exception as inst:
self.refersmf = True
@@ -201,6 +195,7 @@
progress=None):
repo = self.repo
ui = self.ui
+ match = self.match
mfl = self.repo.manifestlog
mf = mfl._revlog.dirlog(dir)
@@ -243,12 +238,14 @@
elif f == "/dev/null": # ignore this in very old repos
continue
fullpath = dir + _normpath(f)
- if not _validpath(repo, fullpath):
- continue
if fl == 't':
+ if not match.visitdir(fullpath):
+ continue
subdirnodes.setdefault(fullpath + '/', {}).setdefault(
fn, []).append(lr)
else:
+ if not match(fullpath):
+ continue
filenodes.setdefault(fullpath, {}).setdefault(fn, lr)
except Exception as inst:
self.exc(lr, _("reading delta %s") % short(n), inst, label)
--- a/mercurial/wireproto.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/wireproto.py Tue Feb 28 11:13:25 2017 -0800
@@ -26,6 +26,7 @@
exchange,
peer,
pushkey as pushkeymod,
+ pycompat,
streamclone,
util,
)
@@ -839,7 +840,6 @@
raise error.Abort(bundle2requiredmain,
hint=bundle2requiredhint)
- #chunks = exchange.getbundlechunks(repo, 'serve', **opts)
try:
chunks = exchange.getbundlechunks(repo, 'serve', **opts)
except error.Abort as exc:
@@ -961,7 +961,7 @@
# write bundle data to temporary file because it can be big
fd, tempname = tempfile.mkstemp(prefix='hg-unbundle-')
- fp = os.fdopen(fd, 'wb+')
+ fp = os.fdopen(fd, pycompat.sysstr('wb+'))
r = 0
try:
proto.getfile(fp)
--- a/mercurial/worker.py Sat Feb 25 12:48:50 2017 +0900
+++ b/mercurial/worker.py Tue Feb 28 11:13:25 2017 -0800
@@ -164,7 +164,7 @@
os._exit(0)
pids.add(pid)
os.close(wfd)
- fp = os.fdopen(rfd, 'rb', 0)
+ fp = os.fdopen(rfd, pycompat.sysstr('rb'), 0)
def cleanup():
signal.signal(signal.SIGINT, oldhandler)
waitforworkers()
--- a/tests/dumbhttp.py Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/dumbhttp.py Tue Feb 28 11:13:25 2017 -0800
@@ -7,7 +7,9 @@
"""
import optparse
+import os
import signal
+import socket
import sys
from mercurial import (
@@ -18,11 +20,17 @@
httpserver = util.httpserver
OptionParser = optparse.OptionParser
+if os.environ.get('HGIPV6', '0') == '1':
+ class simplehttpserver(httpserver.httpserver):
+ address_family = socket.AF_INET6
+else:
+ simplehttpserver = httpserver.httpserver
+
class simplehttpservice(object):
def __init__(self, host, port):
self.address = (host, port)
def init(self):
- self.httpd = httpserver.httpserver(
+ self.httpd = simplehttpserver(
self.address, httpserver.simplehttprequesthandler)
def run(self):
self.httpd.serve_forever()
--- a/tests/dummyssh Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/dummyssh Tue Feb 28 11:13:25 2017 -0800
@@ -10,7 +10,7 @@
if sys.argv[1] != "user@dummy":
sys.exit(-1)
-os.environ["SSH_CLIENT"] = "127.0.0.1 1 2"
+os.environ["SSH_CLIENT"] = "%s 1 2" % os.environ.get('LOCALIP', '127.0.0.1')
log = open("dummylog", "ab")
log.write("Got arguments")
--- a/tests/run-tests.py Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/run-tests.py Tue Feb 28 11:13:25 2017 -0800
@@ -112,18 +112,51 @@
# For Windows support
wifexited = getattr(os, "WIFEXITED", lambda x: False)
-def checkportisavailable(port):
- """return true if a port seems free to bind on localhost"""
+# Whether to use IPv6
+def checksocketfamily(name, port=20058):
+ """return true if we can listen on localhost using family=name
+
+ name should be either 'AF_INET', or 'AF_INET6'.
+ port being used is okay - EADDRINUSE is considered as successful.
+ """
+ family = getattr(socket, name, None)
+ if family is None:
+ return False
try:
- s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ s = socket.socket(family, socket.SOCK_STREAM)
s.bind(('localhost', port))
s.close()
return True
except socket.error as exc:
- if not exc.errno == errno.EADDRINUSE:
+ if exc.errno == errno.EADDRINUSE:
+ return True
+ elif exc.errno in (errno.EADDRNOTAVAIL, errno.EPROTONOSUPPORT):
+ return False
+ else:
raise
+ else:
return False
+# useipv6 will be set by parseargs
+useipv6 = None
+
+def checkportisavailable(port):
+ """return true if a port seems free to bind on localhost"""
+ if useipv6:
+ family = socket.AF_INET6
+ else:
+ family = socket.AF_INET
+ try:
+ s = socket.socket(family, socket.SOCK_STREAM)
+ s.bind(('localhost', port))
+ s.close()
+ return True
+ except socket.error as exc:
+ if exc.errno not in (errno.EADDRINUSE, errno.EADDRNOTAVAIL,
+ errno.EPROTONOSUPPORT):
+ raise
+ return False
+
closefds = os.name == 'posix'
def Popen4(cmd, wd, timeout, env=None):
processlock.acquire()
@@ -269,6 +302,8 @@
help="install and use chg wrapper in place of hg")
parser.add_option("--with-chg", metavar="CHG",
help="use specified chg wrapper in place of hg")
+ parser.add_option("--ipv6", action="store_true",
+ help="prefer IPv6 to IPv4 for network related tests")
parser.add_option("-3", "--py3k-warnings", action="store_true",
help="enable Py3k warnings on Python 2.6+")
# This option should be deleted once test-check-py3-compat.t and other
@@ -338,6 +373,14 @@
parser.error('--chg does not work when --with-hg is specified '
'(use --with-chg instead)')
+ global useipv6
+ if options.ipv6:
+ useipv6 = checksocketfamily('AF_INET6')
+ else:
+ # only use IPv6 if IPv4 is unavailable and IPv6 is available
+ useipv6 = ((not checksocketfamily('AF_INET'))
+ and checksocketfamily('AF_INET6'))
+
options.anycoverage = options.cover or options.annotate or options.htmlcov
if options.anycoverage:
try:
@@ -506,7 +549,8 @@
timeout=defaults['timeout'],
startport=defaults['port'], extraconfigopts=None,
py3kwarnings=False, shell=None, hgcommand=None,
- slowtimeout=defaults['slowtimeout'], usechg=False):
+ slowtimeout=defaults['slowtimeout'], usechg=False,
+ useipv6=False):
"""Create a test from parameters.
path is the full path to the file defining the test.
@@ -554,6 +598,7 @@
self._shell = _bytespath(shell)
self._hgcommand = hgcommand or b'hg'
self._usechg = usechg
+ self._useipv6 = useipv6
self._aborted = False
self._daemonpids = []
@@ -802,6 +847,7 @@
self._portmap(2),
(br'(?m)^(saved backup bundle to .*\.hg)( \(glob\))?$',
br'\1 (glob)'),
+ (br'([^0-9])%s' % re.escape(self._localip()), br'\1$LOCALIP'),
]
r.append((self._escapepath(self._testtmp), b'$TESTTMP'))
@@ -817,6 +863,12 @@
else:
return re.escape(p)
+ def _localip(self):
+ if self._useipv6:
+ return b'::1'
+ else:
+ return b'127.0.0.1'
+
def _getenv(self):
"""Obtain environment variables to use during test execution."""
def defineport(i):
@@ -839,6 +891,11 @@
env["HGUSER"] = "test"
env["HGENCODING"] = "ascii"
env["HGENCODINGMODE"] = "strict"
+ env['HGIPV6'] = str(int(self._useipv6))
+
+ # LOCALIP could be ::1 or 127.0.0.1. Useful for tests that require raw
+ # IP addresses.
+ env['LOCALIP'] = self._localip()
# Reset some environment variables to well-known values so that
# the tests produce repeatable output.
@@ -849,6 +906,7 @@
env['TERM'] = 'xterm'
for k in ('HG HGPROF CDPATH GREP_OPTIONS http_proxy no_proxy ' +
+ 'HGPLAIN HGPLAINEXCEPT ' +
'NO_PROXY CHGDEBUG').split():
if k in env:
del env[k]
@@ -881,6 +939,9 @@
hgrc.write(b'[largefiles]\n')
hgrc.write(b'usercache = %s\n' %
(os.path.join(self._testtmp, b'.cache/largefiles')))
+ hgrc.write(b'[web]\n')
+ hgrc.write(b'address = localhost\n')
+ hgrc.write(b'ipv6 = %s\n' % self._useipv6)
for opt in self._extraconfigopts:
section, key = opt.split('.', 1)
@@ -2288,7 +2349,8 @@
py3kwarnings=self.options.py3k_warnings,
shell=self.options.shell,
hgcommand=self._hgcommand,
- usechg=bool(self.options.with_chg or self.options.chg))
+ usechg=bool(self.options.with_chg or self.options.chg),
+ useipv6=useipv6)
t.should_reload = True
return t
--- a/tests/test-archive.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-archive.t Tue Feb 28 11:13:25 2017 -0800
@@ -99,7 +99,7 @@
> except AttributeError:
> stdout = sys.stdout
> try:
- > f = util.urlreq.urlopen('http://127.0.0.1:%s/?%s'
+ > f = util.urlreq.urlopen('http://$LOCALIP:%s/?%s'
> % (os.environ['HGPORT'], requeststr))
> stdout.write(f.read())
> except util.urlerr.httperror as e:
--- a/tests/test-basic.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-basic.t Tue Feb 28 11:13:25 2017 -0800
@@ -11,6 +11,8 @@
ui.interactive=False
ui.mergemarkers=detailed
ui.promptecho=True
+ web.address=localhost
+ web\.ipv6=(?:True|False) (re)
$ hg init t
$ cd t
--- a/tests/test-bdiff.py Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-bdiff.py Tue Feb 28 11:13:25 2017 -0800
@@ -3,8 +3,6 @@
import struct
import unittest
-import silenttestrunner
-
from mercurial import (
bdiff,
mpatch,
@@ -148,4 +146,5 @@
['a\n', diffreplace(2, 10, 'a\na\na\na\n', '')])
if __name__ == '__main__':
+ import silenttestrunner
silenttestrunner.main(__name__)
--- a/tests/test-bookmarks.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-bookmarks.t Tue Feb 28 11:13:25 2017 -0800
@@ -1,4 +1,5 @@
- $ hg init
+ $ hg init repo
+ $ cd repo
no bookmarks
@@ -630,7 +631,7 @@
Z 2:db815d6d32e6
x y 2:db815d6d32e6
$ hg -R ../cloned-bookmarks-manual-update-with-divergence pull
- pulling from $TESTTMP
+ pulling from $TESTTMP/repo (glob)
searching for changes
adding changesets
adding manifests
@@ -895,3 +896,58 @@
$ touch $TESTTMP/unpause
$ cd ..
+
+check whether HG_PENDING makes pending changes only in related
+repositories visible to an external hook.
+
+(emulate a transaction running concurrently by copied
+.hg/bookmarks.pending in subsequent test)
+
+ $ cat > $TESTTMP/savepending.sh <<EOF
+ > cp .hg/bookmarks.pending .hg/bookmarks.pending.saved
+ > exit 1 # to avoid adding new bookmark for subsequent tests
+ > EOF
+
+ $ hg init unrelated
+ $ cd unrelated
+ $ echo a > a
+ $ hg add a
+ $ hg commit -m '#0'
+ $ hg --config hooks.pretxnclose="sh $TESTTMP/savepending.sh" bookmarks INVISIBLE
+ transaction abort!
+ rollback completed
+ abort: pretxnclose hook exited with status 1
+ [255]
+ $ cp .hg/bookmarks.pending.saved .hg/bookmarks.pending
+
+(check visible bookmarks while transaction running in repo)
+
+ $ cat > $TESTTMP/checkpending.sh <<EOF
+ > echo "@repo"
+ > hg -R $TESTTMP/repo bookmarks
+ > echo "@unrelated"
+ > hg -R $TESTTMP/unrelated bookmarks
+ > exit 1 # to avoid adding new bookmark for subsequent tests
+ > EOF
+
+ $ cd ../repo
+ $ hg --config hooks.pretxnclose="sh $TESTTMP/checkpending.sh" bookmarks NEW
+ @repo
+ * NEW 6:81dcce76aa0b
+ X2 1:925d80f479bb
+ Y 4:125c9a1d6df6
+ Z 5:5fb12f0f2d51
+ Z@1 1:925d80f479bb
+ Z@2 4:125c9a1d6df6
+ foo 3:9ba5f110a0b3
+ foo@1 0:f7b1eb17ad24
+ foo@2 2:db815d6d32e6
+ four 3:9ba5f110a0b3
+ should-end-on-two 2:db815d6d32e6
+ x y 2:db815d6d32e6
+ @unrelated
+ no bookmarks set
+ transaction abort!
+ rollback completed
+ abort: pretxnclose hook exited with status 1
+ [255]
--- a/tests/test-bundle2-exchange.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-bundle2-exchange.t Tue Feb 28 11:13:25 2017 -0800
@@ -340,7 +340,7 @@
remote: lock: free
remote: wlock: free
remote: postclose-tip:5fddd98957c8 draft book_5fdd
- remote: txnclose hook: HG_BOOKMARK_MOVED=1 HG_BUNDLE2=1 HG_NEW_OBSMARKERS=1 HG_NODE=5fddd98957c8a54a4d436dfe1da9d87f21a1b97b HG_NODE_LAST=5fddd98957c8a54a4d436dfe1da9d87f21a1b97b HG_SOURCE=serve HG_TXNID=TXN:* HG_TXNNAME=serve HG_URL=remote:ssh:127.0.0.1 (glob)
+ remote: txnclose hook: HG_BOOKMARK_MOVED=1 HG_BUNDLE2=1 HG_NEW_OBSMARKERS=1 HG_NODE=5fddd98957c8a54a4d436dfe1da9d87f21a1b97b HG_NODE_LAST=5fddd98957c8a54a4d436dfe1da9d87f21a1b97b HG_SOURCE=serve HG_TXNID=TXN:* HG_TXNNAME=serve HG_URL=remote:ssh:$LOCALIP (glob)
updating bookmark book_5fdd
pre-close-tip:02de42196ebe draft book_02de
postclose-tip:02de42196ebe draft book_02de
@@ -394,7 +394,7 @@
remote: lock: free
remote: wlock: free
remote: postclose-tip:32af7686d403 public book_32af
- remote: txnclose hook: HG_BOOKMARK_MOVED=1 HG_BUNDLE2=1 HG_NEW_OBSMARKERS=1 HG_NODE=32af7686d403cf45b5d95f2d70cebea587ac806a HG_NODE_LAST=32af7686d403cf45b5d95f2d70cebea587ac806a HG_PHASES_MOVED=1 HG_SOURCE=serve HG_TXNID=TXN:* HG_TXNNAME=serve HG_URL=remote:http:127.0.0.1: (glob)
+ remote: txnclose hook: HG_BOOKMARK_MOVED=1 HG_BUNDLE2=1 HG_NEW_OBSMARKERS=1 HG_NODE=32af7686d403cf45b5d95f2d70cebea587ac806a HG_NODE_LAST=32af7686d403cf45b5d95f2d70cebea587ac806a HG_PHASES_MOVED=1 HG_SOURCE=serve HG_TXNID=TXN:* HG_TXNNAME=serve HG_URL=remote:http:*: (glob)
updating bookmark book_32af
pre-close-tip:02de42196ebe draft book_02de
postclose-tip:02de42196ebe draft book_02de
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/test-check-help.t Tue Feb 28 11:13:25 2017 -0800
@@ -0,0 +1,25 @@
+#require test-repo
+
+ $ . "$TESTDIR/helpers-testrepo.sh"
+
+ $ cat <<'EOF' > scanhelptopics.py
+ > from __future__ import absolute_import, print_function
+ > import re
+ > import sys
+ > topics = set()
+ > topicre = re.compile(r':hg:`help ([a-z0-9\-.]+)`')
+ > for fname in sys.argv:
+ > with open(fname) as f:
+ > topics.update(m.group(1) for m in topicre.finditer(f.read()))
+ > for s in sorted(topics):
+ > print(s)
+ > EOF
+
+ $ cd "$TESTDIR"/..
+
+Check if ":hg:`help TOPIC`" is valid:
+(use "xargs -n1 -t" to see which help commands are executed)
+
+ $ hg files 'glob:{hgext,mercurial}/**/*.py' \
+ > | xargs python "$TESTTMP/scanhelptopics.py" \
+ > | xargs -n1 hg help > /dev/null
--- a/tests/test-check-py3-compat.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-check-py3-compat.t Tue Feb 28 11:13:25 2017 -0800
@@ -7,7 +7,6 @@
contrib/python-zstandard/setup.py not using absolute_import
contrib/python-zstandard/setup_zstd.py not using absolute_import
contrib/python-zstandard/tests/common.py not using absolute_import
- contrib/python-zstandard/tests/test_cffi.py not using absolute_import
contrib/python-zstandard/tests/test_compressor.py not using absolute_import
contrib/python-zstandard/tests/test_data_structures.py not using absolute_import
contrib/python-zstandard/tests/test_decompressor.py not using absolute_import
--- a/tests/test-check-pyflakes.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-check-pyflakes.t Tue Feb 28 11:13:25 2017 -0800
@@ -7,9 +7,8 @@
(skipping binary file random-seed)
$ hg locate 'set:**.py or grep("^#!.*python")' -X hgext/fsmonitor/pywatchman \
- > -X mercurial/pycompat.py \
+ > -X mercurial/pycompat.py -X contrib/python-zstandard \
> 2>/dev/null \
> | xargs pyflakes 2>/dev/null | "$TESTDIR/filterpyflakes.py"
- contrib/python-zstandard/tests/test_data_structures.py:107: local variable 'size' is assigned to but never used
tests/filterpyflakes.py:39: undefined name 'undefinedname'
--- a/tests/test-chg.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-chg.t Tue Feb 28 11:13:25 2017 -0800
@@ -32,6 +32,46 @@
$ cd ..
+editor
+------
+
+ $ cat >> pushbuffer.py <<EOF
+ > def reposetup(ui, repo):
+ > repo.ui.pushbuffer(subproc=True)
+ > EOF
+
+ $ chg init editor
+ $ cd editor
+
+by default, system() should be redirected to the client:
+
+ $ touch foo
+ $ CHGDEBUG= HGEDITOR=cat chg ci -Am channeled --edit 2>&1 \
+ > | egrep "HG:|run 'cat"
+ chg: debug: run 'cat "*"' at '$TESTTMP/editor' (glob)
+ HG: Enter commit message. Lines beginning with 'HG:' are removed.
+ HG: Leave message empty to abort commit.
+ HG: --
+ HG: user: test
+ HG: branch 'default'
+ HG: added foo
+
+but no redirection should be made if output is captured:
+
+ $ touch bar
+ $ CHGDEBUG= HGEDITOR=cat chg ci -Am bufferred --edit \
+ > --config extensions.pushbuffer="$TESTTMP/pushbuffer.py" 2>&1 \
+ > | egrep "HG:|run 'cat"
+ [1]
+
+check that commit commands succeeded:
+
+ $ hg log -T '{rev}:{desc}\n'
+ 1:bufferred
+ 0:channeled
+
+ $ cd ..
+
pager
-----
--- a/tests/test-clone.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-clone.t Tue Feb 28 11:13:25 2017 -0800
@@ -579,11 +579,11 @@
No remote source
#if windows
- $ hg clone http://127.0.0.1:3121/a b
+ $ hg clone http://$LOCALIP:3121/a b
abort: error: * (glob)
[255]
#else
- $ hg clone http://127.0.0.1:3121/a b
+ $ hg clone http://$LOCALIP:3121/a b
abort: error: *refused* (glob)
[255]
#endif
--- a/tests/test-commandserver.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-commandserver.t Tue Feb 28 11:13:25 2017 -0800
@@ -199,6 +199,8 @@
ui.usehttp2=true (?)
ui.foo=bar
ui.nontty=true
+ web.address=localhost
+ web\.ipv6=(?:True|False) (re)
*** runcommand init foo
*** runcommand -R foo showconfig ui defaults
defaults.backout=-d "0 0"
--- a/tests/test-completion.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-completion.t Tue Feb 28 11:13:25 2017 -0800
@@ -129,6 +129,7 @@
Show the global options
$ hg debugcomplete --options | sort
+ --color
--config
--cwd
--debug
@@ -138,6 +139,7 @@
--help
--hidden
--noninteractive
+ --pager
--profile
--quiet
--repository
@@ -157,6 +159,7 @@
--address
--certificate
--cmdserver
+ --color
--config
--cwd
--daemon
@@ -171,6 +174,7 @@
--ipv6
--name
--noninteractive
+ --pager
--pid-file
--port
--prefix
--- a/tests/test-config.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-config.t Tue Feb 28 11:13:25 2017 -0800
@@ -58,12 +58,12 @@
[
{
"name": "Section.KeY",
- "source": "*.hgrc:16", (glob)
+ "source": "*.hgrc:*", (glob)
"value": "Case Sensitive"
},
{
"name": "Section.key",
- "source": "*.hgrc:17", (glob)
+ "source": "*.hgrc:*", (glob)
"value": "lower case"
}
]
@@ -71,7 +71,7 @@
[
{
"name": "Section.KeY",
- "source": "*.hgrc:16", (glob)
+ "source": "*.hgrc:*", (glob)
"value": "Case Sensitive"
}
]
@@ -158,3 +158,9 @@
$ hg showconfig paths
paths.foo:suboption=~/foo
paths.foo=$TESTTMP/foo
+
+edit failure
+
+ $ HGEDITOR=false hg config --edit
+ abort: edit failed: false exited with status 1
+ [255]
--- a/tests/test-contrib-perf.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-contrib-perf.t Tue Feb 28 11:13:25 2017 -0800
@@ -109,6 +109,7 @@
perfvolatilesets
benchmark the computation of various volatile set
perfwalk (no help text available)
+ perfwrite microbenchmark ui.write
(use 'hg help -v perfstatusext' to show built-in aliases and global options)
$ hg perfaddremove
--- a/tests/test-convert-git.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-convert-git.t Tue Feb 28 11:13:25 2017 -0800
@@ -330,7 +330,7 @@
input validation
$ hg convert --config convert.git.similarity=foo --datesort git-repo2 fullrepo
- abort: convert.git.similarity is not an integer ('foo')
+ abort: convert.git.similarity is not a valid integer ('foo')
[255]
$ hg convert --config convert.git.similarity=-1 --datesort git-repo2 fullrepo
abort: similarity must be between 0 and 100
--- a/tests/test-diff-color.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-diff-color.t Tue Feb 28 11:13:25 2017 -0800
@@ -1,10 +1,10 @@
Setup
$ cat <<EOF >> $HGRCPATH
+ > [ui]
+ > color = always
> [color]
> mode = ansi
- > [extensions]
- > color =
> EOF
$ hg init repo
$ cd repo
@@ -35,7 +35,7 @@
default context
- $ hg diff --nodates --color=always
+ $ hg diff --nodates
\x1b[0;1mdiff -r cf9f4ba66af2 a\x1b[0m (esc)
\x1b[0;31;1m--- a/a\x1b[0m (esc)
\x1b[0;32;1m+++ b/a\x1b[0m (esc)
@@ -51,7 +51,7 @@
--unified=2
- $ hg diff --nodates -U 2 --color=always
+ $ hg diff --nodates -U 2
\x1b[0;1mdiff -r cf9f4ba66af2 a\x1b[0m (esc)
\x1b[0;31;1m--- a/a\x1b[0m (esc)
\x1b[0;32;1m+++ b/a\x1b[0m (esc)
@@ -65,10 +65,11 @@
diffstat
- $ hg diff --stat --color=always
+ $ hg diff --stat
a | 2 \x1b[0;32m+\x1b[0m\x1b[0;31m-\x1b[0m (esc)
1 files changed, 1 insertions(+), 1 deletions(-)
$ cat <<EOF >> $HGRCPATH
+ > [extensions]
> record =
> [ui]
> interactive = true
@@ -81,7 +82,7 @@
record
$ chmod +x a
- $ hg record --color=always -m moda a <<EOF
+ $ hg record -m moda a <<EOF
> y
> y
> EOF
@@ -111,7 +112,7 @@
qrecord
- $ hg qrecord --color=always -m moda patch <<EOF
+ $ hg qrecord -m moda patch <<EOF
> y
> y
> EOF
@@ -151,7 +152,7 @@
$ echo aa >> a
$ echo bb >> sub/b
- $ hg diff --color=always -S
+ $ hg diff -S
\x1b[0;1mdiff --git a/a b/a\x1b[0m (esc)
\x1b[0;31;1m--- a/a\x1b[0m (esc)
\x1b[0;32;1m+++ b/a\x1b[0m (esc)
@@ -176,7 +177,7 @@
> mid tab
> all tabs
> EOF
- $ hg diff --nodates --color=always
+ $ hg diff --nodates
\x1b[0;1mdiff --git a/a b/a\x1b[0m (esc)
\x1b[0;31;1m--- a/a\x1b[0m (esc)
\x1b[0;32;1m+++ b/a\x1b[0m (esc)
@@ -192,7 +193,7 @@
\x1b[0;32m+\x1b[0m \x1b[0;32mall\x1b[0m \x1b[0;32mtabs\x1b[0m\x1b[0;1;41m \x1b[0m (esc)
$ echo "[color]" >> $HGRCPATH
$ echo "diff.tab = bold magenta" >> $HGRCPATH
- $ hg diff --nodates --color=always
+ $ hg diff --nodates
\x1b[0;1mdiff --git a/a b/a\x1b[0m (esc)
\x1b[0;31;1m--- a/a\x1b[0m (esc)
\x1b[0;32;1m+++ b/a\x1b[0m (esc)
--- a/tests/test-doctest.py Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-doctest.py Tue Feb 28 11:13:25 2017 -0800
@@ -28,7 +28,8 @@
testmod('mercurial.patch')
testmod('mercurial.pathutil')
testmod('mercurial.parser')
-testmod('mercurial.revset')
+testmod('mercurial.revsetlang')
+testmod('mercurial.smartset')
testmod('mercurial.store')
testmod('mercurial.subrepo')
testmod('mercurial.templatefilters')
--- a/tests/test-eol.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-eol.t Tue Feb 28 11:13:25 2017 -0800
@@ -470,6 +470,22 @@
> EOF
$ hg commit -m 'consistent'
+ $ hg init subrepo
+ $ hg -R subrepo pull -qu .
+ $ echo "subrepo = subrepo" > .hgsub
+ $ hg ci -Am "add subrepo"
+ adding .hgeol
+ adding .hgsub
+ $ hg archive -S ../archive
+ $ find ../archive/* | sort
+ ../archive/a.txt
+ ../archive/subrepo
+ ../archive/subrepo/a.txt
+ $ cat ../archive/a.txt ../archive/subrepo/a.txt
+ first\r (esc)
+ second\r (esc)
+ first\r (esc)
+ second\r (esc)
Test trailing newline
--- a/tests/test-extension.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-extension.t Tue Feb 28 11:13:25 2017 -0800
@@ -532,6 +532,8 @@
all prompts
-q --quiet suppress output
-v --verbose enable additional output
+ --color TYPE when to colorize (boolean, always, auto, never, or
+ debug) (EXPERIMENTAL)
--config CONFIG [+] set/override config option (use 'section.name=value')
--debug enable debugging output
--debugger start debugger
@@ -543,6 +545,8 @@
--version output version information and exit
-h --help display help and exit
--hidden consider hidden changesets
+ --pager TYPE when to paginate (boolean, always, auto, or never)
+ (default: auto)
@@ -567,6 +571,8 @@
all prompts
-q --quiet suppress output
-v --verbose enable additional output
+ --color TYPE when to colorize (boolean, always, auto, never, or
+ debug) (EXPERIMENTAL)
--config CONFIG [+] set/override config option (use 'section.name=value')
--debug enable debugging output
--debugger start debugger
@@ -578,6 +584,8 @@
--version output version information and exit
-h --help display help and exit
--hidden consider hidden changesets
+ --pager TYPE when to paginate (boolean, always, auto, or never)
+ (default: auto)
@@ -845,6 +853,8 @@
all prompts
-q --quiet suppress output
-v --verbose enable additional output
+ --color TYPE when to colorize (boolean, always, auto, never, or
+ debug) (EXPERIMENTAL)
--config CONFIG [+] set/override config option (use 'section.name=value')
--debug enable debugging output
--debugger start debugger
@@ -856,6 +866,8 @@
--version output version information and exit
-h --help display help and exit
--hidden consider hidden changesets
+ --pager TYPE when to paginate (boolean, always, auto, or never)
+ (default: auto)
Make sure that single '-v' option shows help and built-ins only for 'dodo' command
$ hg help -v dodo
@@ -878,6 +890,8 @@
all prompts
-q --quiet suppress output
-v --verbose enable additional output
+ --color TYPE when to colorize (boolean, always, auto, never, or
+ debug) (EXPERIMENTAL)
--config CONFIG [+] set/override config option (use 'section.name=value')
--debug enable debugging output
--debugger start debugger
@@ -889,6 +903,8 @@
--version output version information and exit
-h --help display help and exit
--hidden consider hidden changesets
+ --pager TYPE when to paginate (boolean, always, auto, or never)
+ (default: auto)
In case when extension name doesn't match any of its commands,
help message should ask for '-v' to get list of built-in aliases
@@ -949,6 +965,8 @@
all prompts
-q --quiet suppress output
-v --verbose enable additional output
+ --color TYPE when to colorize (boolean, always, auto, never, or
+ debug) (EXPERIMENTAL)
--config CONFIG [+] set/override config option (use 'section.name=value')
--debug enable debugging output
--debugger start debugger
@@ -960,6 +978,8 @@
--version output version information and exit
-h --help display help and exit
--hidden consider hidden changesets
+ --pager TYPE when to paginate (boolean, always, auto, or never)
+ (default: auto)
$ hg help -v -e dudu
dudu extension -
@@ -981,6 +1001,8 @@
all prompts
-q --quiet suppress output
-v --verbose enable additional output
+ --color TYPE when to colorize (boolean, always, auto, never, or
+ debug) (EXPERIMENTAL)
--config CONFIG [+] set/override config option (use 'section.name=value')
--debug enable debugging output
--debugger start debugger
@@ -992,6 +1014,8 @@
--version output version information and exit
-h --help display help and exit
--hidden consider hidden changesets
+ --pager TYPE when to paginate (boolean, always, auto, or never)
+ (default: auto)
Disabled extension commands:
--- a/tests/test-gendoc-ro.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-gendoc-ro.t Tue Feb 28 11:13:25 2017 -0800
@@ -1,4 +1,9 @@
#require docutils gettext
+Error: the current ro localization has some rst defects exposed by
+moving pager to core. These two warnings about references are expected
+until the localization is corrected.
$ $TESTDIR/check-gendoc ro
checking for parse errors
+ gendoc.txt:58: (WARNING/2) Inline interpreted text or phrase reference start-string without end-string.
+ gendoc.txt:58: (WARNING/2) Inline interpreted text or phrase reference start-string without end-string.
--- a/tests/test-globalopts.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-globalopts.t Tue Feb 28 11:13:25 2017 -0800
@@ -351,6 +351,7 @@
hgweb Configuring hgweb
internals Technical implementation topics
merge-tools Merge Tools
+ pager Pager Support
patterns File Name Patterns
phases Working with Phases
revisions Specifying Revisions
@@ -432,6 +433,7 @@
hgweb Configuring hgweb
internals Technical implementation topics
merge-tools Merge Tools
+ pager Pager Support
patterns File Name Patterns
phases Working with Phases
revisions Specifying Revisions
--- a/tests/test-glog.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-glog.t Tue Feb 28 11:13:25 2017 -0800
@@ -82,18 +82,18 @@
> }
$ cat > printrevset.py <<EOF
- > from mercurial import extensions, revset, commands, cmdutil
+ > from mercurial import extensions, revsetlang, commands, cmdutil
>
> def uisetup(ui):
> def printrevset(orig, ui, repo, *pats, **opts):
> if opts.get('print_revset'):
> expr = cmdutil.getgraphlogrevs(repo, pats, opts)[1]
> if expr:
- > tree = revset.parse(expr)
+ > tree = revsetlang.parse(expr)
> else:
> tree = []
> ui.write('%r\n' % (opts.get('rev', []),))
- > ui.write(revset.prettyformat(tree) + '\n')
+ > ui.write(revsetlang.prettyformat(tree) + '\n')
> return 0
> return orig(ui, repo, *pats, **opts)
> entry = extensions.wrapcommand(commands.table, 'log', printrevset)
--- a/tests/test-help.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-help.t Tue Feb 28 11:13:25 2017 -0800
@@ -113,6 +113,7 @@
hgweb Configuring hgweb
internals Technical implementation topics
merge-tools Merge Tools
+ pager Pager Support
patterns File Name Patterns
phases Working with Phases
revisions Specifying Revisions
@@ -188,6 +189,7 @@
hgweb Configuring hgweb
internals Technical implementation topics
merge-tools Merge Tools
+ pager Pager Support
patterns File Name Patterns
phases Working with Phases
revisions Specifying Revisions
@@ -262,7 +264,6 @@
largefiles track large binary files
mq manage a stack of patches
notify hooks for sending email push notifications
- pager browse command output with an external pager
patchbomb command to send changesets as (a series of) patch emails
purge command to delete untracked files from the working
directory
@@ -315,6 +316,8 @@
all prompts
-q --quiet suppress output
-v --verbose enable additional output
+ --color TYPE when to colorize (boolean, always, auto, never, or
+ debug) (EXPERIMENTAL)
--config CONFIG [+] set/override config option (use 'section.name=value')
--debug enable debugging output
--debugger start debugger
@@ -326,6 +329,8 @@
--version output version information and exit
-h --help display help and exit
--hidden consider hidden changesets
+ --pager TYPE when to paginate (boolean, always, auto, or never)
+ (default: auto)
(use 'hg help' for the full list of commands)
@@ -411,6 +416,8 @@
all prompts
-q --quiet suppress output
-v --verbose enable additional output
+ --color TYPE when to colorize (boolean, always, auto, never, or
+ debug) (EXPERIMENTAL)
--config CONFIG [+] set/override config option (use 'section.name=value')
--debug enable debugging output
--debugger start debugger
@@ -422,6 +429,8 @@
--version output version information and exit
-h --help display help and exit
--hidden consider hidden changesets
+ --pager TYPE when to paginate (boolean, always, auto, or never)
+ (default: auto)
Test the textwidth config option
@@ -678,6 +687,7 @@
> ('', 'newline', '', 'line1\nline2')],
> 'hg nohelp',
> norepo=True)
+ > @command('debugoptADV', [('', 'aopt', None, 'option is (ADVANCED)')])
> @command('debugoptDEP', [('', 'dopt', None, 'option is (DEPRECATED)')])
> @command('debugoptEXP', [('', 'eopt', None, 'option is (EXPERIMENTAL)')])
> def nohelp(ui, *args, **kwargs):
@@ -827,6 +837,7 @@
hgweb Configuring hgweb
internals Technical implementation topics
merge-tools Merge Tools
+ pager Pager Support
patterns File Name Patterns
phases Working with Phases
revisions Specifying Revisions
@@ -889,6 +900,7 @@
complete "names" - tags, open branch names, bookmark names
debugobsolete
create arbitrary obsolete marker
+ debugoptADV (no help text available)
debugoptDEP (no help text available)
debugoptEXP (no help text available)
debugpathcomplete
@@ -1102,7 +1114,15 @@
(use 'hg help -v helpext' to show built-in aliases and global options)
-test deprecated and experimental options are hidden in command help
+test advanced, deprecated and experimental options are hidden in command help
+ $ hg help debugoptADV
+ hg debugoptADV
+
+ (no help text available)
+
+ options:
+
+ (some details hidden, use --verbose to show complete help)
$ hg help debugoptDEP
hg debugoptDEP
@@ -1121,7 +1141,9 @@
(some details hidden, use --verbose to show complete help)
-test deprecated and experimental options is shown with -v
+test advanced, deprecated and experimental options are shown with -v
+ $ hg help -v debugoptADV | grep aopt
+ --aopt option is (ADVANCED)
$ hg help -v debugoptDEP | grep dopt
--dopt option is (DEPRECATED)
$ hg help -v debugoptEXP | grep eopt
@@ -1547,11 +1569,11 @@
"default:pushurl" should be used instead.
$ hg help glossary.mcguffin
- abort: help section not found
+ abort: help section not found: glossary.mcguffin
[255]
$ hg help glossary.mc.guffin
- abort: help section not found
+ abort: help section not found: glossary.mc.guffin
[255]
$ hg help template.files
@@ -1792,7 +1814,7 @@
$ hg serve -R "$TESTTMP/test" -n test -p $HGPORT -d --pid-file=hg.pid
$ cat hg.pid >> $DAEMON_PIDS
- $ get-with-headers.py 127.0.0.1:$HGPORT "help"
+ $ get-with-headers.py $LOCALIP:$HGPORT "help"
200 Script output follows
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
@@ -1914,6 +1936,13 @@
Merge Tools
</td></tr>
<tr><td>
+ <a href="/help/pager">
+ pager
+ </a>
+ </td><td>
+ Pager Support
+ </td></tr>
+ <tr><td>
<a href="/help/patterns">
patterns
</a>
@@ -2361,7 +2390,7 @@
</html>
- $ get-with-headers.py 127.0.0.1:$HGPORT "help/add"
+ $ get-with-headers.py $LOCALIP:$HGPORT "help/add"
200 Script output follows
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
@@ -2491,6 +2520,9 @@
<td>--verbose</td>
<td>enable additional output</td></tr>
<tr><td></td>
+ <td>--color TYPE</td>
+ <td>when to colorize (boolean, always, auto, never, or debug) (EXPERIMENTAL)</td></tr>
+ <tr><td></td>
<td>--config CONFIG [+]</td>
<td>set/override config option (use 'section.name=value')</td></tr>
<tr><td></td>
@@ -2523,6 +2555,9 @@
<tr><td></td>
<td>--hidden</td>
<td>consider hidden changesets</td></tr>
+ <tr><td></td>
+ <td>--pager TYPE</td>
+ <td>when to paginate (boolean, always, auto, or never) (default: auto)</td></tr>
</table>
</div>
@@ -2535,7 +2570,7 @@
</html>
- $ get-with-headers.py 127.0.0.1:$HGPORT "help/remove"
+ $ get-with-headers.py $LOCALIP:$HGPORT "help/remove"
200 Script output follows
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
@@ -2686,6 +2721,9 @@
<td>--verbose</td>
<td>enable additional output</td></tr>
<tr><td></td>
+ <td>--color TYPE</td>
+ <td>when to colorize (boolean, always, auto, never, or debug) (EXPERIMENTAL)</td></tr>
+ <tr><td></td>
<td>--config CONFIG [+]</td>
<td>set/override config option (use 'section.name=value')</td></tr>
<tr><td></td>
@@ -2718,6 +2756,9 @@
<tr><td></td>
<td>--hidden</td>
<td>consider hidden changesets</td></tr>
+ <tr><td></td>
+ <td>--pager TYPE</td>
+ <td>when to paginate (boolean, always, auto, or never) (default: auto)</td></tr>
</table>
</div>
@@ -2730,7 +2771,7 @@
</html>
- $ get-with-headers.py 127.0.0.1:$HGPORT "help/dates"
+ $ get-with-headers.py $LOCALIP:$HGPORT "help/dates"
200 Script output follows
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
@@ -2837,7 +2878,7 @@
Sub-topic indexes rendered properly
- $ get-with-headers.py 127.0.0.1:$HGPORT "help/internals"
+ $ get-with-headers.py $LOCALIP:$HGPORT "help/internals"
200 Script output follows
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
@@ -2933,7 +2974,7 @@
Sub-topic topics rendered properly
- $ get-with-headers.py 127.0.0.1:$HGPORT "help/internals.changegroups"
+ $ get-with-headers.py $LOCALIP:$HGPORT "help/internals.changegroups"
200 Script output follows
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
--- a/tests/test-hgweb-commands.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-hgweb-commands.t Tue Feb 28 11:13:25 2017 -0800
@@ -58,7 +58,7 @@
Logs and changes
- $ get-with-headers.py 127.0.0.1:$HGPORT 'log/?style=atom'
+ $ get-with-headers.py $LOCALIP:$HGPORT 'log/?style=atom'
200 Script output follows
<?xml version="1.0" encoding="ascii"?>
@@ -244,7 +244,7 @@
</entry>
</feed>
- $ get-with-headers.py 127.0.0.1:$HGPORT 'log/?style=rss'
+ $ get-with-headers.py $LOCALIP:$HGPORT 'log/?style=rss'
200 Script output follows
<?xml version="1.0" encoding="ascii"?>
@@ -422,7 +422,7 @@
</channel>
</rss> (no-eol)
- $ get-with-headers.py 127.0.0.1:$HGPORT 'log/1/?style=atom'
+ $ get-with-headers.py $LOCALIP:$HGPORT 'log/1/?style=atom'
200 Script output follows
<?xml version="1.0" encoding="ascii"?>
@@ -522,7 +522,7 @@
</entry>
</feed>
- $ get-with-headers.py 127.0.0.1:$HGPORT 'log/1/?style=rss'
+ $ get-with-headers.py $LOCALIP:$HGPORT 'log/1/?style=rss'
200 Script output follows
<?xml version="1.0" encoding="ascii"?>
@@ -618,7 +618,7 @@
</channel>
</rss> (no-eol)
- $ get-with-headers.py 127.0.0.1:$HGPORT 'log/1/foo/?style=atom'
+ $ get-with-headers.py $LOCALIP:$HGPORT 'log/1/foo/?style=atom'
200 Script output follows
<?xml version="1.0" encoding="ascii"?>
@@ -673,7 +673,7 @@
</entry>
</feed>
- $ get-with-headers.py 127.0.0.1:$HGPORT 'log/1/foo/?style=rss'
+ $ get-with-headers.py $LOCALIP:$HGPORT 'log/1/foo/?style=rss'
200 Script output follows
<?xml version="1.0" encoding="ascii"?>
@@ -694,7 +694,7 @@
</channel>
</rss>
- $ get-with-headers.py 127.0.0.1:$HGPORT 'shortlog/'
+ $ get-with-headers.py $LOCALIP:$HGPORT 'shortlog/'
200 Script output follows
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
@@ -834,7 +834,7 @@
</body>
</html>
- $ get-with-headers.py 127.0.0.1:$HGPORT 'rev/0/'
+ $ get-with-headers.py $LOCALIP:$HGPORT 'rev/0/'
200 Script output follows
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
@@ -965,7 +965,7 @@
</body>
</html>
- $ get-with-headers.py 127.0.0.1:$HGPORT 'rev/1/?style=raw'
+ $ get-with-headers.py $LOCALIP:$HGPORT 'rev/1/?style=raw'
200 Script output follows
@@ -982,7 +982,7 @@
@@ -0,0 +1,1 @@
+2ef0ac749a14e4f57a5a822464a0902c6f7f448f 1.0
- $ get-with-headers.py 127.0.0.1:$HGPORT 'log?rev=base'
+ $ get-with-headers.py $LOCALIP:$HGPORT 'log?rev=base'
200 Script output follows
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
@@ -1071,12 +1071,12 @@
</body>
</html>
- $ get-with-headers.py 127.0.0.1:$HGPORT 'log?rev=stable&style=raw' | grep 'revision:'
+ $ get-with-headers.py $LOCALIP:$HGPORT 'log?rev=stable&style=raw' | grep 'revision:'
revision: 2
Search with revset syntax
- $ get-with-headers.py 127.0.0.1:$HGPORT 'log?rev=tip^&style=raw'
+ $ get-with-headers.py $LOCALIP:$HGPORT 'log?rev=tip^&style=raw'
200 Script output follows
@@ -1093,7 +1093,7 @@
branch: stable
- $ get-with-headers.py 127.0.0.1:$HGPORT 'log?rev=last(all(),2)^&style=raw'
+ $ get-with-headers.py $LOCALIP:$HGPORT 'log?rev=last(all(),2)^&style=raw'
200 Script output follows
@@ -1117,7 +1117,7 @@
branch: default
- $ get-with-headers.py 127.0.0.1:$HGPORT 'log?rev=last(all(,2)^&style=raw'
+ $ get-with-headers.py $LOCALIP:$HGPORT 'log?rev=last(all(,2)^&style=raw'
200 Script output follows
@@ -1127,7 +1127,7 @@
# Mode literal keyword search
- $ get-with-headers.py 127.0.0.1:$HGPORT 'log?rev=last(al(),2)^&style=raw'
+ $ get-with-headers.py $LOCALIP:$HGPORT 'log?rev=last(al(),2)^&style=raw'
200 Script output follows
@@ -1137,7 +1137,7 @@
# Mode literal keyword search
- $ get-with-headers.py 127.0.0.1:$HGPORT 'log?rev=bookmark(anotherthing)&style=raw'
+ $ get-with-headers.py $LOCALIP:$HGPORT 'log?rev=bookmark(anotherthing)&style=raw'
200 Script output follows
@@ -1155,7 +1155,7 @@
bookmark: anotherthing
- $ get-with-headers.py 127.0.0.1:$HGPORT 'log?rev=bookmark(abc)&style=raw'
+ $ get-with-headers.py $LOCALIP:$HGPORT 'log?rev=bookmark(abc)&style=raw'
200 Script output follows
@@ -1165,7 +1165,7 @@
# Mode literal keyword search
- $ get-with-headers.py 127.0.0.1:$HGPORT 'log?rev=deadbeef:&style=raw'
+ $ get-with-headers.py $LOCALIP:$HGPORT 'log?rev=deadbeef:&style=raw'
200 Script output follows
@@ -1176,7 +1176,7 @@
- $ get-with-headers.py 127.0.0.1:$HGPORT 'log?rev=user("test")&style=raw'
+ $ get-with-headers.py $LOCALIP:$HGPORT 'log?rev=user("test")&style=raw'
200 Script output follows
@@ -1217,7 +1217,7 @@
bookmark: anotherthing
- $ get-with-headers.py 127.0.0.1:$HGPORT 'log?rev=user("re:test")&style=raw'
+ $ get-with-headers.py $LOCALIP:$HGPORT 'log?rev=user("re:test")&style=raw'
200 Script output follows
@@ -1230,11 +1230,11 @@
File-related
- $ get-with-headers.py 127.0.0.1:$HGPORT 'file/1/foo/?style=raw'
+ $ get-with-headers.py $LOCALIP:$HGPORT 'file/1/foo/?style=raw'
200 Script output follows
foo
- $ get-with-headers.py 127.0.0.1:$HGPORT 'annotate/1/foo/?style=raw'
+ $ get-with-headers.py $LOCALIP:$HGPORT 'annotate/1/foo/?style=raw'
200 Script output follows
@@ -1243,7 +1243,7 @@
- $ get-with-headers.py 127.0.0.1:$HGPORT 'file/1/?style=raw'
+ $ get-with-headers.py $LOCALIP:$HGPORT 'file/1/?style=raw'
200 Script output follows
@@ -1259,7 +1259,7 @@
$ hg parents --template "{node|short}\n" -r 1 foo
2ef0ac749a14
- $ get-with-headers.py 127.0.0.1:$HGPORT 'file/1/foo'
+ $ get-with-headers.py $LOCALIP:$HGPORT 'file/1/foo'
200 Script output follows
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
@@ -1354,7 +1354,7 @@
</body>
</html>
- $ get-with-headers.py 127.0.0.1:$HGPORT 'filediff/0/foo/?style=raw'
+ $ get-with-headers.py $LOCALIP:$HGPORT 'filediff/0/foo/?style=raw'
200 Script output follows
@@ -1368,7 +1368,7 @@
- $ get-with-headers.py 127.0.0.1:$HGPORT 'filediff/1/foo/?style=raw'
+ $ get-with-headers.py $LOCALIP:$HGPORT 'filediff/1/foo/?style=raw'
200 Script output follows
@@ -1384,7 +1384,7 @@
$ hg parents --template "{node|short}\n" -r 2 foo
2ef0ac749a14
- $ get-with-headers.py 127.0.0.1:$HGPORT 'file/2/foo'
+ $ get-with-headers.py $LOCALIP:$HGPORT 'file/2/foo'
200 Script output follows
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
@@ -1483,23 +1483,23 @@
Overviews
- $ get-with-headers.py 127.0.0.1:$HGPORT 'raw-tags'
+ $ get-with-headers.py $LOCALIP:$HGPORT 'raw-tags'
200 Script output follows
tip cad8025a2e87f88c06259790adfa15acb4080123
1.0 2ef0ac749a14e4f57a5a822464a0902c6f7f448f
- $ get-with-headers.py 127.0.0.1:$HGPORT 'raw-branches'
+ $ get-with-headers.py $LOCALIP:$HGPORT 'raw-branches'
200 Script output follows
unstable cad8025a2e87f88c06259790adfa15acb4080123 open
stable 1d22e65f027e5a0609357e7d8e7508cd2ba5d2fe inactive
default a4f92ed23982be056b9852de5dfe873eaac7f0de inactive
- $ get-with-headers.py 127.0.0.1:$HGPORT 'raw-bookmarks'
+ $ get-with-headers.py $LOCALIP:$HGPORT 'raw-bookmarks'
200 Script output follows
something cad8025a2e87f88c06259790adfa15acb4080123
anotherthing 2ef0ac749a14e4f57a5a822464a0902c6f7f448f
- $ get-with-headers.py 127.0.0.1:$HGPORT 'summary/?style=gitweb'
+ $ get-with-headers.py $LOCALIP:$HGPORT 'summary/?style=gitweb'
200 Script output follows
<?xml version="1.0" encoding="ascii"?>
@@ -1697,7 +1697,7 @@
</body>
</html>
- $ get-with-headers.py 127.0.0.1:$HGPORT 'graph/?style=gitweb'
+ $ get-with-headers.py $LOCALIP:$HGPORT 'graph/?style=gitweb'
200 Script output follows
<?xml version="1.0" encoding="ascii"?>
@@ -1843,7 +1843,7 @@
raw graph
- $ get-with-headers.py 127.0.0.1:$HGPORT 'graph/?style=raw'
+ $ get-with-headers.py $LOCALIP:$HGPORT 'graph/?style=raw'
200 Script output follows
@@ -1893,28 +1893,28 @@
capabilities
- $ get-with-headers.py 127.0.0.1:$HGPORT '?cmd=capabilities'; echo
+ $ get-with-headers.py $LOCALIP:$HGPORT '?cmd=capabilities'; echo
200 Script output follows
lookup changegroupsubset branchmap pushkey known getbundle unbundlehash batch bundle2=HG20%0Achangegroup%3D01%2C02%0Adigests%3Dmd5%2Csha1%2Csha512%0Aerror%3Dabort%2Cunsupportedcontent%2Cpushraced%2Cpushkey%0Ahgtagsfnodes%0Alistkeys%0Apushkey%0Aremote-changegroup%3Dhttp%2Chttps unbundle=HG10GZ,HG10BZ,HG10UN httpheader=1024 httpmediatype=0.1rx,0.1tx,0.2tx compression=*zlib (glob)
heads
- $ get-with-headers.py 127.0.0.1:$HGPORT '?cmd=heads'
+ $ get-with-headers.py $LOCALIP:$HGPORT '?cmd=heads'
200 Script output follows
cad8025a2e87f88c06259790adfa15acb4080123
branches
- $ get-with-headers.py 127.0.0.1:$HGPORT '?cmd=branches&nodes=0000000000000000000000000000000000000000'
+ $ get-with-headers.py $LOCALIP:$HGPORT '?cmd=branches&nodes=0000000000000000000000000000000000000000'
200 Script output follows
0000000000000000000000000000000000000000 0000000000000000000000000000000000000000 0000000000000000000000000000000000000000 0000000000000000000000000000000000000000
changegroup
- $ get-with-headers.py 127.0.0.1:$HGPORT '?cmd=changegroup&roots=0000000000000000000000000000000000000000'
+ $ get-with-headers.py $LOCALIP:$HGPORT '?cmd=changegroup&roots=0000000000000000000000000000000000000000'
200 Script output follows
x\x9c\xbd\x94MHTQ\x14\xc7'+\x9d\xc66\x81\x89P\xc1\xa3\x14\xcct\xba\xef\xbe\xfb\xde\xbb\xcfr0\xb3"\x02\x11[%\x98\xdcO\xa7\xd2\x19\x98y\xd2\x07h"\x96\xa0e\xda\xa6lUY-\xca\x08\xa2\x82\x16\x96\xd1\xa2\xf0#\xc8\x95\x1b\xdd$!m*"\xc8\x82\xea\xbe\x9c\x01\x85\xc9\x996\x1d\xf8\xc1\xe3~\x9d\xff9\xef\x7f\xaf\xcf\xe7\xbb\x19\xfc4\xec^\xcb\x9b\xfbz\xa6\xbe\xb3\x90_\xef/\x8d\x9e\xad\xbe\xe4\xcb0\xd2\xec\xad\x12X:\xc8\x12\x12\xd9:\x95\xba \x1cG\xb7$\xc5\xc44\x1c(\x1d\x03\x03\xdb\x84\x0cK#\xe0\x8a\xb8\x1b\x00\x1a\x08p\xb2SF\xa3\x01\x8f\x00%q\xa1Ny{k!8\xe5t>[{\xe2j\xddl\xc3\xcf\xee\xd0\xddW\x9ff3U\x9djobj\xbb\x87E\x88\x05l\x001\x12\x18\x13\xc6 \xb7(\xe3\x02a\x80\x81\xcel.u\x9b\x1b\x8c\x91\x80Z\x0c\x15\x15 (esc)
@@ -1925,14 +1925,14 @@
stream_out
- $ get-with-headers.py 127.0.0.1:$HGPORT '?cmd=stream_out'
+ $ get-with-headers.py $LOCALIP:$HGPORT '?cmd=stream_out'
200 Script output follows
1
failing unbundle, requires POST request
- $ get-with-headers.py 127.0.0.1:$HGPORT '?cmd=unbundle'
+ $ get-with-headers.py $LOCALIP:$HGPORT '?cmd=unbundle'
405 push requires POST request
0
@@ -1941,7 +1941,7 @@
Static files
- $ get-with-headers.py 127.0.0.1:$HGPORT 'static/style.css'
+ $ get-with-headers.py $LOCALIP:$HGPORT 'static/style.css'
200 Script output follows
a { text-decoration:none; }
@@ -2077,7 +2077,7 @@
> --cwd .. -R `pwd`
$ cat hg.pid >> $DAEMON_PIDS
- $ get-with-headers.py 127.0.0.1:$HGPORT 'log?rev=adds("foo")&style=raw'
+ $ get-with-headers.py $LOCALIP:$HGPORT 'log?rev=adds("foo")&style=raw'
200 Script output follows
@@ -2110,7 +2110,7 @@
Graph json escape of multibyte character
- $ get-with-headers.py 127.0.0.1:$HGPORT 'graph/' > out
+ $ get-with-headers.py $LOCALIP:$HGPORT 'graph/' > out
>>> from __future__ import print_function
>>> for line in open("out"):
... if line.startswith("var data ="):
@@ -2121,14 +2121,14 @@
(plain version to check the format)
- $ get-with-headers.py 127.0.0.1:$HGPORT '?cmd=capabilities' | dd ibs=75 count=1 2> /dev/null; echo
+ $ get-with-headers.py $LOCALIP:$HGPORT '?cmd=capabilities' | dd ibs=75 count=1 2> /dev/null; echo
200 Script output follows
lookup changegroupsubset branchmap pushkey known
(spread version to check the content)
- $ get-with-headers.py 127.0.0.1:$HGPORT '?cmd=capabilities' | tr ' ' '\n'; echo
+ $ get-with-headers.py $LOCALIP:$HGPORT '?cmd=capabilities' | tr ' ' '\n'; echo
200
Script
output
@@ -2194,23 +2194,23 @@
Test paging
- $ get-with-headers.py 127.0.0.1:$HGPORT \
+ $ get-with-headers.py $LOCALIP:$HGPORT \
> 'graph/?style=raw' | grep changeset
changeset: aed2d9c1d0e7
changeset: b60a39a85a01
- $ get-with-headers.py 127.0.0.1:$HGPORT \
+ $ get-with-headers.py $LOCALIP:$HGPORT \
> 'graph/?style=raw&revcount=3' | grep changeset
changeset: aed2d9c1d0e7
changeset: b60a39a85a01
changeset: ada793dcc118
- $ get-with-headers.py 127.0.0.1:$HGPORT \
+ $ get-with-headers.py $LOCALIP:$HGPORT \
> 'graph/e06180cbfb0?style=raw&revcount=3' | grep changeset
changeset: e06180cbfb0c
changeset: b4e73ffab476
- $ get-with-headers.py 127.0.0.1:$HGPORT \
+ $ get-with-headers.py $LOCALIP:$HGPORT \
> 'graph/b4e73ffab47?style=raw&revcount=3' | grep changeset
changeset: b4e73ffab476
--- a/tests/test-hgweb-descend-empties.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-hgweb-descend-empties.t Tue Feb 28 11:13:25 2017 -0800
@@ -29,7 +29,7 @@
manifest with descending (paper)
- $ get-with-headers.py 127.0.0.1:$HGPORT 'file'
+ $ get-with-headers.py $LOCALIP:$HGPORT 'file'
200 Script output follows
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
@@ -147,7 +147,7 @@
manifest with descending (coal)
- $ get-with-headers.py 127.0.0.1:$HGPORT 'file?style=coal'
+ $ get-with-headers.py $LOCALIP:$HGPORT 'file?style=coal'
200 Script output follows
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
@@ -266,7 +266,7 @@
manifest with descending (monoblue)
- $ get-with-headers.py 127.0.0.1:$HGPORT 'file?style=monoblue'
+ $ get-with-headers.py $LOCALIP:$HGPORT 'file?style=monoblue'
200 Script output follows
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
@@ -379,7 +379,7 @@
manifest with descending (gitweb)
- $ get-with-headers.py 127.0.0.1:$HGPORT 'file?style=gitweb'
+ $ get-with-headers.py $LOCALIP:$HGPORT 'file?style=gitweb'
200 Script output follows
<?xml version="1.0" encoding="ascii"?>
@@ -482,7 +482,7 @@
manifest with descending (spartan)
- $ get-with-headers.py 127.0.0.1:$HGPORT 'file?style=spartan'
+ $ get-with-headers.py $LOCALIP:$HGPORT 'file?style=spartan'
200 Script output follows
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
--- a/tests/test-hgweb-json.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-hgweb-json.t Tue Feb 28 11:13:25 2017 -0800
@@ -1593,6 +1593,10 @@
"topic": "merge-tools"
},
{
+ "summary": "Pager Support",
+ "topic": "pager"
+ },
+ {
"summary": "File Name Patterns",
"topic": "patterns"
},
--- a/tests/test-hgweb-no-path-info.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-hgweb-no-path-info.t Tue Feb 28 11:13:25 2017 -0800
@@ -49,7 +49,7 @@
> 'REQUEST_METHOD': 'GET',
> 'PATH_INFO': '/',
> 'SCRIPT_NAME': '',
- > 'SERVER_NAME': '127.0.0.1',
+ > 'SERVER_NAME': '$LOCALIP',
> 'SERVER_PORT': os.environ['HGPORT'],
> 'SERVER_PROTOCOL': 'HTTP/1.0'
> }
@@ -79,16 +79,16 @@
<?xml version="1.0" encoding="ascii"?>
<feed xmlns="http://www.w3.org/2005/Atom">
<!-- Changelog -->
- <id>http://127.0.0.1:$HGPORT/</id> (glob)
- <link rel="self" href="http://127.0.0.1:$HGPORT/atom-log"/> (glob)
- <link rel="alternate" href="http://127.0.0.1:$HGPORT/"/> (glob)
+ <id>http://$LOCALIP:$HGPORT/</id> (glob)
+ <link rel="self" href="http://$LOCALIP:$HGPORT/atom-log"/> (glob)
+ <link rel="alternate" href="http://$LOCALIP:$HGPORT/"/> (glob)
<title>repo Changelog</title>
<updated>1970-01-01T00:00:00+00:00</updated>
<entry>
<title>[default] test</title>
- <id>http://127.0.0.1:$HGPORT/#changeset-61c9426e69fef294feed5e2bbfc97d39944a5b1c</id> (glob)
- <link href="http://127.0.0.1:$HGPORT/rev/61c9426e69fe"/> (glob)
+ <id>http://$LOCALIP:$HGPORT/#changeset-61c9426e69fef294feed5e2bbfc97d39944a5b1c</id> (glob)
+ <link href="http://$LOCALIP:$HGPORT/rev/61c9426e69fe"/> (glob)
<author>
<name>test</name>
<email>test</email>
--- a/tests/test-hgweb-no-request-uri.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-hgweb-no-request-uri.t Tue Feb 28 11:13:25 2017 -0800
@@ -48,7 +48,7 @@
> 'wsgi.run_once': False,
> 'REQUEST_METHOD': 'GET',
> 'SCRIPT_NAME': '',
- > 'SERVER_NAME': '127.0.0.1',
+ > 'SERVER_NAME': '$LOCALIP',
> 'SERVER_PORT': os.environ['HGPORT'],
> 'SERVER_PROTOCOL': 'HTTP/1.0'
> }
@@ -90,16 +90,16 @@
<?xml version="1.0" encoding="ascii"?>
<feed xmlns="http://www.w3.org/2005/Atom">
<!-- Changelog -->
- <id>http://127.0.0.1:$HGPORT/</id> (glob)
- <link rel="self" href="http://127.0.0.1:$HGPORT/atom-log"/> (glob)
- <link rel="alternate" href="http://127.0.0.1:$HGPORT/"/> (glob)
+ <id>http://$LOCALIP:$HGPORT/</id> (glob)
+ <link rel="self" href="http://$LOCALIP:$HGPORT/atom-log"/> (glob)
+ <link rel="alternate" href="http://$LOCALIP:$HGPORT/"/> (glob)
<title>repo Changelog</title>
<updated>1970-01-01T00:00:00+00:00</updated>
<entry>
<title>[default] test</title>
- <id>http://127.0.0.1:$HGPORT/#changeset-61c9426e69fef294feed5e2bbfc97d39944a5b1c</id> (glob)
- <link href="http://127.0.0.1:$HGPORT/rev/61c9426e69fe"/> (glob)
+ <id>http://$LOCALIP:$HGPORT/#changeset-61c9426e69fef294feed5e2bbfc97d39944a5b1c</id> (glob)
+ <link href="http://$LOCALIP:$HGPORT/rev/61c9426e69fe"/> (glob)
<author>
<name>test</name>
<email>test</email>
--- a/tests/test-hgweb-non-interactive.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-hgweb-non-interactive.t Tue Feb 28 11:13:25 2017 -0800
@@ -60,7 +60,7 @@
> 'SCRIPT_NAME': '',
> 'PATH_INFO': '',
> 'QUERY_STRING': '',
- > 'SERVER_NAME': '127.0.0.1',
+ > 'SERVER_NAME': '$LOCALIP',
> 'SERVER_PORT': os.environ['HGPORT'],
> 'SERVER_PROTOCOL': 'HTTP/1.0'
> }
--- a/tests/test-hgweb-raw.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-hgweb-raw.t Tue Feb 28 11:13:25 2017 -0800
@@ -32,7 +32,7 @@
It is very boring to read, but computers don't
care about things like that.
$ cat access.log error.log
- 127.0.0.1 - - [*] "GET /?f=bf0ff59095c9;file=sub/some%20text%25.txt;style=raw HTTP/1.1" 200 - (glob)
+ $LOCALIP - - [*] "GET /?f=bf0ff59095c9;file=sub/some%20text%25.txt;style=raw HTTP/1.1" 200 - (glob)
$ rm access.log error.log
$ hg serve -p $HGPORT -A access.log -E error.log -d --pid-file=hg.pid \
@@ -53,6 +53,6 @@
It is very boring to read, but computers don't
care about things like that.
$ cat access.log error.log
- 127.0.0.1 - - [*] "GET /?f=bf0ff59095c9;file=sub/some%20text%25.txt;style=raw HTTP/1.1" 200 - (glob)
+ $LOCALIP - - [*] "GET /?f=bf0ff59095c9;file=sub/some%20text%25.txt;style=raw HTTP/1.1" 200 - (glob)
$ cd ..
--- a/tests/test-hgweb-symrev.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-hgweb-symrev.t Tue Feb 28 11:13:25 2017 -0800
@@ -37,7 +37,7 @@
(De)referencing symbolic revisions (paper)
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'shortlog?style=paper' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'shortlog?style=paper' | egrep $REVLINKS
<li><a href="/graph/tip?style=paper">graph</a></li>
<li><a href="/rev/tip?style=paper">changeset</a></li>
<li><a href="/file/tip?style=paper">browse</a></li>
@@ -52,7 +52,7 @@
<a href="/shortlog/tip?revcount=120&style=paper">more</a>
| rev 2: <a href="/shortlog/43c799df6e75?style=paper">(0)</a> <a href="/shortlog/tip?style=paper">tip</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'graph?style=paper' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'graph?style=paper' | egrep $REVLINKS
<li><a href="/shortlog/tip?style=paper">log</a></li>
<li><a href="/rev/tip?style=paper">changeset</a></li>
<li><a href="/file/tip?style=paper">browse</a></li>
@@ -63,7 +63,7 @@
<a href="/graph/tip?revcount=120&style=paper">more</a>
| rev 2: <a href="/graph/43c799df6e75?style=paper">(0)</a> <a href="/graph/tip?style=paper">tip</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'file?style=paper' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'file?style=paper' | egrep $REVLINKS
<li><a href="/shortlog/tip?style=paper">log</a></li>
<li><a href="/graph/tip?style=paper">graph</a></li>
<li><a href="/rev/tip?style=paper">changeset</a></li>
@@ -74,24 +74,24 @@
<a href="/file/tip/dir/?style=paper">
<a href="/file/tip/foo?style=paper">
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'branches?style=paper' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'branches?style=paper' | egrep $REVLINKS
<a href="/shortlog/default?style=paper" class="open">
<a href="/shortlog/9d8c40cba617?style=paper" class="open">
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'tags?style=paper' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'tags?style=paper' | egrep $REVLINKS
<a href="/rev/tip?style=paper">
<a href="/rev/9d8c40cba617?style=paper">
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'bookmarks?style=paper' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'bookmarks?style=paper' | egrep $REVLINKS
<a href="/rev/xyzzy?style=paper">
<a href="/rev/a7c1559b7bba?style=paper">
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'shortlog?style=paper&rev=all()' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'shortlog?style=paper&rev=all()' | egrep $REVLINKS
<a href="/rev/9d8c40cba617?style=paper">third</a>
<a href="/rev/a7c1559b7bba?style=paper">second</a>
<a href="/rev/43c799df6e75?style=paper">first</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'rev/xyzzy?style=paper' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'rev/xyzzy?style=paper' | egrep $REVLINKS
<li><a href="/shortlog/xyzzy?style=paper">log</a></li>
<li><a href="/graph/xyzzy?style=paper">graph</a></li>
<li><a href="/raw-rev/xyzzy?style=paper">raw</a></li>
@@ -102,7 +102,7 @@
<td class="author"> <a href="/rev/9d8c40cba617?style=paper">9d8c40cba617</a></td>
<td class="files"><a href="/file/a7c1559b7bba/foo?style=paper">foo</a> </td>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'shortlog/xyzzy?style=paper' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'shortlog/xyzzy?style=paper' | egrep $REVLINKS
<li><a href="/graph/xyzzy?style=paper">graph</a></li>
<li><a href="/rev/xyzzy?style=paper">changeset</a></li>
<li><a href="/file/xyzzy?style=paper">browse</a></li>
@@ -116,7 +116,7 @@
<a href="/shortlog/xyzzy?revcount=120&style=paper">more</a>
| rev 1: <a href="/shortlog/43c799df6e75?style=paper">(0)</a> <a href="/shortlog/tip?style=paper">tip</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'graph/xyzzy?style=paper' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'graph/xyzzy?style=paper' | egrep $REVLINKS
<li><a href="/shortlog/xyzzy?style=paper">log</a></li>
<li><a href="/rev/xyzzy?style=paper">changeset</a></li>
<li><a href="/file/xyzzy?style=paper">browse</a></li>
@@ -127,7 +127,7 @@
<a href="/graph/xyzzy?revcount=120&style=paper">more</a>
| rev 1: <a href="/graph/43c799df6e75?style=paper">(0)</a> <a href="/graph/tip?style=paper">tip</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'file/xyzzy?style=paper' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'file/xyzzy?style=paper' | egrep $REVLINKS
<li><a href="/shortlog/xyzzy?style=paper">log</a></li>
<li><a href="/graph/xyzzy?style=paper">graph</a></li>
<li><a href="/rev/xyzzy?style=paper">changeset</a></li>
@@ -138,7 +138,7 @@
<a href="/file/xyzzy/dir/?style=paper">
<a href="/file/xyzzy/foo?style=paper">
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'file/xyzzy/foo?style=paper' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'file/xyzzy/foo?style=paper' | egrep $REVLINKS
<li><a href="/shortlog/xyzzy?style=paper">log</a></li>
<li><a href="/graph/xyzzy?style=paper">graph</a></li>
<li><a href="/rev/xyzzy?style=paper">changeset</a></li>
@@ -153,7 +153,7 @@
<td class="author"><a href="/file/43c799df6e75/foo?style=paper">43c799df6e75</a> </td>
<td class="author"><a href="/file/9d8c40cba617/foo?style=paper">9d8c40cba617</a> </td>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'log/xyzzy/foo?style=paper' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'log/xyzzy/foo?style=paper' | egrep $REVLINKS
href="/atom-log/tip/foo" title="Atom feed for test:foo" />
href="/rss-log/tip/foo" title="RSS feed for test:foo" />
<li><a href="/shortlog/xyzzy?style=paper">log</a></li>
@@ -176,7 +176,7 @@
<a href="/log/xyzzy/foo?revcount=120&style=paper">more</a>
| <a href="/log/43c799df6e75/foo?style=paper">(0)</a> <a href="/log/tip/foo?style=paper">tip</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'annotate/xyzzy/foo?style=paper' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'annotate/xyzzy/foo?style=paper' | egrep $REVLINKS
<li><a href="/shortlog/xyzzy?style=paper">log</a></li>
<li><a href="/graph/xyzzy?style=paper">graph</a></li>
<li><a href="/rev/xyzzy?style=paper">changeset</a></li>
@@ -200,7 +200,7 @@
<a href="/diff/a7c1559b7bba/foo?style=paper">diff</a>
<a href="/rev/a7c1559b7bba?style=paper">changeset</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'diff/xyzzy/foo?style=paper' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'diff/xyzzy/foo?style=paper' | egrep $REVLINKS
<li><a href="/shortlog/xyzzy?style=paper">log</a></li>
<li><a href="/graph/xyzzy?style=paper">graph</a></li>
<li><a href="/rev/xyzzy?style=paper">changeset</a></li>
@@ -215,7 +215,7 @@
<td><a href="/file/43c799df6e75/foo?style=paper">43c799df6e75</a> </td>
<td><a href="/file/9d8c40cba617/foo?style=paper">9d8c40cba617</a> </td>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'comparison/xyzzy/foo?style=paper' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'comparison/xyzzy/foo?style=paper' | egrep $REVLINKS
<li><a href="/shortlog/xyzzy?style=paper">log</a></li>
<li><a href="/graph/xyzzy?style=paper">graph</a></li>
<li><a href="/rev/xyzzy?style=paper">changeset</a></li>
@@ -232,7 +232,7 @@
(De)referencing symbolic revisions (coal)
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'shortlog?style=coal' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'shortlog?style=coal' | egrep $REVLINKS
<li><a href="/graph/tip?style=coal">graph</a></li>
<li><a href="/rev/tip?style=coal">changeset</a></li>
<li><a href="/file/tip?style=coal">browse</a></li>
@@ -247,7 +247,7 @@
<a href="/shortlog/tip?revcount=120&style=coal">more</a>
| rev 2: <a href="/shortlog/43c799df6e75?style=coal">(0)</a> <a href="/shortlog/tip?style=coal">tip</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'graph?style=coal' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'graph?style=coal' | egrep $REVLINKS
<li><a href="/shortlog/tip?style=coal">log</a></li>
<li><a href="/rev/tip?style=coal">changeset</a></li>
<li><a href="/file/tip?style=coal">browse</a></li>
@@ -258,7 +258,7 @@
<a href="/graph/tip?revcount=120&style=coal">more</a>
| rev 2: <a href="/graph/43c799df6e75?style=coal">(0)</a> <a href="/graph/tip?style=coal">tip</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'file?style=coal' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'file?style=coal' | egrep $REVLINKS
<li><a href="/shortlog/tip?style=coal">log</a></li>
<li><a href="/graph/tip?style=coal">graph</a></li>
<li><a href="/rev/tip?style=coal">changeset</a></li>
@@ -269,24 +269,24 @@
<a href="/file/tip/dir/?style=coal">
<a href="/file/tip/foo?style=coal">
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'branches?style=coal' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'branches?style=coal' | egrep $REVLINKS
<a href="/shortlog/default?style=coal" class="open">
<a href="/shortlog/9d8c40cba617?style=coal" class="open">
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'tags?style=coal' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'tags?style=coal' | egrep $REVLINKS
<a href="/rev/tip?style=coal">
<a href="/rev/9d8c40cba617?style=coal">
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'bookmarks?style=coal' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'bookmarks?style=coal' | egrep $REVLINKS
<a href="/rev/xyzzy?style=coal">
<a href="/rev/a7c1559b7bba?style=coal">
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'shortlog?style=coal&rev=all()' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'shortlog?style=coal&rev=all()' | egrep $REVLINKS
<a href="/rev/9d8c40cba617?style=coal">third</a>
<a href="/rev/a7c1559b7bba?style=coal">second</a>
<a href="/rev/43c799df6e75?style=coal">first</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'rev/xyzzy?style=coal' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'rev/xyzzy?style=coal' | egrep $REVLINKS
<li><a href="/shortlog/xyzzy?style=coal">log</a></li>
<li><a href="/graph/xyzzy?style=coal">graph</a></li>
<li><a href="/raw-rev/xyzzy?style=coal">raw</a></li>
@@ -297,7 +297,7 @@
<td class="author"> <a href="/rev/9d8c40cba617?style=coal">9d8c40cba617</a></td>
<td class="files"><a href="/file/a7c1559b7bba/foo?style=coal">foo</a> </td>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'shortlog/xyzzy?style=coal' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'shortlog/xyzzy?style=coal' | egrep $REVLINKS
<li><a href="/graph/xyzzy?style=coal">graph</a></li>
<li><a href="/rev/xyzzy?style=coal">changeset</a></li>
<li><a href="/file/xyzzy?style=coal">browse</a></li>
@@ -311,7 +311,7 @@
<a href="/shortlog/xyzzy?revcount=120&style=coal">more</a>
| rev 1: <a href="/shortlog/43c799df6e75?style=coal">(0)</a> <a href="/shortlog/tip?style=coal">tip</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'graph/xyzzy?style=coal' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'graph/xyzzy?style=coal' | egrep $REVLINKS
<li><a href="/shortlog/xyzzy?style=coal">log</a></li>
<li><a href="/rev/xyzzy?style=coal">changeset</a></li>
<li><a href="/file/xyzzy?style=coal">browse</a></li>
@@ -322,7 +322,7 @@
<a href="/graph/xyzzy?revcount=120&style=coal">more</a>
| rev 1: <a href="/graph/43c799df6e75?style=coal">(0)</a> <a href="/graph/tip?style=coal">tip</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'file/xyzzy?style=coal' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'file/xyzzy?style=coal' | egrep $REVLINKS
<li><a href="/shortlog/xyzzy?style=coal">log</a></li>
<li><a href="/graph/xyzzy?style=coal">graph</a></li>
<li><a href="/rev/xyzzy?style=coal">changeset</a></li>
@@ -333,7 +333,7 @@
<a href="/file/xyzzy/dir/?style=coal">
<a href="/file/xyzzy/foo?style=coal">
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'file/xyzzy/foo?style=coal' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'file/xyzzy/foo?style=coal' | egrep $REVLINKS
<li><a href="/shortlog/xyzzy?style=coal">log</a></li>
<li><a href="/graph/xyzzy?style=coal">graph</a></li>
<li><a href="/rev/xyzzy?style=coal">changeset</a></li>
@@ -348,7 +348,7 @@
<td class="author"><a href="/file/43c799df6e75/foo?style=coal">43c799df6e75</a> </td>
<td class="author"><a href="/file/9d8c40cba617/foo?style=coal">9d8c40cba617</a> </td>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'log/xyzzy/foo?style=coal' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'log/xyzzy/foo?style=coal' | egrep $REVLINKS
href="/atom-log/tip/foo" title="Atom feed for test:foo" />
href="/rss-log/tip/foo" title="RSS feed for test:foo" />
<li><a href="/shortlog/xyzzy?style=coal">log</a></li>
@@ -371,7 +371,7 @@
<a href="/log/xyzzy/foo?revcount=120&style=coal">more</a>
| <a href="/log/43c799df6e75/foo?style=coal">(0)</a> <a href="/log/tip/foo?style=coal">tip</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'annotate/xyzzy/foo?style=coal' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'annotate/xyzzy/foo?style=coal' | egrep $REVLINKS
<li><a href="/shortlog/xyzzy?style=coal">log</a></li>
<li><a href="/graph/xyzzy?style=coal">graph</a></li>
<li><a href="/rev/xyzzy?style=coal">changeset</a></li>
@@ -395,7 +395,7 @@
<a href="/diff/a7c1559b7bba/foo?style=coal">diff</a>
<a href="/rev/a7c1559b7bba?style=coal">changeset</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'diff/xyzzy/foo?style=coal' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'diff/xyzzy/foo?style=coal' | egrep $REVLINKS
<li><a href="/shortlog/xyzzy?style=coal">log</a></li>
<li><a href="/graph/xyzzy?style=coal">graph</a></li>
<li><a href="/rev/xyzzy?style=coal">changeset</a></li>
@@ -410,7 +410,7 @@
<td><a href="/file/43c799df6e75/foo?style=coal">43c799df6e75</a> </td>
<td><a href="/file/9d8c40cba617/foo?style=coal">9d8c40cba617</a> </td>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'comparison/xyzzy/foo?style=coal' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'comparison/xyzzy/foo?style=coal' | egrep $REVLINKS
<li><a href="/shortlog/xyzzy?style=coal">log</a></li>
<li><a href="/graph/xyzzy?style=coal">graph</a></li>
<li><a href="/rev/xyzzy?style=coal">changeset</a></li>
@@ -427,7 +427,7 @@
(De)referencing symbolic revisions (gitweb)
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'summary?style=gitweb' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'summary?style=gitweb' | egrep $REVLINKS
<a href="/file?style=gitweb">files</a> | <a href="/archive/tip.zip">zip</a> |
<a class="list" href="/rev/9d8c40cba617?style=gitweb">
<a href="/rev/9d8c40cba617?style=gitweb">changeset</a> |
@@ -447,7 +447,7 @@
<a href="/log/9d8c40cba617?style=gitweb">changelog</a> |
<a href="/file/9d8c40cba617?style=gitweb">files</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'shortlog?style=gitweb' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'shortlog?style=gitweb' | egrep $REVLINKS
<a href="/log/tip?style=gitweb">changelog</a> |
<a href="/graph/tip?style=gitweb">graph</a> |
<a href="/file/tip?style=gitweb">files</a> | <a href="/archive/tip.zip">zip</a> |
@@ -463,7 +463,7 @@
<a href="/file/43c799df6e75?style=gitweb">files</a>
<a href="/shortlog/43c799df6e75?style=gitweb">(0)</a> <a href="/shortlog/tip?style=gitweb">tip</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'log?style=gitweb' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'log?style=gitweb' | egrep $REVLINKS
<a href="/shortlog/tip?style=gitweb">shortlog</a> |
<a href="/graph/tip?style=gitweb">graph</a> |
<a href="/file/tip?style=gitweb">files</a> | <a href="/archive/tip.zip">zip</a> |
@@ -476,7 +476,7 @@
<a href="/rev/43c799df6e75?style=gitweb">changeset</a><br/>
<a href="/log/43c799df6e75?style=gitweb">(0)</a> <a href="/log/tip?style=gitweb">tip</a> <br/>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'graph?style=gitweb' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'graph?style=gitweb' | egrep $REVLINKS
<a href="/shortlog/tip?style=gitweb">shortlog</a> |
<a href="/log/tip?style=gitweb">changelog</a> |
<a href="/file/tip?style=gitweb">files</a> |
@@ -487,25 +487,25 @@
<a href="/graph/tip?revcount=120&style=gitweb">more</a>
| <a href="/graph/43c799df6e75?style=gitweb">(0)</a> <a href="/graph/tip?style=gitweb">tip</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'tags?style=gitweb' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'tags?style=gitweb' | egrep $REVLINKS
<td><a class="list" href="/rev/tip?style=gitweb"><b>tip</b></a></td>
<a href="/rev/9d8c40cba617?style=gitweb">changeset</a> |
<a href="/log/9d8c40cba617?style=gitweb">changelog</a> |
<a href="/file/9d8c40cba617?style=gitweb">files</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'bookmarks?style=gitweb' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'bookmarks?style=gitweb' | egrep $REVLINKS
<td><a class="list" href="/rev/xyzzy?style=gitweb"><b>xyzzy</b></a></td>
<a href="/rev/a7c1559b7bba?style=gitweb">changeset</a> |
<a href="/log/a7c1559b7bba?style=gitweb">changelog</a> |
<a href="/file/a7c1559b7bba?style=gitweb">files</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'branches?style=gitweb' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'branches?style=gitweb' | egrep $REVLINKS
<td class="open"><a class="list" href="/shortlog/default?style=gitweb"><b>default</b></a></td>
<a href="/changeset/9d8c40cba617?style=gitweb">changeset</a> |
<a href="/log/9d8c40cba617?style=gitweb">changelog</a> |
<a href="/file/9d8c40cba617?style=gitweb">files</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'file?style=gitweb' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'file?style=gitweb' | egrep $REVLINKS
<a href="/rev/tip?style=gitweb">changeset</a> | <a href="/archive/tip.zip">zip</a> |
<td><a href="/file/tip/?style=gitweb">[up]</a></td>
<a href="/file/tip/dir?style=gitweb">dir</a>
@@ -516,7 +516,7 @@
<a href="/log/tip/foo?style=gitweb">revisions</a> |
<a href="/annotate/tip/foo?style=gitweb">annotate</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'shortlog?style=gitweb&rev=all()' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'shortlog?style=gitweb&rev=all()' | egrep $REVLINKS
<a href="/file?style=gitweb">files</a> | <a href="/archive/tip.zip">zip</a>
<a class="title" href="/rev/9d8c40cba617?style=gitweb"><span class="age">Thu, 01 Jan 1970 00:00:00 +0000</span>third<span class="logtags"> <span class="branchtag" title="default">default</span> <span class="tagtag" title="tip">tip</span> </span></a>
<a href="/rev/9d8c40cba617?style=gitweb">changeset</a><br/>
@@ -525,7 +525,7 @@
<a class="title" href="/rev/43c799df6e75?style=gitweb"><span class="age">Thu, 01 Jan 1970 00:00:00 +0000</span>first<span class="logtags"> </span></a>
<a href="/rev/43c799df6e75?style=gitweb">changeset</a><br/>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'rev/xyzzy?style=gitweb' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'rev/xyzzy?style=gitweb' | egrep $REVLINKS
<a href="/shortlog/xyzzy?style=gitweb">shortlog</a> |
<a href="/log/xyzzy?style=gitweb">changelog</a> |
<a href="/graph/xyzzy?style=gitweb">graph</a> |
@@ -542,7 +542,7 @@
<a href="/comparison/a7c1559b7bba/foo?style=gitweb">comparison</a> |
<a href="/log/a7c1559b7bba/foo?style=gitweb">revisions</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'shortlog/xyzzy?style=gitweb' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'shortlog/xyzzy?style=gitweb' | egrep $REVLINKS
<a href="/log/xyzzy?style=gitweb">changelog</a> |
<a href="/graph/xyzzy?style=gitweb">graph</a> |
<a href="/file/xyzzy?style=gitweb">files</a> | <a href="/archive/xyzzy.zip">zip</a> |
@@ -555,7 +555,7 @@
<a href="/file/43c799df6e75?style=gitweb">files</a>
<a href="/shortlog/43c799df6e75?style=gitweb">(0)</a> <a href="/shortlog/tip?style=gitweb">tip</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'log/xyzzy?style=gitweb' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'log/xyzzy?style=gitweb' | egrep $REVLINKS
<a href="/shortlog/xyzzy?style=gitweb">shortlog</a> |
<a href="/graph/xyzzy?style=gitweb">graph</a> |
<a href="/file/xyzzy?style=gitweb">files</a> | <a href="/archive/xyzzy.zip">zip</a> |
@@ -566,7 +566,7 @@
<a href="/rev/43c799df6e75?style=gitweb">changeset</a><br/>
<a href="/log/43c799df6e75?style=gitweb">(0)</a> <a href="/log/tip?style=gitweb">tip</a> <br/>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'graph/xyzzy?style=gitweb' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'graph/xyzzy?style=gitweb' | egrep $REVLINKS
<a href="/shortlog/xyzzy?style=gitweb">shortlog</a> |
<a href="/log/xyzzy?style=gitweb">changelog</a> |
<a href="/file/xyzzy?style=gitweb">files</a> |
@@ -577,7 +577,7 @@
<a href="/graph/xyzzy?revcount=120&style=gitweb">more</a>
| <a href="/graph/43c799df6e75?style=gitweb">(0)</a> <a href="/graph/tip?style=gitweb">tip</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'file/xyzzy?style=gitweb' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'file/xyzzy?style=gitweb' | egrep $REVLINKS
<a href="/rev/xyzzy?style=gitweb">changeset</a> | <a href="/archive/xyzzy.zip">zip</a> |
<td><a href="/file/xyzzy/?style=gitweb">[up]</a></td>
<a href="/file/xyzzy/dir?style=gitweb">dir</a>
@@ -588,7 +588,7 @@
<a href="/log/xyzzy/foo?style=gitweb">revisions</a> |
<a href="/annotate/xyzzy/foo?style=gitweb">annotate</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'file/xyzzy/foo?style=gitweb' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'file/xyzzy/foo?style=gitweb' | egrep $REVLINKS
<a href="/file/xyzzy/?style=gitweb">files</a> |
<a href="/rev/xyzzy?style=gitweb">changeset</a> |
<a href="/file/tip/foo?style=gitweb">latest</a> |
@@ -601,7 +601,7 @@
<a class="list" href="/file/43c799df6e75/foo?style=gitweb">
<a class="list" href="/file/9d8c40cba617/foo?style=gitweb">9d8c40cba617</a></td>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'log/xyzzy/foo?style=gitweb' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'log/xyzzy/foo?style=gitweb' | egrep $REVLINKS
<a href="/file/xyzzy/foo?style=gitweb">file</a> |
<a href="/annotate/xyzzy/foo?style=gitweb">annotate</a> |
<a href="/diff/xyzzy/foo?style=gitweb">diff</a> |
@@ -616,9 +616,11 @@
<a href="/file/43c799df6e75/foo?style=gitweb">file</a> |
<a href="/diff/43c799df6e75/foo?style=gitweb">diff</a> |
<a href="/annotate/43c799df6e75/foo?style=gitweb">annotate</a>
+ <a href="/log/xyzzy/foo?revcount=30&style=gitweb">less</a>
+ <a href="/log/xyzzy/foo?revcount=120&style=gitweb">more</a>
<a href="/log/43c799df6e75/foo?style=gitweb">(0)</a> <a href="/log/tip/foo?style=gitweb">tip</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'annotate/xyzzy/foo?style=gitweb' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'annotate/xyzzy/foo?style=gitweb' | egrep $REVLINKS
<a href="/file/xyzzy/?style=gitweb">files</a> |
<a href="/rev/xyzzy?style=gitweb">changeset</a> |
<a href="/file/xyzzy/foo?style=gitweb">file</a> |
@@ -640,7 +642,7 @@
<a href="/diff/a7c1559b7bba/foo?style=gitweb">diff</a>
<a href="/rev/a7c1559b7bba?style=gitweb">changeset</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'diff/xyzzy/foo?style=gitweb' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'diff/xyzzy/foo?style=gitweb' | egrep $REVLINKS
<a href="/file/xyzzy?style=gitweb">files</a> |
<a href="/rev/xyzzy?style=gitweb">changeset</a> |
<a href="/file/xyzzy/foo?style=gitweb">file</a> |
@@ -653,7 +655,7 @@
<a class="list" href="/diff/43c799df6e75/foo?style=gitweb">
<a class="list" href="/diff/9d8c40cba617/foo?style=gitweb">9d8c40cba617</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'comparison/xyzzy/foo?style=gitweb' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'comparison/xyzzy/foo?style=gitweb' | egrep $REVLINKS
<a href="/file/xyzzy?style=gitweb">files</a> |
<a href="/rev/xyzzy?style=gitweb">changeset</a> |
<a href="/file/xyzzy/foo?style=gitweb">file</a> |
@@ -668,7 +670,7 @@
(De)referencing symbolic revisions (monoblue)
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'summary?style=monoblue' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'summary?style=monoblue' | egrep $REVLINKS
<li><a href="/archive/tip.zip">zip</a></li>
<a href="/rev/9d8c40cba617?style=monoblue">
<a href="/rev/9d8c40cba617?style=monoblue">changeset</a> |
@@ -688,7 +690,7 @@
<a href="/log/9d8c40cba617?style=monoblue">changelog</a> |
<a href="/file/9d8c40cba617?style=monoblue">files</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'shortlog?style=monoblue' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'shortlog?style=monoblue' | egrep $REVLINKS
<li><a href="/graph/tip?style=monoblue">graph</a></li>
<li><a href="/file/tip?style=monoblue">files</a></li>
<li><a href="/archive/tip.zip">zip</a></li>
@@ -703,7 +705,7 @@
<a href="/file/43c799df6e75?style=monoblue">files</a>
<a href="/shortlog/43c799df6e75?style=monoblue">(0)</a> <a href="/shortlog/tip?style=monoblue">tip</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'log?style=monoblue' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'log?style=monoblue' | egrep $REVLINKS
<li><a href="/graph/tip?style=monoblue">graph</a></li>
<li><a href="/file/tip?style=monoblue">files</a></li>
<li><a href="/archive/tip.zip">zip</a></li>
@@ -712,31 +714,31 @@
<h3 class="changelog"><a class="title" href="/rev/43c799df6e75?style=monoblue">first<span class="logtags"> </span></a></h3>
<a href="/log/43c799df6e75?style=monoblue">(0)</a> <a href="/log/tip?style=monoblue">tip</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'graph?style=monoblue' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'graph?style=monoblue' | egrep $REVLINKS
<li><a href="/file/tip?style=monoblue">files</a></li>
<a href="/graph/tip?revcount=30&style=monoblue">less</a>
<a href="/graph/tip?revcount=120&style=monoblue">more</a>
| <a href="/graph/43c799df6e75?style=monoblue">(0)</a> <a href="/graph/tip?style=monoblue">tip</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'tags?style=monoblue' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'tags?style=monoblue' | egrep $REVLINKS
<td><a href="/rev/tip?style=monoblue">tip</a></td>
<a href="/rev/9d8c40cba617?style=monoblue">changeset</a> |
<a href="/log/9d8c40cba617?style=monoblue">changelog</a> |
<a href="/file/9d8c40cba617?style=monoblue">files</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'bookmarks?style=monoblue' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'bookmarks?style=monoblue' | egrep $REVLINKS
<td><a href="/rev/xyzzy?style=monoblue">xyzzy</a></td>
<a href="/rev/a7c1559b7bba?style=monoblue">changeset</a> |
<a href="/log/a7c1559b7bba?style=monoblue">changelog</a> |
<a href="/file/a7c1559b7bba?style=monoblue">files</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'branches?style=monoblue' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'branches?style=monoblue' | egrep $REVLINKS
<td class="open"><a href="/shortlog/default?style=monoblue">default</a></td>
<a href="/rev/9d8c40cba617?style=monoblue">changeset</a> |
<a href="/log/9d8c40cba617?style=monoblue">changelog</a> |
<a href="/file/9d8c40cba617?style=monoblue">files</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'file?style=monoblue' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'file?style=monoblue' | egrep $REVLINKS
<li><a href="/graph/tip?style=monoblue">graph</a></li>
<li><a href="/rev/tip?style=monoblue">changeset</a></li>
<li><a href="/archive/tip.zip">zip</a></li>
@@ -749,13 +751,13 @@
<a href="/log/tip/foo?style=monoblue">revisions</a> |
<a href="/annotate/tip/foo?style=monoblue">annotate</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'shortlog?style=monoblue&rev=all()' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'shortlog?style=monoblue&rev=all()' | egrep $REVLINKS
<li><a href="/archive/tip.zip">zip</a></li>
<h3 class="changelog"><a class="title" href="/rev/9d8c40cba617?style=monoblue">third<span class="logtags"> <span class="branchtag" title="default">default</span> <span class="tagtag" title="tip">tip</span> </span></a></h3>
<h3 class="changelog"><a class="title" href="/rev/a7c1559b7bba?style=monoblue">second<span class="logtags"> <span class="bookmarktag" title="xyzzy">xyzzy</span> </span></a></h3>
<h3 class="changelog"><a class="title" href="/rev/43c799df6e75?style=monoblue">first<span class="logtags"> </span></a></h3>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'rev/xyzzy?style=monoblue' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'rev/xyzzy?style=monoblue' | egrep $REVLINKS
<li><a href="/graph/xyzzy?style=monoblue">graph</a></li>
<li><a href="/file/xyzzy?style=monoblue">files</a></li>
<li><a href="/raw-rev/xyzzy">raw</a></li>
@@ -771,7 +773,7 @@
<a href="/comparison/a7c1559b7bba/foo?style=monoblue">comparison</a> |
<a href="/log/a7c1559b7bba/foo?style=monoblue">revisions</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'shortlog/xyzzy?style=monoblue' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'shortlog/xyzzy?style=monoblue' | egrep $REVLINKS
<li><a href="/graph/xyzzy?style=monoblue">graph</a></li>
<li><a href="/file/xyzzy?style=monoblue">files</a></li>
<li><a href="/archive/xyzzy.zip">zip</a></li>
@@ -783,7 +785,7 @@
<a href="/file/43c799df6e75?style=monoblue">files</a>
<a href="/shortlog/43c799df6e75?style=monoblue">(0)</a> <a href="/shortlog/tip?style=monoblue">tip</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'log/xyzzy?style=monoblue' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'log/xyzzy?style=monoblue' | egrep $REVLINKS
<li><a href="/graph/xyzzy?style=monoblue">graph</a></li>
<li><a href="/file/xyzzy?style=monoblue">files</a></li>
<li><a href="/archive/xyzzy.zip">zip</a></li>
@@ -791,13 +793,13 @@
<h3 class="changelog"><a class="title" href="/rev/43c799df6e75?style=monoblue">first<span class="logtags"> </span></a></h3>
<a href="/log/43c799df6e75?style=monoblue">(0)</a> <a href="/log/tip?style=monoblue">tip</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'graph/xyzzy?style=monoblue' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'graph/xyzzy?style=monoblue' | egrep $REVLINKS
<li><a href="/file/xyzzy?style=monoblue">files</a></li>
<a href="/graph/xyzzy?revcount=30&style=monoblue">less</a>
<a href="/graph/xyzzy?revcount=120&style=monoblue">more</a>
| <a href="/graph/43c799df6e75?style=monoblue">(0)</a> <a href="/graph/tip?style=monoblue">tip</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'file/xyzzy?style=monoblue' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'file/xyzzy?style=monoblue' | egrep $REVLINKS
<li><a href="/graph/xyzzy?style=monoblue">graph</a></li>
<li><a href="/rev/xyzzy?style=monoblue">changeset</a></li>
<li><a href="/archive/xyzzy.zip">zip</a></li>
@@ -810,7 +812,7 @@
<a href="/log/xyzzy/foo?style=monoblue">revisions</a> |
<a href="/annotate/xyzzy/foo?style=monoblue">annotate</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'file/xyzzy/foo?style=monoblue' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'file/xyzzy/foo?style=monoblue' | egrep $REVLINKS
<li><a href="/graph/xyzzy?style=monoblue">graph</a></li>
<li><a href="/file/xyzzy/?style=monoblue">files</a></li>
<li><a href="/file/tip/foo?style=monoblue">latest</a></li>
@@ -823,7 +825,7 @@
<a href="/file/43c799df6e75/foo?style=monoblue">
<a href="/file/9d8c40cba617/foo?style=monoblue">9d8c40cba617</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'log/xyzzy/foo?style=monoblue' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'log/xyzzy/foo?style=monoblue' | egrep $REVLINKS
<li><a href="/graph/xyzzy?style=monoblue">graph</a></li>
<li><a href="/file/xyzzy?style=monoblue">files</a></li>
<li><a href="/file/xyzzy/foo?style=monoblue">file</a></li>
@@ -841,7 +843,7 @@
<a href="/annotate/43c799df6e75/foo?style=monoblue">annotate</a>
<a href="/log/43c799df6e75/foo?style=monoblue">(0)</a> <a href="/log/tip/foo?style=monoblue">tip</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'annotate/xyzzy/foo?style=monoblue' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'annotate/xyzzy/foo?style=monoblue' | egrep $REVLINKS
<li><a href="/graph/xyzzy?style=monoblue">graph</a></li>
<li><a href="/file/xyzzy/?style=monoblue">files</a></li>
<li><a href="/file/xyzzy/foo?style=monoblue">file</a></li>
@@ -863,7 +865,7 @@
<a href="/diff/a7c1559b7bba/foo?style=monoblue">diff</a>
<a href="/rev/a7c1559b7bba?style=monoblue">changeset</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'diff/xyzzy/foo?style=monoblue' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'diff/xyzzy/foo?style=monoblue' | egrep $REVLINKS
<li><a href="/graph/xyzzy?style=monoblue">graph</a></li>
<li><a href="/file/xyzzy?style=monoblue">files</a></li>
<li><a href="/file/xyzzy/foo?style=monoblue">file</a></li>
@@ -876,7 +878,7 @@
<dd><a href="/diff/43c799df6e75/foo?style=monoblue">43c799df6e75</a></dd>
<dd><a href="/diff/9d8c40cba617/foo?style=monoblue">9d8c40cba617</a></dd>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'comparison/xyzzy/foo?style=monoblue' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'comparison/xyzzy/foo?style=monoblue' | egrep $REVLINKS
<li><a href="/graph/xyzzy?style=monoblue">graph</a></li>
<li><a href="/file/xyzzy?style=monoblue">files</a></li>
<li><a href="/file/xyzzy/foo?style=monoblue">file</a></li>
@@ -891,7 +893,7 @@
(De)referencing symbolic revisions (spartan)
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'shortlog?style=spartan' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'shortlog?style=spartan' | egrep $REVLINKS
<a href="/log/tip?style=spartan">changelog</a>
<a href="/graph/tip?style=spartan">graph</a>
<a href="/file/tip/?style=spartan">files</a>
@@ -902,7 +904,7 @@
<td class="node"><a href="/rev/43c799df6e75?style=spartan">first</a></td>
navigate: <small class="navigate"><a href="/shortlog/43c799df6e75?style=spartan">(0)</a> <a href="/shortlog/tip?style=spartan">tip</a> </small>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'log?style=spartan' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'log?style=spartan' | egrep $REVLINKS
<a href="/shortlog/tip?style=spartan">shortlog</a>
<a href="/graph/tip?style=spartan">graph</a>
<a href="/file/tip?style=spartan">files</a>
@@ -919,20 +921,20 @@
<td class="files"><a href="/diff/43c799df6e75/dir/bar?style=spartan">dir/bar</a> <a href="/diff/43c799df6e75/foo?style=spartan">foo</a> </td>
navigate: <small class="navigate"><a href="/log/43c799df6e75?style=spartan">(0)</a> <a href="/log/tip?style=spartan">tip</a> </small>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'graph?style=spartan' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'graph?style=spartan' | egrep $REVLINKS
<a href="/log/tip?style=spartan">changelog</a>
<a href="/shortlog/tip?style=spartan">shortlog</a>
<a href="/file/tip/?style=spartan">files</a>
navigate: <small class="navigate"><a href="/graph/43c799df6e75?style=spartan">(0)</a> <a href="/graph/tip?style=spartan">tip</a> </small>
navigate: <small class="navigate"><a href="/graph/43c799df6e75?style=spartan">(0)</a> <a href="/graph/tip?style=spartan">tip</a> </small>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'tags?style=spartan' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'tags?style=spartan' | egrep $REVLINKS
<a href="/rev/9d8c40cba617?style=spartan">tip</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'branches?style=spartan' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'branches?style=spartan' | egrep $REVLINKS
<a href="/shortlog/9d8c40cba617?style=spartan" class="open">default</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'file?style=spartan' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'file?style=spartan' | egrep $REVLINKS
<a href="/log/tip?style=spartan">changelog</a>
<a href="/shortlog/tip?style=spartan">shortlog</a>
<a href="/graph/tip?style=spartan">graph</a>
@@ -944,7 +946,7 @@
<a href="/file/tip/dir/?style=spartan">
<td><a href="/file/tip/foo?style=spartan">foo</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'shortlog?style=spartan&rev=all()' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'shortlog?style=spartan&rev=all()' | egrep $REVLINKS
<a href="/archive/tip.zip">zip</a>
<td class="node"><a href="/rev/9d8c40cba617?style=spartan">9d8c40cba617</a></td>
<a href="/rev/a7c1559b7bba?style=spartan">a7c1559b7bba</a>
@@ -960,7 +962,7 @@
<th class="files"><a href="/file/43c799df6e75?style=spartan">files</a>:</th>
<td class="files"><a href="/diff/43c799df6e75/dir/bar?style=spartan">dir/bar</a> <a href="/diff/43c799df6e75/foo?style=spartan">foo</a> </td>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'rev/xyzzy?style=spartan' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'rev/xyzzy?style=spartan' | egrep $REVLINKS
<a href="/log/xyzzy?style=spartan">changelog</a>
<a href="/shortlog/xyzzy?style=spartan">shortlog</a>
<a href="/graph/xyzzy?style=spartan">graph</a>
@@ -972,7 +974,7 @@
<td class="child"><a href="/rev/9d8c40cba617?style=spartan">9d8c40cba617</a></td>
<td class="files"><a href="/file/a7c1559b7bba/foo?style=spartan">foo</a> </td>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'shortlog/xyzzy?style=spartan' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'shortlog/xyzzy?style=spartan' | egrep $REVLINKS
<a href="/log/xyzzy?style=spartan">changelog</a>
<a href="/graph/xyzzy?style=spartan">graph</a>
<a href="/file/xyzzy/?style=spartan">files</a>
@@ -982,7 +984,7 @@
<td class="node"><a href="/rev/43c799df6e75?style=spartan">first</a></td>
navigate: <small class="navigate"><a href="/shortlog/43c799df6e75?style=spartan">(0)</a> <a href="/shortlog/tip?style=spartan">tip</a> </small>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'log/xyzzy?style=spartan' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'log/xyzzy?style=spartan' | egrep $REVLINKS
<a href="/shortlog/xyzzy?style=spartan">shortlog</a>
<a href="/graph/xyzzy?style=spartan">graph</a>
<a href="/file/xyzzy?style=spartan">files</a>
@@ -996,14 +998,14 @@
<td class="files"><a href="/diff/43c799df6e75/dir/bar?style=spartan">dir/bar</a> <a href="/diff/43c799df6e75/foo?style=spartan">foo</a> </td>
navigate: <small class="navigate"><a href="/log/43c799df6e75?style=spartan">(0)</a> <a href="/log/tip?style=spartan">tip</a> </small>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'graph/xyzzy?style=spartan' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'graph/xyzzy?style=spartan' | egrep $REVLINKS
<a href="/log/xyzzy?style=spartan">changelog</a>
<a href="/shortlog/xyzzy?style=spartan">shortlog</a>
<a href="/file/xyzzy/?style=spartan">files</a>
navigate: <small class="navigate"><a href="/graph/43c799df6e75?style=spartan">(0)</a> <a href="/graph/tip?style=spartan">tip</a> </small>
navigate: <small class="navigate"><a href="/graph/43c799df6e75?style=spartan">(0)</a> <a href="/graph/tip?style=spartan">tip</a> </small>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'file/xyzzy?style=spartan' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'file/xyzzy?style=spartan' | egrep $REVLINKS
<a href="/log/xyzzy?style=spartan">changelog</a>
<a href="/shortlog/xyzzy?style=spartan">shortlog</a>
<a href="/graph/xyzzy?style=spartan">graph</a>
@@ -1015,7 +1017,7 @@
<a href="/file/xyzzy/dir/?style=spartan">
<td><a href="/file/xyzzy/foo?style=spartan">foo</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'file/xyzzy/foo?style=spartan' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'file/xyzzy/foo?style=spartan' | egrep $REVLINKS
<a href="/log/xyzzy?style=spartan">changelog</a>
<a href="/shortlog/xyzzy?style=spartan">shortlog</a>
<a href="/graph/xyzzy?style=spartan">graph</a>
@@ -1028,7 +1030,7 @@
<a href="/file/43c799df6e75/foo?style=spartan">
<td><a href="/file/9d8c40cba617/foo?style=spartan">9d8c40cba617</a></td>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'log/xyzzy/foo?style=spartan' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'log/xyzzy/foo?style=spartan' | egrep $REVLINKS
href="/atom-log/tip/foo" title="Atom feed for test:foo">
href="/rss-log/tip/foo" title="RSS feed for test:foo">
<a href="/file/xyzzy/foo?style=spartan">file</a>
@@ -1045,7 +1047,7 @@
<a href="/diff/43c799df6e75/foo?style=spartan">(diff)</a>
<a href="/annotate/43c799df6e75/foo?style=spartan">(annotate)</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'annotate/xyzzy/foo?style=spartan' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'annotate/xyzzy/foo?style=spartan' | egrep $REVLINKS
<a href="/log/xyzzy?style=spartan">changelog</a>
<a href="/shortlog/xyzzy?style=spartan">shortlog</a>
<a href="/graph/xyzzy?style=spartan">graph</a>
@@ -1067,7 +1069,7 @@
<a href="/diff/a7c1559b7bba/foo?style=spartan">diff</a>
<a href="/rev/a7c1559b7bba?style=spartan">changeset</a>
- $ "$TESTDIR/get-with-headers.py" 127.0.0.1:$HGPORT 'diff/xyzzy/foo?style=spartan' | egrep $REVLINKS
+ $ "$TESTDIR/get-with-headers.py" $LOCALIP:$HGPORT 'diff/xyzzy/foo?style=spartan' | egrep $REVLINKS
<a href="/log/xyzzy?style=spartan">changelog</a>
<a href="/shortlog/xyzzy?style=spartan">shortlog</a>
<a href="/graph/xyzzy?style=spartan">graph</a>
--- a/tests/test-hgwebdir.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-hgwebdir.t Tue Feb 28 11:13:25 2017 -0800
@@ -1421,7 +1421,7 @@
> EOF
$ hg serve -d --pid-file=hg.pid --web-conf paths.conf \
> -A access-paths.log -E error-paths-9.log
- listening at http://*:$HGPORT1/ (bound to 127.0.0.1:$HGPORT1) (glob)
+ listening at http://*:$HGPORT1/ (bound to *$LOCALIP*:$HGPORT1) (glob)
$ cat hg.pid >> $DAEMON_PIDS
$ get-with-headers.py localhost:$HGPORT1 '?style=raw'
200 Script output follows
@@ -1433,7 +1433,7 @@
$ killdaemons.py
$ hg serve -p $HGPORT2 -d -v --pid-file=hg.pid --web-conf paths.conf \
> -A access-paths.log -E error-paths-10.log
- listening at http://*:$HGPORT2/ (bound to 127.0.0.1:$HGPORT2) (glob)
+ listening at http://*:$HGPORT2/ (bound to *$LOCALIP*:$HGPORT2) (glob)
$ cat hg.pid >> $DAEMON_PIDS
$ get-with-headers.py localhost:$HGPORT2 '?style=raw'
200 Script output follows
--- a/tests/test-histedit-arguments.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-histedit-arguments.t Tue Feb 28 11:13:25 2017 -0800
@@ -72,7 +72,7 @@
# p, pick = use commit
# d, drop = remove commit from history
# f, fold = use commit, but combine it with the one above
- # r, roll = like fold, but discard this commit's description
+ # r, roll = like fold, but discard this commit's description and date
#
Run on a revision not ancestors of the current working directory.
@@ -308,7 +308,7 @@
# p, pick = use commit
# d, drop = remove commit from history
# f, fold = use commit, but combine it with the one above
- # r, roll = like fold, but discard this commit's description
+ # r, roll = like fold, but discard this commit's description and date
#
Test --continue with --keep
@@ -544,7 +544,7 @@
# p, pick = use commit
# d, drop = remove commit from history
# f, fold = use commit, but combine it with the one above
- # r, roll = like fold, but discard this commit's description
+ # r, roll = like fold, but discard this commit's description and date
#
$ cd ..
--- a/tests/test-histedit-bookmark-motion.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-histedit-bookmark-motion.t Tue Feb 28 11:13:25 2017 -0800
@@ -78,7 +78,7 @@
# p, pick = use commit
# d, drop = remove commit from history
# f, fold = use commit, but combine it with the one above
- # r, roll = like fold, but discard this commit's description
+ # r, roll = like fold, but discard this commit's description and date
#
$ hg histedit 1 --commands - --verbose << EOF | grep histedit
> pick 177f92b77385 2 c
@@ -141,7 +141,7 @@
# p, pick = use commit
# d, drop = remove commit from history
# f, fold = use commit, but combine it with the one above
- # r, roll = like fold, but discard this commit's description
+ # r, roll = like fold, but discard this commit's description and date
#
$ hg histedit 1 --commands - --verbose << EOF | grep histedit
> pick b346ab9a313d 1 c
--- a/tests/test-histedit-commute.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-histedit-commute.t Tue Feb 28 11:13:25 2017 -0800
@@ -72,7 +72,7 @@
# p, pick = use commit
# d, drop = remove commit from history
# f, fold = use commit, but combine it with the one above
- # r, roll = like fold, but discard this commit's description
+ # r, roll = like fold, but discard this commit's description and date
#
edit the history
@@ -350,7 +350,7 @@
# p, pick = use commit
# d, drop = remove commit from history
# f, fold = use commit, but combine it with the one above
- # r, roll = like fold, but discard this commit's description
+ # r, roll = like fold, but discard this commit's description and date
#
should also work if a commit message is missing
--- a/tests/test-histedit-edit.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-histedit-edit.t Tue Feb 28 11:13:25 2017 -0800
@@ -478,5 +478,5 @@
# p, fold = use commit
# d, drop = remove commit from history
# f, fold = use commit, but combine it with the one above
- # r, roll = like fold, but discard this commit's description
+ # r, roll = like fold, but discard this commit's description and date
#
--- a/tests/test-histedit-fold-non-commute.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-histedit-fold-non-commute.t Tue Feb 28 11:13:25 2017 -0800
@@ -5,6 +5,12 @@
> histedit=
> EOF
+ $ modwithdate ()
+ > {
+ > echo $1 > $1
+ > hg ci -m $1 -d "$2 0"
+ > }
+
$ initrepo ()
> {
> hg init $1
@@ -14,12 +20,14 @@
> hg add $x
> done
> hg ci -m 'Initial commit'
- > for x in a b c d e f ; do
- > echo $x > $x
- > hg ci -m $x
- > done
+ > modwithdate a 1
+ > modwithdate b 2
+ > modwithdate c 3
+ > modwithdate d 4
+ > modwithdate e 5
+ > modwithdate f 6
> echo 'I can haz no commute' > e
- > hg ci -m 'does not commute with e'
+ > hg ci -m 'does not commute with e' -d '7 0'
> cd ..
> }
@@ -34,48 +42,48 @@
$ hg log --template 'pick {node|short} {rev} {desc}\n' -r 5 >> $EDITED
$ hg log --template 'pick {node|short} {rev} {desc}\n' -r 6 >> $EDITED
$ cat $EDITED
- pick 65a9a84f33fd 3 c
- pick 00f1c5383965 4 d
- fold 39522b764e3d 7 does not commute with e
- pick 7b4e2f4b7bcd 5 e
- pick 500cac37a696 6 f
+ pick 092e4ce14829 3 c
+ pick ae78f4c9d74f 4 d
+ fold 42abbb61bede 7 does not commute with e
+ pick 7f3755409b00 5 e
+ pick dd184f2faeb0 6 f
log before edit
$ hg log --graph
- @ changeset: 7:39522b764e3d
+ @ changeset: 7:42abbb61bede
| tag: tip
| user: test
- | date: Thu Jan 01 00:00:00 1970 +0000
+ | date: Thu Jan 01 00:00:07 1970 +0000
| summary: does not commute with e
|
- o changeset: 6:500cac37a696
+ o changeset: 6:dd184f2faeb0
| user: test
- | date: Thu Jan 01 00:00:00 1970 +0000
+ | date: Thu Jan 01 00:00:06 1970 +0000
| summary: f
|
- o changeset: 5:7b4e2f4b7bcd
+ o changeset: 5:7f3755409b00
| user: test
- | date: Thu Jan 01 00:00:00 1970 +0000
+ | date: Thu Jan 01 00:00:05 1970 +0000
| summary: e
|
- o changeset: 4:00f1c5383965
+ o changeset: 4:ae78f4c9d74f
| user: test
- | date: Thu Jan 01 00:00:00 1970 +0000
+ | date: Thu Jan 01 00:00:04 1970 +0000
| summary: d
|
- o changeset: 3:65a9a84f33fd
+ o changeset: 3:092e4ce14829
| user: test
- | date: Thu Jan 01 00:00:00 1970 +0000
+ | date: Thu Jan 01 00:00:03 1970 +0000
| summary: c
|
- o changeset: 2:da6535b52e45
+ o changeset: 2:40ccdd8beb95
| user: test
- | date: Thu Jan 01 00:00:00 1970 +0000
+ | date: Thu Jan 01 00:00:02 1970 +0000
| summary: b
|
- o changeset: 1:c1f09da44841
+ o changeset: 1:cd997a145b29
| user: test
- | date: Thu Jan 01 00:00:00 1970 +0000
+ | date: Thu Jan 01 00:00:01 1970 +0000
| summary: a
|
o changeset: 0:1715188a53c7
@@ -89,7 +97,7 @@
2 files updated, 0 files merged, 0 files removed, 0 files unresolved
merging e
warning: conflicts while merging e! (edit, then use 'hg resolve --mark')
- Fix up the change (fold 39522b764e3d)
+ Fix up the change (fold 42abbb61bede)
(hg histedit --continue to resume)
fix up
@@ -113,7 +121,7 @@
HG: changed e
merging e
warning: conflicts while merging e! (edit, then use 'hg resolve --mark')
- Fix up the change (pick 7b4e2f4b7bcd)
+ Fix up the change (pick 7f3755409b00)
(hg histedit --continue to resume)
just continue this time
@@ -124,34 +132,34 @@
continue: hg histedit --continue
$ hg diff
$ hg histedit --continue 2>&1 | fixbundle
- 7b4e2f4b7bcd: skipping changeset (no changes)
+ 7f3755409b00: skipping changeset (no changes)
log after edit
$ hg log --graph
- @ changeset: 5:d9cf42e54966
+ @ changeset: 5:1300355b1a54
| tag: tip
| user: test
- | date: Thu Jan 01 00:00:00 1970 +0000
+ | date: Thu Jan 01 00:00:06 1970 +0000
| summary: f
|
- o changeset: 4:10486af2e984
+ o changeset: 4:e2ac33269083
| user: test
- | date: Thu Jan 01 00:00:00 1970 +0000
+ | date: Thu Jan 01 00:00:07 1970 +0000
| summary: d
|
- o changeset: 3:65a9a84f33fd
+ o changeset: 3:092e4ce14829
| user: test
- | date: Thu Jan 01 00:00:00 1970 +0000
+ | date: Thu Jan 01 00:00:03 1970 +0000
| summary: c
|
- o changeset: 2:da6535b52e45
+ o changeset: 2:40ccdd8beb95
| user: test
- | date: Thu Jan 01 00:00:00 1970 +0000
+ | date: Thu Jan 01 00:00:02 1970 +0000
| summary: b
|
- o changeset: 1:c1f09da44841
+ o changeset: 1:cd997a145b29
| user: test
- | date: Thu Jan 01 00:00:00 1970 +0000
+ | date: Thu Jan 01 00:00:01 1970 +0000
| summary: a
|
o changeset: 0:1715188a53c7
@@ -175,7 +183,7 @@
$ cd ..
-Repeat test using "roll", not "fold". "roll" folds in changes but drops message
+Repeat test using "roll", not "fold". "roll" folds in changes but drops message and date
$ initrepo r2
$ cd r2
@@ -189,48 +197,48 @@
$ hg log --template 'pick {node|short} {rev} {desc}\n' -r 5 >> $EDITED
$ hg log --template 'pick {node|short} {rev} {desc}\n' -r 6 >> $EDITED
$ cat $EDITED
- pick 65a9a84f33fd 3 c
- pick 00f1c5383965 4 d
- roll 39522b764e3d 7 does not commute with e
- pick 7b4e2f4b7bcd 5 e
- pick 500cac37a696 6 f
+ pick 092e4ce14829 3 c
+ pick ae78f4c9d74f 4 d
+ roll 42abbb61bede 7 does not commute with e
+ pick 7f3755409b00 5 e
+ pick dd184f2faeb0 6 f
log before edit
$ hg log --graph
- @ changeset: 7:39522b764e3d
+ @ changeset: 7:42abbb61bede
| tag: tip
| user: test
- | date: Thu Jan 01 00:00:00 1970 +0000
+ | date: Thu Jan 01 00:00:07 1970 +0000
| summary: does not commute with e
|
- o changeset: 6:500cac37a696
+ o changeset: 6:dd184f2faeb0
| user: test
- | date: Thu Jan 01 00:00:00 1970 +0000
+ | date: Thu Jan 01 00:00:06 1970 +0000
| summary: f
|
- o changeset: 5:7b4e2f4b7bcd
+ o changeset: 5:7f3755409b00
| user: test
- | date: Thu Jan 01 00:00:00 1970 +0000
+ | date: Thu Jan 01 00:00:05 1970 +0000
| summary: e
|
- o changeset: 4:00f1c5383965
+ o changeset: 4:ae78f4c9d74f
| user: test
- | date: Thu Jan 01 00:00:00 1970 +0000
+ | date: Thu Jan 01 00:00:04 1970 +0000
| summary: d
|
- o changeset: 3:65a9a84f33fd
+ o changeset: 3:092e4ce14829
| user: test
- | date: Thu Jan 01 00:00:00 1970 +0000
+ | date: Thu Jan 01 00:00:03 1970 +0000
| summary: c
|
- o changeset: 2:da6535b52e45
+ o changeset: 2:40ccdd8beb95
| user: test
- | date: Thu Jan 01 00:00:00 1970 +0000
+ | date: Thu Jan 01 00:00:02 1970 +0000
| summary: b
|
- o changeset: 1:c1f09da44841
+ o changeset: 1:cd997a145b29
| user: test
- | date: Thu Jan 01 00:00:00 1970 +0000
+ | date: Thu Jan 01 00:00:01 1970 +0000
| summary: a
|
o changeset: 0:1715188a53c7
@@ -244,7 +252,7 @@
2 files updated, 0 files merged, 0 files removed, 0 files unresolved
merging e
warning: conflicts while merging e! (edit, then use 'hg resolve --mark')
- Fix up the change (roll 39522b764e3d)
+ Fix up the change (roll 42abbb61bede)
(hg histedit --continue to resume)
fix up
@@ -255,7 +263,7 @@
$ hg histedit --continue 2>&1 | fixbundle | grep -v '2 files removed'
merging e
warning: conflicts while merging e! (edit, then use 'hg resolve --mark')
- Fix up the change (pick 7b4e2f4b7bcd)
+ Fix up the change (pick 7f3755409b00)
(hg histedit --continue to resume)
just continue this time
@@ -264,34 +272,34 @@
(no more unresolved files)
continue: hg histedit --continue
$ hg histedit --continue 2>&1 | fixbundle
- 7b4e2f4b7bcd: skipping changeset (no changes)
+ 7f3755409b00: skipping changeset (no changes)
log after edit
$ hg log --graph
- @ changeset: 5:e7c4f5d4eb75
+ @ changeset: 5:b538bcb461be
| tag: tip
| user: test
- | date: Thu Jan 01 00:00:00 1970 +0000
+ | date: Thu Jan 01 00:00:06 1970 +0000
| summary: f
|
- o changeset: 4:803d1bb561fc
+ o changeset: 4:317e37cb6d66
| user: test
- | date: Thu Jan 01 00:00:00 1970 +0000
+ | date: Thu Jan 01 00:00:04 1970 +0000
| summary: d
|
- o changeset: 3:65a9a84f33fd
+ o changeset: 3:092e4ce14829
| user: test
- | date: Thu Jan 01 00:00:00 1970 +0000
+ | date: Thu Jan 01 00:00:03 1970 +0000
| summary: c
|
- o changeset: 2:da6535b52e45
+ o changeset: 2:40ccdd8beb95
| user: test
- | date: Thu Jan 01 00:00:00 1970 +0000
+ | date: Thu Jan 01 00:00:02 1970 +0000
| summary: b
|
- o changeset: 1:c1f09da44841
+ o changeset: 1:cd997a145b29
| user: test
- | date: Thu Jan 01 00:00:00 1970 +0000
+ | date: Thu Jan 01 00:00:01 1970 +0000
| summary: a
|
o changeset: 0:1715188a53c7
@@ -316,16 +324,16 @@
description is taken from rollup target commit
$ hg log --debug --rev 4
- changeset: 4:803d1bb561fceac3129ec778db9da249a3106fc3
+ changeset: 4:317e37cb6d66c1c84628c00e5bf4c8c292831951
phase: draft
- parent: 3:65a9a84f33fdeb1ad5679b3941ec885d2b24027b
+ parent: 3:092e4ce14829f4974399ce4316d59f64ef0b6725
parent: -1:0000000000000000000000000000000000000000
manifest: 4:b068a323d969f22af1296ec6a5ea9384cef437ac
user: test
- date: Thu Jan 01 00:00:00 1970 +0000
+ date: Thu Jan 01 00:00:04 1970 +0000
files: d e
extra: branch=default
- extra: histedit_source=00f1c53839651fa5c76d423606811ea5455a79d0,39522b764e3d26103f08bd1fa2ccd3e3d7dbcf4e
+ extra: histedit_source=ae78f4c9d74ffa4b6cb5045001c303fe9204e890,42abbb61bede6f4366fa1e74a664343e5d558a70
description:
d
--- a/tests/test-histedit-fold.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-histedit-fold.t Tue Feb 28 11:13:25 2017 -0800
@@ -20,52 +20,60 @@
Simple folding
--------------------
+ $ addwithdate ()
+ > {
+ > echo $1 > $1
+ > hg add $1
+ > hg ci -m $1 -d "$2 0"
+ > }
+
$ initrepo ()
> {
> hg init r
> cd r
- > for x in a b c d e f ; do
- > echo $x > $x
- > hg add $x
- > hg ci -m $x
- > done
+ > addwithdate a 1
+ > addwithdate b 2
+ > addwithdate c 3
+ > addwithdate d 4
+ > addwithdate e 5
+ > addwithdate f 6
> }
$ initrepo
log before edit
$ hg logt --graph
- @ 5:652413bf663e f
+ @ 5:178e35e0ce73 f
|
- o 4:e860deea161a e
+ o 4:1ddb6c90f2ee e
|
- o 3:055a42cdd887 d
+ o 3:532247a8969b d
|
- o 2:177f92b77385 c
+ o 2:ff2c9fa2018b c
|
- o 1:d2ae7f538514 b
+ o 1:97d72e5f12c7 b
|
- o 0:cb9a9f314b8b a
+ o 0:8580ff50825a a
- $ hg histedit 177f92b77385 --commands - 2>&1 <<EOF | fixbundle
- > pick e860deea161a e
- > pick 652413bf663e f
- > fold 177f92b77385 c
- > pick 055a42cdd887 d
+ $ hg histedit ff2c9fa2018b --commands - 2>&1 <<EOF | fixbundle
+ > pick 1ddb6c90f2ee e
+ > pick 178e35e0ce73 f
+ > fold ff2c9fa2018b c
+ > pick 532247a8969b d
> EOF
log after edit
$ hg logt --graph
- @ 4:9c277da72c9b d
+ @ 4:c4d7f3def76d d
|
- o 3:6de59d13424a f
+ o 3:575228819b7e f
|
- o 2:ee283cb5f2d5 e
+ o 2:505a591af19e e
|
- o 1:d2ae7f538514 b
+ o 1:97d72e5f12c7 b
|
- o 0:cb9a9f314b8b a
+ o 0:8580ff50825a a
post-fold manifest
@@ -78,19 +86,19 @@
f
-check histedit_source
+check histedit_source, including that it uses the later date, from the first changeset
$ hg log --debug --rev 3
- changeset: 3:6de59d13424a8a13acd3e975514aed29dd0d9b2d
+ changeset: 3:575228819b7e6ed69e8c0a6a383ee59a80db7358
phase: draft
- parent: 2:ee283cb5f2d5955443f23a27b697a04339e9a39a
+ parent: 2:505a591af19eed18f560af827b9e03d2076773dc
parent: -1:0000000000000000000000000000000000000000
manifest: 3:81eede616954057198ead0b2c73b41d1f392829a
user: test
- date: Thu Jan 01 00:00:00 1970 +0000
+ date: Thu Jan 01 00:00:06 1970 +0000
files+: c f
extra: branch=default
- extra: histedit_source=a4f7421b80f79fcc59fff01bcbf4a53d127dd6d3,177f92b773850b59254aa5e923436f921b55483b
+ extra: histedit_source=7cad1d7030207872dfd1c3a7cb430f24f2884086,ff2c9fa2018b15fa74b33363bda9527323e2a99f
description:
f
***
@@ -98,43 +106,43 @@
-rollup will fold without preserving the folded commit's message
+rollup will fold without preserving the folded commit's message or date
$ OLDHGEDITOR=$HGEDITOR
$ HGEDITOR=false
- $ hg histedit d2ae7f538514 --commands - 2>&1 <<EOF | fixbundle
- > pick d2ae7f538514 b
- > roll ee283cb5f2d5 e
- > pick 6de59d13424a f
- > pick 9c277da72c9b d
+ $ hg histedit 97d72e5f12c7 --commands - 2>&1 <<EOF | fixbundle
+ > pick 97d72e5f12c7 b
+ > roll 505a591af19e e
+ > pick 575228819b7e f
+ > pick c4d7f3def76d d
> EOF
$ HGEDITOR=$OLDHGEDITOR
log after edit
$ hg logt --graph
- @ 3:c4a9eb7989fc d
+ @ 3:bab801520cec d
|
- o 2:8e03a72b6f83 f
+ o 2:58c8f2bfc151 f
|
- o 1:391ee782c689 b
+ o 1:5d939c56c72e b
|
- o 0:cb9a9f314b8b a
+ o 0:8580ff50825a a
description is taken from rollup target commit
$ hg log --debug --rev 1
- changeset: 1:391ee782c68930be438ccf4c6a403daedbfbffa5
+ changeset: 1:5d939c56c72e77e29f5167696218e2131a40f5cf
phase: draft
- parent: 0:cb9a9f314b8b07ba71012fcdbc544b5a4d82ff5b
+ parent: 0:8580ff50825a50c8f716709acdf8de0deddcd6ab
parent: -1:0000000000000000000000000000000000000000
manifest: 1:b5e112a3a8354e269b1524729f0918662d847c38
user: test
- date: Thu Jan 01 00:00:00 1970 +0000
+ date: Thu Jan 01 00:00:02 1970 +0000
files+: b e
extra: branch=default
- extra: histedit_source=d2ae7f538514cd87c17547b0de4cea71fe1af9fb,ee283cb5f2d5955443f23a27b697a04339e9a39a
+ extra: histedit_source=97d72e5f12c7e84f85064aa72e5a297142c36ed9,505a591af19eed18f560af827b9e03d2076773dc
description:
b
@@ -163,13 +171,13 @@
> EOF
$ rm -f .hg/last-message.txt
- $ hg status --rev '8e03a72b6f83^1::c4a9eb7989fc'
+ $ hg status --rev '58c8f2bfc151^1::bab801520cec'
A c
A d
A f
- $ HGEDITOR="sh $TESTTMP/editor.sh" hg histedit 8e03a72b6f83 --commands - 2>&1 <<EOF
- > pick 8e03a72b6f83 f
- > fold c4a9eb7989fc d
+ $ HGEDITOR="sh $TESTTMP/editor.sh" hg histedit 58c8f2bfc151 --commands - 2>&1 <<EOF
+ > pick 58c8f2bfc151 f
+ > fold bab801520cec d
> EOF
allow non-folding commit
==== before editing
@@ -209,37 +217,37 @@
$ cd ..
$ rm -r r
-folding preserves initial author
---------------------------------
+folding preserves initial author but uses later date
+----------------------------------------------------
$ initrepo
- $ hg ci --user "someone else" --amend --quiet
+ $ hg ci -d '7 0' --user "someone else" --amend --quiet
tip before edit
$ hg log --rev .
- changeset: 5:a00ad806cb55
+ changeset: 5:10c36dd37515
tag: tip
user: someone else
- date: Thu Jan 01 00:00:00 1970 +0000
+ date: Thu Jan 01 00:00:07 1970 +0000
summary: f
$ hg --config progress.debug=1 --debug \
- > histedit e860deea161a --commands - 2>&1 <<EOF | \
+ > histedit 1ddb6c90f2ee --commands - 2>&1 <<EOF | \
> egrep 'editing|unresolved'
- > pick e860deea161a e
- > fold a00ad806cb55 f
+ > pick 1ddb6c90f2ee e
+ > fold 10c36dd37515 f
> EOF
- editing: pick e860deea161a 4 e 1/2 changes (50.00%)
- editing: fold a00ad806cb55 5 f 2/2 changes (100.00%)
+ editing: pick 1ddb6c90f2ee 4 e 1/2 changes (50.00%)
+ editing: fold 10c36dd37515 5 f 2/2 changes (100.00%)
-tip after edit
+tip after edit, which should use the later date, from the second changeset
$ hg log --rev .
- changeset: 4:698d4e8040a1
+ changeset: 4:e4f3ec5d0b40
tag: tip
user: test
- date: Thu Jan 01 00:00:00 1970 +0000
+ date: Thu Jan 01 00:00:07 1970 +0000
summary: e
--- a/tests/test-histedit-obsolete.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-histedit-obsolete.t Tue Feb 28 11:13:25 2017 -0800
@@ -136,7 +136,7 @@
# p, pick = use commit
# d, drop = remove commit from history
# f, fold = use commit, but combine it with the one above
- # r, roll = like fold, but discard this commit's description
+ # r, roll = like fold, but discard this commit's description and date
#
$ hg histedit 1 --commands - --verbose <<EOF | grep histedit
> pick 177f92b77385 2 c
--- a/tests/test-histedit-outgoing.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-histedit-outgoing.t Tue Feb 28 11:13:25 2017 -0800
@@ -54,7 +54,7 @@
# p, pick = use commit
# d, drop = remove commit from history
# f, fold = use commit, but combine it with the one above
- # r, roll = like fold, but discard this commit's description
+ # r, roll = like fold, but discard this commit's description and date
#
$ cd ..
@@ -88,7 +88,7 @@
# p, pick = use commit
# d, drop = remove commit from history
# f, fold = use commit, but combine it with the one above
- # r, roll = like fold, but discard this commit's description
+ # r, roll = like fold, but discard this commit's description and date
#
$ cd ..
@@ -114,7 +114,7 @@
# p, pick = use commit
# d, drop = remove commit from history
# f, fold = use commit, but combine it with the one above
- # r, roll = like fold, but discard this commit's description
+ # r, roll = like fold, but discard this commit's description and date
#
test to check number of roots in outgoing revisions
--- a/tests/test-hook.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-hook.t Tue Feb 28 11:13:25 2017 -0800
@@ -832,6 +832,50 @@
[1]
$ cd ..
+check whether HG_PENDING makes pending changes only in related
+repositories visible to an external hook.
+
+(emulate a transaction running concurrently by copied
+.hg/store/00changelog.i.a in subsequent test)
+
+ $ cat > $TESTTMP/savepending.sh <<EOF
+ > cp .hg/store/00changelog.i.a .hg/store/00changelog.i.a.saved
+ > exit 1 # to avoid adding new revision for subsequent tests
+ > EOF
+ $ cd a
+ $ hg tip -q
+ 4:539e4b31b6dc
+ $ hg --config hooks.pretxnclose="sh $TESTTMP/savepending.sh" commit -m "invisible"
+ transaction abort!
+ rollback completed
+ abort: pretxnclose hook exited with status 1
+ [255]
+ $ cp .hg/store/00changelog.i.a.saved .hg/store/00changelog.i.a
+
+(check (in)visibility of new changeset while transaction running in
+repo)
+
+ $ cat > $TESTTMP/checkpending.sh <<EOF
+ > echo '@a'
+ > hg -R $TESTTMP/a tip -q
+ > echo '@a/nested'
+ > hg -R $TESTTMP/a/nested tip -q
+ > exit 1 # to avoid adding new revision for subsequent tests
+ > EOF
+ $ hg init nested
+ $ cd nested
+ $ echo a > a
+ $ hg add a
+ $ hg --config hooks.pretxnclose="sh $TESTTMP/checkpending.sh" commit -m '#0'
+ @a
+ 4:539e4b31b6dc
+ @a/nested
+ 0:bf5e395ced2c
+ transaction abort!
+ rollback completed
+ abort: pretxnclose hook exited with status 1
+ [255]
+
Hook from untrusted hgrc are reported as failure
================================================
--- a/tests/test-http-bundle1.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-http-bundle1.t Tue Feb 28 11:13:25 2017 -0800
@@ -28,11 +28,11 @@
#if windows
$ hg serve -p $HGPORT1 2>&1
- abort: cannot start server at ':$HGPORT1': * (glob)
+ abort: cannot start server at 'localhost:$HGPORT1': * (glob)
[255]
#else
$ hg serve -p $HGPORT1 2>&1
- abort: cannot start server at ':$HGPORT1': Address already in use
+ abort: cannot start server at 'localhost:$HGPORT1': Address already in use
[255]
#endif
$ cd ..
--- a/tests/test-http-protocol.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-http-protocol.t Tue Feb 28 11:13:25 2017 -0800
@@ -16,9 +16,9 @@
compression formats are advertised in compression capability
#if zstd
- $ get-with-headers.py 127.0.0.1:$HGPORT '?cmd=capabilities' | tr ' ' '\n' | grep '^compression=zstd,zlib$' > /dev/null
+ $ get-with-headers.py $LOCALIP:$HGPORT '?cmd=capabilities' | tr ' ' '\n' | grep '^compression=zstd,zlib$' > /dev/null
#else
- $ get-with-headers.py 127.0.0.1:$HGPORT '?cmd=capabilities' | tr ' ' '\n' | grep '^compression=zlib$' > /dev/null
+ $ get-with-headers.py $LOCALIP:$HGPORT '?cmd=capabilities' | tr ' ' '\n' | grep '^compression=zlib$' > /dev/null
#endif
$ killdaemons.py
@@ -27,7 +27,7 @@
$ hg --config server.compressionengines=none -R server serve -p $HGPORT -d --pid-file hg.pid
$ cat hg.pid > $DAEMON_PIDS
- $ get-with-headers.py 127.0.0.1:$HGPORT '?cmd=capabilities' | tr ' ' '\n' | grep '^compression=none$' > /dev/null
+ $ get-with-headers.py $LOCALIP:$HGPORT '?cmd=capabilities' | tr ' ' '\n' | grep '^compression=none$' > /dev/null
$ killdaemons.py
@@ -35,7 +35,7 @@
$ hg --config server.compressionengines=none,zlib -R server serve -p $HGPORT -d --pid-file hg.pid
$ cat hg.pid > $DAEMON_PIDS
- $ get-with-headers.py 127.0.0.1:$HGPORT '?cmd=capabilities' | tr ' ' '\n' | grep '^compression=none,zlib$' > /dev/null
+ $ get-with-headers.py $LOCALIP:$HGPORT '?cmd=capabilities' | tr ' ' '\n' | grep '^compression=none,zlib$' > /dev/null
$ killdaemons.py
@@ -46,7 +46,7 @@
Server should send application/mercurial-0.1 to clients if no Accept is used
- $ get-with-headers.py --headeronly 127.0.0.1:$HGPORT '?cmd=getbundle&heads=e93700bd72895c5addab234c56d4024b487a362f&common=0000000000000000000000000000000000000000' -
+ $ get-with-headers.py --headeronly $LOCALIP:$HGPORT '?cmd=getbundle&heads=e93700bd72895c5addab234c56d4024b487a362f&common=0000000000000000000000000000000000000000' -
200 Script output follows
content-type: application/mercurial-0.1
date: * (glob)
@@ -55,7 +55,7 @@
Server should send application/mercurial-0.1 when client says it wants it
- $ get-with-headers.py --hgproto '0.1' --headeronly 127.0.0.1:$HGPORT '?cmd=getbundle&heads=e93700bd72895c5addab234c56d4024b487a362f&common=0000000000000000000000000000000000000000' -
+ $ get-with-headers.py --hgproto '0.1' --headeronly $LOCALIP:$HGPORT '?cmd=getbundle&heads=e93700bd72895c5addab234c56d4024b487a362f&common=0000000000000000000000000000000000000000' -
200 Script output follows
content-type: application/mercurial-0.1
date: * (glob)
@@ -64,14 +64,14 @@
Server should send application/mercurial-0.2 when client says it wants it
- $ get-with-headers.py --hgproto '0.2' --headeronly 127.0.0.1:$HGPORT '?cmd=getbundle&heads=e93700bd72895c5addab234c56d4024b487a362f&common=0000000000000000000000000000000000000000' -
+ $ get-with-headers.py --hgproto '0.2' --headeronly $LOCALIP:$HGPORT '?cmd=getbundle&heads=e93700bd72895c5addab234c56d4024b487a362f&common=0000000000000000000000000000000000000000' -
200 Script output follows
content-type: application/mercurial-0.2
date: * (glob)
server: * (glob)
transfer-encoding: chunked
- $ get-with-headers.py --hgproto '0.1 0.2' --headeronly 127.0.0.1:$HGPORT '?cmd=getbundle&heads=e93700bd72895c5addab234c56d4024b487a362f&common=0000000000000000000000000000000000000000' -
+ $ get-with-headers.py --hgproto '0.1 0.2' --headeronly $LOCALIP:$HGPORT '?cmd=getbundle&heads=e93700bd72895c5addab234c56d4024b487a362f&common=0000000000000000000000000000000000000000' -
200 Script output follows
content-type: application/mercurial-0.2
date: * (glob)
@@ -80,7 +80,7 @@
Requesting a compression format that server doesn't support results will fall back to 0.1
- $ get-with-headers.py --hgproto '0.2 comp=aa' --headeronly 127.0.0.1:$HGPORT '?cmd=getbundle&heads=e93700bd72895c5addab234c56d4024b487a362f&common=0000000000000000000000000000000000000000' -
+ $ get-with-headers.py --hgproto '0.2 comp=aa' --headeronly $LOCALIP:$HGPORT '?cmd=getbundle&heads=e93700bd72895c5addab234c56d4024b487a362f&common=0000000000000000000000000000000000000000' -
200 Script output follows
content-type: application/mercurial-0.1
date: * (glob)
@@ -90,7 +90,7 @@
#if zstd
zstd is used if available
- $ get-with-headers.py --hgproto '0.2 comp=zstd' 127.0.0.1:$HGPORT '?cmd=getbundle&heads=e93700bd72895c5addab234c56d4024b487a362f&common=0000000000000000000000000000000000000000' > resp
+ $ get-with-headers.py --hgproto '0.2 comp=zstd' $LOCALIP:$HGPORT '?cmd=getbundle&heads=e93700bd72895c5addab234c56d4024b487a362f&common=0000000000000000000000000000000000000000' > resp
$ f --size --hexdump --bytes 36 --sha1 resp
resp: size=248, sha1=4d8d8f87fb82bd542ce52881fdc94f850748
0000: 32 30 30 20 53 63 72 69 70 74 20 6f 75 74 70 75 |200 Script outpu|
@@ -101,7 +101,7 @@
application/mercurial-0.2 is not yet used on non-streaming responses
- $ get-with-headers.py --hgproto '0.2' 127.0.0.1:$HGPORT '?cmd=heads' -
+ $ get-with-headers.py --hgproto '0.2' $LOCALIP:$HGPORT '?cmd=heads' -
200 Script output follows
content-length: 41
content-type: application/mercurial-0.1
@@ -118,11 +118,11 @@
No Accept will send 0.1+zlib, even though "none" is preferred b/c "none" isn't supported on 0.1
- $ get-with-headers.py --headeronly 127.0.0.1:$HGPORT '?cmd=getbundle&heads=e93700bd72895c5addab234c56d4024b487a362f&common=0000000000000000000000000000000000000000' Content-Type
+ $ get-with-headers.py --headeronly $LOCALIP:$HGPORT '?cmd=getbundle&heads=e93700bd72895c5addab234c56d4024b487a362f&common=0000000000000000000000000000000000000000' Content-Type
200 Script output follows
content-type: application/mercurial-0.1
- $ get-with-headers.py 127.0.0.1:$HGPORT '?cmd=getbundle&heads=e93700bd72895c5addab234c56d4024b487a362f&common=0000000000000000000000000000000000000000' > resp
+ $ get-with-headers.py $LOCALIP:$HGPORT '?cmd=getbundle&heads=e93700bd72895c5addab234c56d4024b487a362f&common=0000000000000000000000000000000000000000' > resp
$ f --size --hexdump --bytes 28 --sha1 resp
resp: size=227, sha1=35a4c074da74f32f5440da3cbf04
0000: 32 30 30 20 53 63 72 69 70 74 20 6f 75 74 70 75 |200 Script outpu|
@@ -130,7 +130,7 @@
Explicit 0.1 will send zlib because "none" isn't supported on 0.1
- $ get-with-headers.py --hgproto '0.1' 127.0.0.1:$HGPORT '?cmd=getbundle&heads=e93700bd72895c5addab234c56d4024b487a362f&common=0000000000000000000000000000000000000000' > resp
+ $ get-with-headers.py --hgproto '0.1' $LOCALIP:$HGPORT '?cmd=getbundle&heads=e93700bd72895c5addab234c56d4024b487a362f&common=0000000000000000000000000000000000000000' > resp
$ f --size --hexdump --bytes 28 --sha1 resp
resp: size=227, sha1=35a4c074da74f32f5440da3cbf04
0000: 32 30 30 20 53 63 72 69 70 74 20 6f 75 74 70 75 |200 Script outpu|
@@ -139,7 +139,7 @@
0.2 with no compression will get "none" because that is server's preference
(spec says ZL and UN are implicitly supported)
- $ get-with-headers.py --hgproto '0.2' 127.0.0.1:$HGPORT '?cmd=getbundle&heads=e93700bd72895c5addab234c56d4024b487a362f&common=0000000000000000000000000000000000000000' > resp
+ $ get-with-headers.py --hgproto '0.2' $LOCALIP:$HGPORT '?cmd=getbundle&heads=e93700bd72895c5addab234c56d4024b487a362f&common=0000000000000000000000000000000000000000' > resp
$ f --size --hexdump --bytes 32 --sha1 resp
resp: size=432, sha1=ac931b412ec185a02e0e5bcff98dac83
0000: 32 30 30 20 53 63 72 69 70 74 20 6f 75 74 70 75 |200 Script outpu|
@@ -147,7 +147,7 @@
Client receives server preference even if local order doesn't match
- $ get-with-headers.py --hgproto '0.2 comp=zlib,none' 127.0.0.1:$HGPORT '?cmd=getbundle&heads=e93700bd72895c5addab234c56d4024b487a362f&common=0000000000000000000000000000000000000000' > resp
+ $ get-with-headers.py --hgproto '0.2 comp=zlib,none' $LOCALIP:$HGPORT '?cmd=getbundle&heads=e93700bd72895c5addab234c56d4024b487a362f&common=0000000000000000000000000000000000000000' > resp
$ f --size --hexdump --bytes 32 --sha1 resp
resp: size=432, sha1=ac931b412ec185a02e0e5bcff98dac83
0000: 32 30 30 20 53 63 72 69 70 74 20 6f 75 74 70 75 |200 Script outpu|
@@ -155,7 +155,7 @@
Client receives only supported format even if not server preferred format
- $ get-with-headers.py --hgproto '0.2 comp=zlib' 127.0.0.1:$HGPORT '?cmd=getbundle&heads=e93700bd72895c5addab234c56d4024b487a362f&common=0000000000000000000000000000000000000000' > resp
+ $ get-with-headers.py --hgproto '0.2 comp=zlib' $LOCALIP:$HGPORT '?cmd=getbundle&heads=e93700bd72895c5addab234c56d4024b487a362f&common=0000000000000000000000000000000000000000' > resp
$ f --size --hexdump --bytes 33 --sha1 resp
resp: size=232, sha1=a1c727f0c9693ca15742a75c30419bc36
0000: 32 30 30 20 53 63 72 69 70 74 20 6f 75 74 70 75 |200 Script outpu|
--- a/tests/test-http.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-http.t Tue Feb 28 11:13:25 2017 -0800
@@ -23,7 +23,7 @@
[255]
#else
$ hg serve -p $HGPORT1 2>&1
- abort: cannot start server at ':$HGPORT1': Address already in use
+ abort: cannot start server at 'localhost:$HGPORT1': Address already in use
[255]
#endif
$ cd ..
--- a/tests/test-https.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-https.t Tue Feb 28 11:13:25 2017 -0800
@@ -36,11 +36,11 @@
#if windows
$ hg serve -p $HGPORT --certificate=$PRIV 2>&1
- abort: cannot start server at ':$HGPORT':
+ abort: cannot start server at 'localhost:$HGPORT':
[255]
#else
$ hg serve -p $HGPORT --certificate=$PRIV 2>&1
- abort: cannot start server at ':$HGPORT': Address already in use
+ abort: cannot start server at 'localhost:$HGPORT': Address already in use
[255]
#endif
$ cd ..
@@ -278,17 +278,17 @@
cacert mismatch
$ hg -R copy-pull pull --config web.cacerts="$CERTSDIR/pub.pem" \
- > https://127.0.0.1:$HGPORT/
- pulling from https://127.0.0.1:$HGPORT/ (glob)
- warning: connecting to 127.0.0.1 using legacy security technology (TLS 1.0); see https://mercurial-scm.org/wiki/SecureConnections for more info (?)
- abort: 127.0.0.1 certificate error: certificate is for localhost (glob)
- (set hostsecurity.127.0.0.1:certfingerprints=sha256:20:de:b3:ad:b4:cd:a5:42:f0:74:41:1c:a2:70:1e:da:6e:c0:5c:16:9e:e7:22:0f:f1:b7:e5:6e:e4:92:af:7e config setting or use --insecure to connect insecurely) (glob)
+ > https://$LOCALIP:$HGPORT/
+ pulling from https://*:$HGPORT/ (glob)
+ warning: connecting to $LOCALIP using legacy security technology (TLS 1.0); see https://mercurial-scm.org/wiki/SecureConnections for more info (?)
+ abort: $LOCALIP certificate error: certificate is for localhost
+ (set hostsecurity.$LOCALIP:certfingerprints=sha256:20:de:b3:ad:b4:cd:a5:42:f0:74:41:1c:a2:70:1e:da:6e:c0:5c:16:9e:e7:22:0f:f1:b7:e5:6e:e4:92:af:7e config setting or use --insecure to connect insecurely)
[255]
$ hg -R copy-pull pull --config web.cacerts="$CERTSDIR/pub.pem" \
- > https://127.0.0.1:$HGPORT/ --insecure
- pulling from https://127.0.0.1:$HGPORT/ (glob)
- warning: connecting to 127.0.0.1 using legacy security technology (TLS 1.0); see https://mercurial-scm.org/wiki/SecureConnections for more info (?)
- warning: connection security to 127.0.0.1 is disabled per current settings; communication is susceptible to eavesdropping and tampering (glob)
+ > https://$LOCALIP:$HGPORT/ --insecure
+ pulling from https://*:$HGPORT/ (glob)
+ warning: connecting to $LOCALIP using legacy security technology (TLS 1.0); see https://mercurial-scm.org/wiki/SecureConnections for more info (?)
+ warning: connection security to $LOCALIP is disabled per current settings; communication is susceptible to eavesdropping and tampering
searching for changes
no changes found
$ hg -R copy-pull pull --config web.cacerts="$CERTSDIR/pub-other.pem"
@@ -434,8 +434,8 @@
- ignores that certificate doesn't match hostname
- $ hg -R copy-pull id https://127.0.0.1:$HGPORT/ --config hostfingerprints.127.0.0.1=ecd87cd6b386d04fc1b8b41c9d8f5e168eef1c03
- warning: connecting to 127.0.0.1 using legacy security technology (TLS 1.0); see https://mercurial-scm.org/wiki/SecureConnections for more info (?)
+ $ hg -R copy-pull id https://$LOCALIP:$HGPORT/ --config hostfingerprints.$LOCALIP=ecd87cd6b386d04fc1b8b41c9d8f5e168eef1c03
+ warning: connecting to $LOCALIP using legacy security technology (TLS 1.0); see https://mercurial-scm.org/wiki/SecureConnections for more info (?)
5fed3813f7f5
Ports used by next test. Kill servers.
@@ -571,9 +571,9 @@
warning: connecting to localhost using legacy security technology (TLS 1.0); see https://mercurial-scm.org/wiki/SecureConnections for more info (?)
searching for changes
no changes found
- $ http_proxy=http://localhost:$HGPORT1/ hg -R copy-pull pull https://127.0.0.1:$HGPORT/ --config hostfingerprints.127.0.0.1=ecd87cd6b386d04fc1b8b41c9d8f5e168eef1c03
- pulling from https://127.0.0.1:$HGPORT/ (glob)
- warning: connecting to 127.0.0.1 using legacy security technology (TLS 1.0); see https://mercurial-scm.org/wiki/SecureConnections for more info (?)
+ $ http_proxy=http://localhost:$HGPORT1/ hg -R copy-pull pull https://localhost:$HGPORT/ --config hostfingerprints.localhost=ecd87cd6b386d04fc1b8b41c9d8f5e168eef1c03 --trace
+ pulling from https://*:$HGPORT/ (glob)
+ warning: connecting to localhost using legacy security technology (TLS 1.0); see https://mercurial-scm.org/wiki/SecureConnections for more info (?)
searching for changes
no changes found
--- a/tests/test-i18n.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-i18n.t Tue Feb 28 11:13:25 2017 -0800
@@ -29,14 +29,15 @@
Test keyword search in translated help text:
- $ HGENCODING=UTF-8 LANGUAGE=de hg help -k blättern
+ $ HGENCODING=UTF-8 LANGUAGE=de hg help -k Aktualisiert
Themen:
- extensions Benutzung erweiterter Funktionen
+ subrepos Unterarchive
- Erweiterungen:
+ Befehle:
- pager Verwendet einen externen Pager zum Bl\xc3\xa4ttern in der Ausgabe von Befehlen (esc)
+ pull Ruft \xc3\x84nderungen von der angegebenen Quelle ab (esc)
+ update Aktualisiert das Arbeitsverzeichnis (oder wechselt die Version)
#endif
--- a/tests/test-largefiles-wireproto.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-largefiles-wireproto.t Tue Feb 28 11:13:25 2017 -0800
@@ -347,7 +347,7 @@
searching 2 changesets for largefiles
verified existence of 2 revisions of 2 largefiles
$ tail -1 access.log
- 127.0.0.1 - - [*] "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=statlfile+sha%3D972a1a11f19934401291cc99117ec614933374ce%3Bstatlfile+sha%3Dc801c9cfe94400963fcb683246217d5db77f9a9a x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob)
+ $LOCALIP - - [*] "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=statlfile+sha%3D972a1a11f19934401291cc99117ec614933374ce%3Bstatlfile+sha%3Dc801c9cfe94400963fcb683246217d5db77f9a9a x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob)
$ hg -R batchverifyclone update
getting changed largefiles
2 largefiles updated, 0 removed
@@ -384,7 +384,7 @@
searching 3 changesets for largefiles
verified existence of 3 revisions of 3 largefiles
$ tail -1 access.log
- 127.0.0.1 - - [*] "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=statlfile+sha%3Dc8559c3c9cfb42131794b7d8009230403b9b454c x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob)
+ $LOCALIP - - [*] "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=statlfile+sha%3Dc8559c3c9cfb42131794b7d8009230403b9b454c x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob)
$ killdaemons.py
--- a/tests/test-largefiles.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-largefiles.t Tue Feb 28 11:13:25 2017 -0800
@@ -192,7 +192,7 @@
$ hg serve -d -p $HGPORT --pid-file ../hg.pid
$ cat ../hg.pid >> $DAEMON_PIDS
- $ get-with-headers.py 127.0.0.1:$HGPORT 'file/tip/?style=raw'
+ $ get-with-headers.py $LOCALIP:$HGPORT 'file/tip/?style=raw'
200 Script output follows
@@ -201,7 +201,7 @@
-rw-r--r-- 9 normal3
- $ get-with-headers.py 127.0.0.1:$HGPORT 'file/tip/sub/?style=raw'
+ $ get-with-headers.py $LOCALIP:$HGPORT 'file/tip/sub/?style=raw'
200 Script output follows
--- a/tests/test-logtoprocess.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-logtoprocess.t Tue Feb 28 11:13:25 2017 -0800
@@ -1,3 +1,7 @@
+ATTENTION: logtoprocess runs commands asynchronously. Be sure to append "| cat"
+to hg commands, to wait for the output, if you want to test its output.
+Otherwise the test will be flaky.
+
Test if logtoprocess correctly captures command-related log calls.
$ hg init
@@ -10,6 +14,7 @@
> def foo(ui, repo):
> ui.log('foo', 'a message: %(bar)s\n', bar='spam')
> EOF
+ $ cp $HGRCPATH $HGRCPATH.bak
$ cat >> $HGRCPATH << EOF
> [extensions]
> logtoprocess=
@@ -33,9 +38,8 @@
Running a command triggers both a ui.log('command') and a
ui.log('commandfinish') call. The foo command also uses ui.log.
-Use head to ensure we wait for all lines to be produced, and sort to avoid
-ordering issues between the various processes we spawn:
- $ hg foo | head -n 17 | sort
+Use sort to avoid ordering issues between the various processes we spawn:
+ $ hg foo | cat | sort
@@ -52,3 +56,18 @@
logtoprocess commandfinish output:
logtoprocess foo output:
spam
+
+Confirm that logging blocked time catches stdio properly:
+ $ cp $HGRCPATH.bak $HGRCPATH
+ $ cat >> $HGRCPATH << EOF
+ > [extensions]
+ > logtoprocess=
+ > pager=
+ > [logtoprocess]
+ > uiblocked=echo "\$EVENT stdio \$OPT_STDIO_BLOCKED ms command \$OPT_COMMAND_DURATION ms"
+ > [ui]
+ > logblockedtimes=True
+ > EOF
+
+ $ hg log | cat
+ uiblocked stdio [0-9]+.[0-9]* ms command [0-9]+.[0-9]* ms (re)
--- a/tests/test-mq.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-mq.t Tue Feb 28 11:13:25 2017 -0800
@@ -25,7 +25,7 @@
Known patches are represented as patch files in the .hg/patches directory.
Applied patches are both patch files and changesets.
- Common tasks (use 'hg help command' for more details):
+ Common tasks (use 'hg help COMMAND' for more details):
create new patch qnew
import existing patch qimport
--- a/tests/test-pager.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-pager.t Tue Feb 28 11:13:25 2017 -0800
@@ -26,7 +26,7 @@
> hg ci -m "modify a $x"
> done
-By default diff and log are paged, but summary is not:
+By default diff and log are paged, but id is not:
$ hg diff -c 2 --pager=yes
paged! 'diff -r f4be7687d414 -r bce265549556 a\n'
@@ -50,25 +50,16 @@
paged! 'summary: modify a 9\n'
paged! '\n'
- $ hg summary
- parent: 10:46106edeeb38 tip
- modify a 10
- branch: default
- commit: (clean)
- update: (current)
- phases: 11 draft
+ $ hg id
+ 46106edeeb38 tip
-We can enable the pager on summary:
+We can enable the pager on id:
- $ hg --config pager.attend-summary=yes summary
- paged! 'parent: 10:46106edeeb38 tip\n'
- paged! ' modify a 10\n'
- paged! 'branch: default\n'
- paged! 'commit: (clean)\n'
- paged! 'update: (current)\n'
- paged! 'phases: 11 draft\n'
+ $ hg --config pager.attend-id=yes id
+ paged! '46106edeeb38 tip\n'
-If we completely change the attend list that's respected:
+Setting attend-$COMMAND to a false value works, even with pager in
+core:
$ hg --config pager.attend-diff=no diff -c 2
diff -r f4be7687d414 -r bce265549556 a
@@ -79,15 +70,6 @@
a 1
+a 2
- $ hg --config pager.attend=summary diff -c 2
- diff -r f4be7687d414 -r bce265549556 a
- --- a/a Thu Jan 01 00:00:00 1970 +0000
- +++ b/a Thu Jan 01 00:00:00 1970 +0000
- @@ -1,2 +1,3 @@
- a
- a 1
- +a 2
-
If 'log' is in attend, then 'history' should also be paged:
$ hg history --limit 2 --config pager.attend=log
paged! 'changeset: 10:46106edeeb38\n'
@@ -102,61 +84,17 @@
paged! 'summary: modify a 9\n'
paged! '\n'
-Possible bug: history is explicitly ignored in pager config, but
-because log is in the attend list it still gets pager treatment.
-
- $ hg history --limit 2 --config pager.attend=log \
- > --config pager.ignore=history
- paged! 'changeset: 10:46106edeeb38\n'
- paged! 'tag: tip\n'
- paged! 'user: test\n'
- paged! 'date: Thu Jan 01 00:00:00 1970 +0000\n'
- paged! 'summary: modify a 10\n'
- paged! '\n'
- paged! 'changeset: 9:6dd8ea7dd621\n'
- paged! 'user: test\n'
- paged! 'date: Thu Jan 01 00:00:00 1970 +0000\n'
- paged! 'summary: modify a 9\n'
- paged! '\n'
-
-Possible bug: history is explicitly marked as attend-history=no, but
-it doesn't fail to get paged because log is still in the attend list.
-
- $ hg history --limit 2 --config pager.attend-history=no
- paged! 'changeset: 10:46106edeeb38\n'
- paged! 'tag: tip\n'
- paged! 'user: test\n'
- paged! 'date: Thu Jan 01 00:00:00 1970 +0000\n'
- paged! 'summary: modify a 10\n'
- paged! '\n'
- paged! 'changeset: 9:6dd8ea7dd621\n'
- paged! 'user: test\n'
- paged! 'date: Thu Jan 01 00:00:00 1970 +0000\n'
- paged! 'summary: modify a 9\n'
- paged! '\n'
-
-Possible bug: disabling pager for log but enabling it for history
-doesn't result in history being paged.
-
- $ hg history --limit 2 --config pager.attend-log=no \
- > --config pager.attend-history=yes
- changeset: 10:46106edeeb38
- tag: tip
- user: test
- date: Thu Jan 01 00:00:00 1970 +0000
- summary: modify a 10
-
- changeset: 9:6dd8ea7dd621
- user: test
- date: Thu Jan 01 00:00:00 1970 +0000
- summary: modify a 9
-
-
Pager should not start if stdout is not a tty.
$ hg log -l1 -q --config ui.formatted=False
10:46106edeeb38
+Pager should be disabled if pager.pager is empty (otherwise the output would
+be silently lost.)
+
+ $ hg log -l1 -q --config pager.pager=
+ 10:46106edeeb38
+
Pager with color enabled allows colors to come through by default,
even though stdout is no longer a tty.
$ cat >> $HGRCPATH <<EOF
@@ -207,6 +145,11 @@
$ A=2 hg --config pager.attend-printa=yes printa
paged! '2\n'
+Something that's explicitly attended is still not paginated if the
+pager is globally set to off using a flag:
+ $ A=2 hg --config pager.attend-printa=yes printa --pager=no
+ 2
+
Pager should not override the exit code of other commands
$ cat >> $TESTTMP/fortytwo.py <<'EOF'
@@ -227,3 +170,61 @@
$ hg fortytwo --pager=on
paged! '42\n'
[42]
+
+A command that asks for paging using ui.pager() directly works:
+ $ hg blame a
+ paged! ' 0: a\n'
+ paged! ' 1: a 1\n'
+ paged! ' 2: a 2\n'
+ paged! ' 3: a 3\n'
+ paged! ' 4: a 4\n'
+ paged! ' 5: a 5\n'
+ paged! ' 6: a 6\n'
+ paged! ' 7: a 7\n'
+ paged! ' 8: a 8\n'
+ paged! ' 9: a 9\n'
+ paged! '10: a 10\n'
+but not with HGPLAIN
+ $ HGPLAIN=1 hg blame a
+ 0: a
+ 1: a 1
+ 2: a 2
+ 3: a 3
+ 4: a 4
+ 5: a 5
+ 6: a 6
+ 7: a 7
+ 8: a 8
+ 9: a 9
+ 10: a 10
+explicit flags work too:
+ $ hg blame --pager=no a
+ 0: a
+ 1: a 1
+ 2: a 2
+ 3: a 3
+ 4: a 4
+ 5: a 5
+ 6: a 6
+ 7: a 7
+ 8: a 8
+ 9: a 9
+ 10: a 10
+
+Put annotate in the ignore list for pager:
+ $ cat >> $HGRCPATH <<EOF
+ > [pager]
+ > ignore = annotate
+ > EOF
+ $ hg blame a
+ 0: a
+ 1: a 1
+ 2: a 2
+ 3: a 3
+ 4: a 4
+ 5: a 5
+ 6: a 6
+ 7: a 7
+ 8: a 8
+ 9: a 9
+ 10: a 10
--- a/tests/test-phases.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-phases.t Tue Feb 28 11:13:25 2017 -0800
@@ -590,3 +590,47 @@
crosschecking files in changesets and manifests
checking files
7 files, 8 changesets, 7 total revisions
+
+ $ cd ..
+
+check whether HG_PENDING makes pending changes only in related
+repositories visible to an external hook.
+
+(emulate a transaction running concurrently by copied
+.hg/phaseroots.pending in subsequent test)
+
+ $ cat > $TESTTMP/savepending.sh <<EOF
+ > cp .hg/store/phaseroots.pending .hg/store/phaseroots.pending.saved
+ > exit 1 # to avoid changing phase for subsequent tests
+ > EOF
+ $ cd push-dest
+ $ hg phase 6
+ 6: draft
+ $ hg --config hooks.pretxnclose="sh $TESTTMP/savepending.sh" phase -f -s 6
+ transaction abort!
+ rollback completed
+ abort: pretxnclose hook exited with status 1
+ [255]
+ $ cp .hg/store/phaseroots.pending.saved .hg/store/phaseroots.pending
+
+(check (in)visibility of phaseroot while transaction running in repo)
+
+ $ cat > $TESTTMP/checkpending.sh <<EOF
+ > echo '@initialrepo'
+ > hg -R $TESTTMP/initialrepo phase 7
+ > echo '@push-dest'
+ > hg -R $TESTTMP/push-dest phase 6
+ > exit 1 # to avoid changing phase for subsequent tests
+ > EOF
+ $ cd ../initialrepo
+ $ hg phase 7
+ 7: public
+ $ hg --config hooks.pretxnclose="sh $TESTTMP/checkpending.sh" phase -f -s 7
+ @initialrepo
+ 7: secret
+ @push-dest
+ 6: draft
+ transaction abort!
+ rollback completed
+ abort: pretxnclose hook exited with status 1
+ [255]
--- a/tests/test-push-http-bundle1.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-push-http-bundle1.t Tue Feb 28 11:13:25 2017 -0800
@@ -79,7 +79,7 @@
remote: adding manifests
remote: adding file changes
remote: added 1 changesets with 1 changes to 1 files
- remote: changegroup hook: HG_NODE=ba677d0156c1196c1a699fa53f390dcfc3ce3872 HG_NODE_LAST=ba677d0156c1196c1a699fa53f390dcfc3ce3872 HG_SOURCE=serve HG_TXNID=TXN:* HG_URL=remote:http:127.0.0.1: (glob)
+ remote: changegroup hook: HG_NODE=ba677d0156c1196c1a699fa53f390dcfc3ce3872 HG_NODE_LAST=ba677d0156c1196c1a699fa53f390dcfc3ce3872 HG_SOURCE=serve HG_TXNID=TXN:* HG_URL=remote:http:*: (glob)
% serve errors
$ hg rollback
repository tip rolled back to revision 0 (undo serve)
@@ -95,7 +95,7 @@
remote: adding manifests
remote: adding file changes
remote: added 1 changesets with 1 changes to 1 files
- remote: changegroup hook: HG_NODE=ba677d0156c1196c1a699fa53f390dcfc3ce3872 HG_NODE_LAST=ba677d0156c1196c1a699fa53f390dcfc3ce3872 HG_SOURCE=serve HG_TXNID=TXN:* HG_URL=remote:http:127.0.0.1: (glob)
+ remote: changegroup hook: HG_NODE=ba677d0156c1196c1a699fa53f390dcfc3ce3872 HG_NODE_LAST=ba677d0156c1196c1a699fa53f390dcfc3ce3872 HG_SOURCE=serve HG_TXNID=TXN:* HG_URL=remote:http:*: (glob)
% serve errors
$ hg rollback
repository tip rolled back to revision 0 (undo serve)
@@ -111,7 +111,7 @@
remote: adding manifests
remote: adding file changes
remote: added 1 changesets with 1 changes to 1 files
- remote: changegroup hook: HG_NODE=ba677d0156c1196c1a699fa53f390dcfc3ce3872 HG_NODE_LAST=ba677d0156c1196c1a699fa53f390dcfc3ce3872 HG_SOURCE=serve HG_TXNID=TXN:* HG_URL=remote:http:127.0.0.1: (glob)
+ remote: changegroup hook: HG_NODE=ba677d0156c1196c1a699fa53f390dcfc3ce3872 HG_NODE_LAST=ba677d0156c1196c1a699fa53f390dcfc3ce3872 HG_SOURCE=serve HG_TXNID=TXN:* HG_URL=remote:http:*: (glob)
% serve errors
$ hg rollback
repository tip rolled back to revision 0 (undo serve)
--- a/tests/test-push-http.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-push-http.t Tue Feb 28 11:13:25 2017 -0800
@@ -70,7 +70,7 @@
remote: adding file changes
remote: added 1 changesets with 1 changes to 1 files
remote: pushkey hook: HG_KEY=ba677d0156c1196c1a699fa53f390dcfc3ce3872 HG_NAMESPACE=phases HG_NEW=0 HG_OLD=1 HG_RET=1
- remote: changegroup hook: HG_BUNDLE2=1 HG_NODE=ba677d0156c1196c1a699fa53f390dcfc3ce3872 HG_NODE_LAST=ba677d0156c1196c1a699fa53f390dcfc3ce3872 HG_SOURCE=serve HG_TXNID=TXN:* HG_URL=remote:http:127.0.0.1: (glob)
+ remote: changegroup hook: HG_BUNDLE2=1 HG_NODE=ba677d0156c1196c1a699fa53f390dcfc3ce3872 HG_NODE_LAST=ba677d0156c1196c1a699fa53f390dcfc3ce3872 HG_SOURCE=serve HG_TXNID=TXN:* HG_URL=remote:http:*: (glob)
% serve errors
$ hg rollback
repository tip rolled back to revision 0 (undo serve)
@@ -87,7 +87,7 @@
remote: adding file changes
remote: added 1 changesets with 1 changes to 1 files
remote: pushkey hook: HG_KEY=ba677d0156c1196c1a699fa53f390dcfc3ce3872 HG_NAMESPACE=phases HG_NEW=0 HG_OLD=1 HG_RET=1
- remote: changegroup hook: HG_BUNDLE2=1 HG_NODE=ba677d0156c1196c1a699fa53f390dcfc3ce3872 HG_NODE_LAST=ba677d0156c1196c1a699fa53f390dcfc3ce3872 HG_SOURCE=serve HG_TXNID=TXN:* HG_URL=remote:http:127.0.0.1: (glob)
+ remote: changegroup hook: HG_BUNDLE2=1 HG_NODE=ba677d0156c1196c1a699fa53f390dcfc3ce3872 HG_NODE_LAST=ba677d0156c1196c1a699fa53f390dcfc3ce3872 HG_SOURCE=serve HG_TXNID=TXN:* HG_URL=remote:http:*: (glob)
% serve errors
$ hg rollback
repository tip rolled back to revision 0 (undo serve)
@@ -104,7 +104,7 @@
remote: adding file changes
remote: added 1 changesets with 1 changes to 1 files
remote: pushkey hook: HG_KEY=ba677d0156c1196c1a699fa53f390dcfc3ce3872 HG_NAMESPACE=phases HG_NEW=0 HG_OLD=1 HG_RET=1
- remote: changegroup hook: HG_BUNDLE2=1 HG_NODE=ba677d0156c1196c1a699fa53f390dcfc3ce3872 HG_NODE_LAST=ba677d0156c1196c1a699fa53f390dcfc3ce3872 HG_SOURCE=serve HG_TXNID=TXN:* HG_URL=remote:http:127.0.0.1: (glob)
+ remote: changegroup hook: HG_BUNDLE2=1 HG_NODE=ba677d0156c1196c1a699fa53f390dcfc3ce3872 HG_NODE_LAST=ba677d0156c1196c1a699fa53f390dcfc3ce3872 HG_SOURCE=serve HG_TXNID=TXN:* HG_URL=remote:http:*: (glob)
% serve errors
$ hg rollback
repository tip rolled back to revision 0 (undo serve)
@@ -125,7 +125,7 @@
remote: adding manifests
remote: adding file changes
remote: added 1 changesets with 1 changes to 1 files
- remote: prepushkey hook: HG_BUNDLE2=1 HG_KEY=ba677d0156c1196c1a699fa53f390dcfc3ce3872 HG_NAMESPACE=phases HG_NEW=0 HG_NODE=ba677d0156c1196c1a699fa53f390dcfc3ce3872 HG_NODE_LAST=ba677d0156c1196c1a699fa53f390dcfc3ce3872 HG_OLD=1 HG_PENDING=$TESTTMP/test HG_PHASES_MOVED=1 HG_SOURCE=serve HG_TXNID=TXN:* HG_URL=remote:http:127.0.0.1: (glob)
+ remote: prepushkey hook: HG_BUNDLE2=1 HG_KEY=ba677d0156c1196c1a699fa53f390dcfc3ce3872 HG_NAMESPACE=phases HG_NEW=0 HG_NODE=ba677d0156c1196c1a699fa53f390dcfc3ce3872 HG_NODE_LAST=ba677d0156c1196c1a699fa53f390dcfc3ce3872 HG_OLD=1 HG_PENDING=$TESTTMP/test HG_PHASES_MOVED=1 HG_SOURCE=serve HG_TXNID=TXN:* HG_URL=remote:http:*: (glob)
remote: pushkey-abort: prepushkey hook exited with status 1
remote: transaction abort!
remote: rollback completed
@@ -145,7 +145,7 @@
remote: adding manifests
remote: adding file changes
remote: added 1 changesets with 1 changes to 1 files
- remote: prepushkey hook: HG_BUNDLE2=1 HG_KEY=ba677d0156c1196c1a699fa53f390dcfc3ce3872 HG_NAMESPACE=phases HG_NEW=0 HG_NODE=ba677d0156c1196c1a699fa53f390dcfc3ce3872 HG_NODE_LAST=ba677d0156c1196c1a699fa53f390dcfc3ce3872 HG_OLD=1 HG_PENDING=$TESTTMP/test HG_PHASES_MOVED=1 HG_SOURCE=serve HG_TXNID=TXN:* HG_URL=remote:http:127.0.0.1: (glob)
+ remote: prepushkey hook: HG_BUNDLE2=1 HG_KEY=ba677d0156c1196c1a699fa53f390dcfc3ce3872 HG_NAMESPACE=phases HG_NEW=0 HG_NODE=ba677d0156c1196c1a699fa53f390dcfc3ce3872 HG_NODE_LAST=ba677d0156c1196c1a699fa53f390dcfc3ce3872 HG_OLD=1 HG_PENDING=$TESTTMP/test HG_PHASES_MOVED=1 HG_SOURCE=serve HG_TXNID=TXN:* HG_URL=remote:http:*: (glob)
% serve errors
$ hg rollback
repository tip rolled back to revision 0 (undo serve)
--- a/tests/test-revset.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-revset.t Tue Feb 28 11:13:25 2017 -0800
@@ -40,6 +40,8 @@
> cmdutil,
> node as nodemod,
> revset,
+ > revsetlang,
+ > smartset,
> )
> cmdtable = {}
> command = cmdutil.command(cmdtable)
@@ -49,17 +51,18 @@
> def debugrevlistspec(ui, repo, fmt, *args, **opts):
> if opts['bin']:
> args = map(nodemod.bin, args)
- > expr = revset.formatspec(fmt, list(args))
+ > expr = revsetlang.formatspec(fmt, list(args))
> if ui.verbose:
- > tree = revset.parse(expr, lookup=repo.__contains__)
- > ui.note(revset.prettyformat(tree), "\n")
+ > tree = revsetlang.parse(expr, lookup=repo.__contains__)
+ > ui.note(revsetlang.prettyformat(tree), "\n")
> if opts["optimize"]:
- > opttree = revset.optimize(revset.analyze(tree))
- > ui.note("* optimized:\n", revset.prettyformat(opttree), "\n")
+ > opttree = revsetlang.optimize(revsetlang.analyze(tree))
+ > ui.note("* optimized:\n", revsetlang.prettyformat(opttree),
+ > "\n")
> func = revset.match(ui, expr, repo)
> revs = func(repo)
> if ui.verbose:
- > ui.note("* set:\n", revset.prettyformatset(revs), "\n")
+ > ui.note("* set:\n", smartset.prettyformat(revs), "\n")
> for c in revs:
> ui.write("%s\n" % c)
> EOF
--- a/tests/test-serve.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-serve.t Tue Feb 28 11:13:25 2017 -0800
@@ -34,13 +34,13 @@
With -v
$ hgserve
- listening at http://localhost/ (bound to 127.0.0.1:HGPORT1) (glob)
+ listening at http://localhost/ (bound to *$LOCALIP*:HGPORT1) (glob)
% errors
With -v and -p HGPORT2
$ hgserve -p "$HGPORT2"
- listening at http://localhost/ (bound to 127.0.0.1:HGPORT2) (glob)
+ listening at http://localhost/ (bound to *$LOCALIP*:HGPORT2) (glob)
% errors
With -v and -p daytime (should fail because low port)
@@ -57,25 +57,25 @@
With --prefix foo
$ hgserve --prefix foo
- listening at http://localhost/foo/ (bound to 127.0.0.1:HGPORT1) (glob)
+ listening at http://localhost/foo/ (bound to *$LOCALIP*:HGPORT1) (glob)
% errors
With --prefix /foo
$ hgserve --prefix /foo
- listening at http://localhost/foo/ (bound to 127.0.0.1:HGPORT1) (glob)
+ listening at http://localhost/foo/ (bound to *$LOCALIP*:HGPORT1) (glob)
% errors
With --prefix foo/
$ hgserve --prefix foo/
- listening at http://localhost/foo/ (bound to 127.0.0.1:HGPORT1) (glob)
+ listening at http://localhost/foo/ (bound to *$LOCALIP*:HGPORT1) (glob)
% errors
With --prefix /foo/
$ hgserve --prefix /foo/
- listening at http://localhost/foo/ (bound to 127.0.0.1:HGPORT1) (glob)
+ listening at http://localhost/foo/ (bound to *$LOCALIP*:HGPORT1) (glob)
% errors
$ cd ..
--- a/tests/test-share.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-share.t Tue Feb 28 11:13:25 2017 -0800
@@ -154,6 +154,67 @@
* bm1 2:c2e0ac586386
bm3 2:c2e0ac586386
+check whether HG_PENDING makes pending changes only in relatd
+repositories visible to an external hook.
+
+In "hg share" case, another transaction can't run in other
+repositories sharing same source repository, because starting
+transaction requires locking store of source repository.
+
+Therefore, this test scenario ignores checking visibility of
+.hg/bookmakrs.pending in repo2, which shares repo1 without bookmarks.
+
+ $ cat > $TESTTMP/checkbookmarks.sh <<EOF
+ > echo "@repo1"
+ > hg -R $TESTTMP/repo1 bookmarks
+ > echo "@repo2"
+ > hg -R $TESTTMP/repo2 bookmarks
+ > echo "@repo3"
+ > hg -R $TESTTMP/repo3 bookmarks
+ > exit 1 # to avoid adding new bookmark for subsequent tests
+ > EOF
+
+ $ cd ../repo1
+ $ hg --config hooks.pretxnclose="sh $TESTTMP/checkbookmarks.sh" -q book bmX
+ @repo1
+ bm1 2:c2e0ac586386
+ bm3 2:c2e0ac586386
+ * bmX 2:c2e0ac586386
+ @repo2
+ * bm2 3:0e6e70d1d5f1
+ @repo3
+ bm1 2:c2e0ac586386
+ * bm3 2:c2e0ac586386
+ bmX 2:c2e0ac586386
+ transaction abort!
+ rollback completed
+ abort: pretxnclose hook exited with status 1
+ [255]
+ $ hg book bm1
+
+FYI, in contrast to above test, bmX is invisible in repo1 (= shared
+src), because (1) HG_PENDING refers only repo3 and (2)
+"bookmarks.pending" is written only into repo3.
+
+ $ cd ../repo3
+ $ hg --config hooks.pretxnclose="sh $TESTTMP/checkbookmarks.sh" -q book bmX
+ @repo1
+ * bm1 2:c2e0ac586386
+ bm3 2:c2e0ac586386
+ @repo2
+ * bm2 3:0e6e70d1d5f1
+ @repo3
+ bm1 2:c2e0ac586386
+ bm3 2:c2e0ac586386
+ * bmX 2:c2e0ac586386
+ transaction abort!
+ rollback completed
+ abort: pretxnclose hook exited with status 1
+ [255]
+ $ hg book bm3
+
+ $ cd ../repo1
+
test that commits work
$ echo 'shared bookmarks' > a
--- a/tests/test-shelve.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-shelve.t Tue Feb 28 11:13:25 2017 -0800
@@ -493,7 +493,7 @@
$ ln -s foo a/a
$ hg shelve -q -n symlink a/a
$ hg status a/a
- $ hg unshelve -q symlink
+ $ hg unshelve -q -n symlink
$ hg status a/a
M a/a
$ hg revert a/a
--- a/tests/test-ssh-bundle1.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-ssh-bundle1.t Tue Feb 28 11:13:25 2017 -0800
@@ -494,7 +494,7 @@
Got arguments 1:user@dummy 2:hg -R local serve --stdio
Got arguments 1:user@dummy 2:hg -R $TESTTMP/local serve --stdio
Got arguments 1:user@dummy 2:hg -R remote serve --stdio
- changegroup-in-remote hook: HG_NODE=a28a9d1a809cab7d4e2fde4bee738a9ede948b60 HG_NODE_LAST=a28a9d1a809cab7d4e2fde4bee738a9ede948b60 HG_SOURCE=serve HG_TXNID=TXN:* HG_URL=remote:ssh:127.0.0.1 (glob)
+ changegroup-in-remote hook: HG_NODE=a28a9d1a809cab7d4e2fde4bee738a9ede948b60 HG_NODE_LAST=a28a9d1a809cab7d4e2fde4bee738a9ede948b60 HG_SOURCE=serve HG_TXNID=TXN:* HG_URL=remote:ssh:$LOCALIP (glob)
Got arguments 1:user@dummy 2:hg -R remote serve --stdio
Got arguments 1:user@dummy 2:hg -R remote serve --stdio
Got arguments 1:user@dummy 2:hg -R remote serve --stdio
@@ -504,7 +504,7 @@
Got arguments 1:user@dummy 2:hg -R remote serve --stdio
Got arguments 1:user@dummy 2:hg -R remote serve --stdio
Got arguments 1:user@dummy 2:hg -R remote serve --stdio
- changegroup-in-remote hook: HG_NODE=1383141674ec756a6056f6a9097618482fe0f4a6 HG_NODE_LAST=1383141674ec756a6056f6a9097618482fe0f4a6 HG_SOURCE=serve HG_TXNID=TXN:* HG_URL=remote:ssh:127.0.0.1 (glob)
+ changegroup-in-remote hook: HG_NODE=1383141674ec756a6056f6a9097618482fe0f4a6 HG_NODE_LAST=1383141674ec756a6056f6a9097618482fe0f4a6 HG_SOURCE=serve HG_TXNID=TXN:* HG_URL=remote:ssh:$LOCALIP (glob)
Got arguments 1:user@dummy 2:hg -R remote serve --stdio
Got arguments 1:user@dummy 2:hg init 'a repo'
Got arguments 1:user@dummy 2:hg -R 'a repo' serve --stdio
@@ -512,7 +512,7 @@
Got arguments 1:user@dummy 2:hg -R 'a repo' serve --stdio
Got arguments 1:user@dummy 2:hg -R 'a repo' serve --stdio
Got arguments 1:user@dummy 2:hg -R remote serve --stdio
- changegroup-in-remote hook: HG_NODE=65c38f4125f9602c8db4af56530cc221d93b8ef8 HG_NODE_LAST=65c38f4125f9602c8db4af56530cc221d93b8ef8 HG_SOURCE=serve HG_TXNID=TXN:* HG_URL=remote:ssh:127.0.0.1 (glob)
+ changegroup-in-remote hook: HG_NODE=65c38f4125f9602c8db4af56530cc221d93b8ef8 HG_NODE_LAST=65c38f4125f9602c8db4af56530cc221d93b8ef8 HG_SOURCE=serve HG_TXNID=TXN:* HG_URL=remote:ssh:$LOCALIP (glob)
Got arguments 1:user@dummy 2:hg -R remote serve --stdio
remote hook failure is attributed to remote
--- a/tests/test-ssh.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-ssh.t Tue Feb 28 11:13:25 2017 -0800
@@ -498,7 +498,7 @@
Got arguments 1:user@dummy 2:hg -R local serve --stdio
Got arguments 1:user@dummy 2:hg -R $TESTTMP/local serve --stdio
Got arguments 1:user@dummy 2:hg -R remote serve --stdio
- changegroup-in-remote hook: HG_BUNDLE2=1 HG_NODE=a28a9d1a809cab7d4e2fde4bee738a9ede948b60 HG_NODE_LAST=a28a9d1a809cab7d4e2fde4bee738a9ede948b60 HG_SOURCE=serve HG_TXNID=TXN:* HG_URL=remote:ssh:127.0.0.1 (glob)
+ changegroup-in-remote hook: HG_BUNDLE2=1 HG_NODE=a28a9d1a809cab7d4e2fde4bee738a9ede948b60 HG_NODE_LAST=a28a9d1a809cab7d4e2fde4bee738a9ede948b60 HG_SOURCE=serve HG_TXNID=TXN:* HG_URL=remote:ssh:$LOCALIP (glob)
Got arguments 1:user@dummy 2:hg -R remote serve --stdio
Got arguments 1:user@dummy 2:hg -R remote serve --stdio
Got arguments 1:user@dummy 2:hg -R remote serve --stdio
@@ -508,7 +508,7 @@
Got arguments 1:user@dummy 2:hg -R remote serve --stdio
Got arguments 1:user@dummy 2:hg -R remote serve --stdio
Got arguments 1:user@dummy 2:hg -R remote serve --stdio
- changegroup-in-remote hook: HG_BUNDLE2=1 HG_NODE=1383141674ec756a6056f6a9097618482fe0f4a6 HG_NODE_LAST=1383141674ec756a6056f6a9097618482fe0f4a6 HG_SOURCE=serve HG_TXNID=TXN:* HG_URL=remote:ssh:127.0.0.1 (glob)
+ changegroup-in-remote hook: HG_BUNDLE2=1 HG_NODE=1383141674ec756a6056f6a9097618482fe0f4a6 HG_NODE_LAST=1383141674ec756a6056f6a9097618482fe0f4a6 HG_SOURCE=serve HG_TXNID=TXN:* HG_URL=remote:ssh:$LOCALIP (glob)
Got arguments 1:user@dummy 2:hg -R remote serve --stdio
Got arguments 1:user@dummy 2:hg init 'a repo'
Got arguments 1:user@dummy 2:hg -R 'a repo' serve --stdio
@@ -516,7 +516,7 @@
Got arguments 1:user@dummy 2:hg -R 'a repo' serve --stdio
Got arguments 1:user@dummy 2:hg -R 'a repo' serve --stdio
Got arguments 1:user@dummy 2:hg -R remote serve --stdio
- changegroup-in-remote hook: HG_BUNDLE2=1 HG_NODE=65c38f4125f9602c8db4af56530cc221d93b8ef8 HG_NODE_LAST=65c38f4125f9602c8db4af56530cc221d93b8ef8 HG_SOURCE=serve HG_TXNID=TXN:* HG_URL=remote:ssh:127.0.0.1 (glob)
+ changegroup-in-remote hook: HG_BUNDLE2=1 HG_NODE=65c38f4125f9602c8db4af56530cc221d93b8ef8 HG_NODE_LAST=65c38f4125f9602c8db4af56530cc221d93b8ef8 HG_SOURCE=serve HG_TXNID=TXN:* HG_URL=remote:ssh:$LOCALIP (glob)
Got arguments 1:user@dummy 2:hg -R remote serve --stdio
remote hook failure is attributed to remote
--- a/tests/test-status-color.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-status-color.t Tue Feb 28 11:13:25 2017 -0800
@@ -1,6 +1,6 @@
$ cat <<EOF >> $HGRCPATH
- > [extensions]
- > color =
+ > [ui]
+ > color = always
> [color]
> mode = ansi
> EOF
@@ -14,7 +14,7 @@
hg status in repo root:
- $ hg status --color=always
+ $ hg status
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4ma/1/in_a_1\x1b[0m (esc)
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4ma/in_a\x1b[0m (esc)
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4mb/1/in_b_1\x1b[0m (esc)
@@ -41,7 +41,7 @@
hg status . in repo root:
- $ hg status --color=always .
+ $ hg status .
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4ma/1/in_a_1\x1b[0m (esc)
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4ma/in_a\x1b[0m (esc)
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4mb/1/in_b_1\x1b[0m (esc)
@@ -49,17 +49,17 @@
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4mb/in_b\x1b[0m (esc)
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4min_root\x1b[0m (esc)
- $ hg status --color=always --cwd a
+ $ hg status --cwd a
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4ma/1/in_a_1\x1b[0m (esc)
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4ma/in_a\x1b[0m (esc)
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4mb/1/in_b_1\x1b[0m (esc)
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4mb/2/in_b_2\x1b[0m (esc)
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4mb/in_b\x1b[0m (esc)
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4min_root\x1b[0m (esc)
- $ hg status --color=always --cwd a .
+ $ hg status --cwd a .
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4m1/in_a_1\x1b[0m (esc)
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4min_a\x1b[0m (esc)
- $ hg status --color=always --cwd a ..
+ $ hg status --cwd a ..
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4m1/in_a_1\x1b[0m (esc)
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4min_a\x1b[0m (esc)
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4m../b/1/in_b_1\x1b[0m (esc)
@@ -67,18 +67,18 @@
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4m../b/in_b\x1b[0m (esc)
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4m../in_root\x1b[0m (esc)
- $ hg status --color=always --cwd b
+ $ hg status --cwd b
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4ma/1/in_a_1\x1b[0m (esc)
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4ma/in_a\x1b[0m (esc)
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4mb/1/in_b_1\x1b[0m (esc)
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4mb/2/in_b_2\x1b[0m (esc)
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4mb/in_b\x1b[0m (esc)
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4min_root\x1b[0m (esc)
- $ hg status --color=always --cwd b .
+ $ hg status --cwd b .
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4m1/in_b_1\x1b[0m (esc)
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4m2/in_b_2\x1b[0m (esc)
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4min_b\x1b[0m (esc)
- $ hg status --color=always --cwd b ..
+ $ hg status --cwd b ..
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4m../a/1/in_a_1\x1b[0m (esc)
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4m../a/in_a\x1b[0m (esc)
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4m1/in_b_1\x1b[0m (esc)
@@ -86,43 +86,43 @@
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4min_b\x1b[0m (esc)
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4m../in_root\x1b[0m (esc)
- $ hg status --color=always --cwd a/1
+ $ hg status --cwd a/1
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4ma/1/in_a_1\x1b[0m (esc)
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4ma/in_a\x1b[0m (esc)
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4mb/1/in_b_1\x1b[0m (esc)
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4mb/2/in_b_2\x1b[0m (esc)
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4mb/in_b\x1b[0m (esc)
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4min_root\x1b[0m (esc)
- $ hg status --color=always --cwd a/1 .
+ $ hg status --cwd a/1 .
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4min_a_1\x1b[0m (esc)
- $ hg status --color=always --cwd a/1 ..
+ $ hg status --cwd a/1 ..
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4min_a_1\x1b[0m (esc)
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4m../in_a\x1b[0m (esc)
- $ hg status --color=always --cwd b/1
+ $ hg status --cwd b/1
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4ma/1/in_a_1\x1b[0m (esc)
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4ma/in_a\x1b[0m (esc)
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4mb/1/in_b_1\x1b[0m (esc)
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4mb/2/in_b_2\x1b[0m (esc)
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4mb/in_b\x1b[0m (esc)
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4min_root\x1b[0m (esc)
- $ hg status --color=always --cwd b/1 .
+ $ hg status --cwd b/1 .
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4min_b_1\x1b[0m (esc)
- $ hg status --color=always --cwd b/1 ..
+ $ hg status --cwd b/1 ..
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4min_b_1\x1b[0m (esc)
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4m../2/in_b_2\x1b[0m (esc)
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4m../in_b\x1b[0m (esc)
- $ hg status --color=always --cwd b/2
+ $ hg status --cwd b/2
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4ma/1/in_a_1\x1b[0m (esc)
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4ma/in_a\x1b[0m (esc)
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4mb/1/in_b_1\x1b[0m (esc)
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4mb/2/in_b_2\x1b[0m (esc)
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4mb/in_b\x1b[0m (esc)
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4min_root\x1b[0m (esc)
- $ hg status --color=always --cwd b/2 .
+ $ hg status --cwd b/2 .
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4min_b_2\x1b[0m (esc)
- $ hg status --color=always --cwd b/2 ..
+ $ hg status --cwd b/2 ..
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4m../1/in_b_1\x1b[0m (esc)
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4min_b_2\x1b[0m (esc)
\x1b[0;35;1;4m? \x1b[0m\x1b[0;35;1;4m../in_b\x1b[0m (esc)
@@ -137,7 +137,7 @@
? in_root
Make sure ui.formatted=False works
- $ hg status --config ui.formatted=False
+ $ hg status --color=auto --config ui.formatted=False
? a/1/in_a_1
? a/in_a
? b/1/in_b_1
@@ -179,7 +179,7 @@
hg status:
- $ hg status --color=always
+ $ hg status
\x1b[0;32;1mA \x1b[0m\x1b[0;32;1madded\x1b[0m (esc)
\x1b[0;31;1mR \x1b[0m\x1b[0;31;1mremoved\x1b[0m (esc)
\x1b[0;36;1;4m! \x1b[0m\x1b[0;36;1;4mdeleted\x1b[0m (esc)
@@ -187,7 +187,7 @@
hg status modified added removed deleted unknown never-existed ignored:
- $ hg status --color=always modified added removed deleted unknown never-existed ignored
+ $ hg status modified added removed deleted unknown never-existed ignored
never-existed: * (glob)
\x1b[0;32;1mA \x1b[0m\x1b[0;32;1madded\x1b[0m (esc)
\x1b[0;31;1mR \x1b[0m\x1b[0;31;1mremoved\x1b[0m (esc)
@@ -198,7 +198,7 @@
hg status -C:
- $ hg status --color=always -C
+ $ hg status -C
\x1b[0;32;1mA \x1b[0m\x1b[0;32;1madded\x1b[0m (esc)
\x1b[0;32;1mA \x1b[0m\x1b[0;32;1mcopied\x1b[0m (esc)
\x1b[0;0m modified\x1b[0m (esc)
@@ -208,7 +208,7 @@
hg status -A:
- $ hg status --color=always -A
+ $ hg status -A
\x1b[0;32;1mA \x1b[0m\x1b[0;32;1madded\x1b[0m (esc)
\x1b[0;32;1mA \x1b[0m\x1b[0;32;1mcopied\x1b[0m (esc)
\x1b[0;0m modified\x1b[0m (esc)
@@ -226,7 +226,7 @@
$ mkdir "$TESTTMP/terminfo"
$ TERMINFO="$TESTTMP/terminfo" tic "$TESTDIR/hgterm.ti"
- $ TERM=hgterm TERMINFO="$TESTTMP/terminfo" hg status --config color.mode=terminfo --color=always -A
+ $ TERM=hgterm TERMINFO="$TESTTMP/terminfo" hg status --config color.mode=terminfo -A
\x1b[30m\x1b[32m\x1b[1mA \x1b[30m\x1b[30m\x1b[32m\x1b[1madded\x1b[30m (esc)
\x1b[30m\x1b[32m\x1b[1mA \x1b[30m\x1b[30m\x1b[32m\x1b[1mcopied\x1b[30m (esc)
\x1b[30m\x1b[30m modified\x1b[30m (esc)
@@ -245,7 +245,7 @@
> # We can override what's in the terminfo database, too
> terminfo.bold = \E[2m
> EOF
- $ TERM=hgterm TERMINFO="$TESTTMP/terminfo" hg status --config color.mode=terminfo --config color.status.clean=dim --color=always -A
+ $ TERM=hgterm TERMINFO="$TESTTMP/terminfo" hg status --config color.mode=terminfo --config color.status.clean=dim -A
\x1b[30m\x1b[32m\x1b[2mA \x1b[30m\x1b[30m\x1b[32m\x1b[2madded\x1b[30m (esc)
\x1b[30m\x1b[32m\x1b[2mA \x1b[30m\x1b[30m\x1b[32m\x1b[2mcopied\x1b[30m (esc)
\x1b[30m\x1b[30m modified\x1b[30m (esc)
@@ -265,11 +265,11 @@
hg status ignoreddir/file:
- $ hg status --color=always ignoreddir/file
+ $ hg status ignoreddir/file
hg status -i ignoreddir/file:
- $ hg status --color=always -i ignoreddir/file
+ $ hg status -i ignoreddir/file
\x1b[0;30;1mI \x1b[0m\x1b[0;30;1mignoreddir/file\x1b[0m (esc)
$ cd ..
@@ -293,7 +293,9 @@
test unknown color
- $ hg --config color.status.modified=periwinkle status --color=always
+ $ hg --config color.status.modified=periwinkle status
+ ignoring unknown color/effect 'periwinkle' (configured in color.status.modified)
+ ignoring unknown color/effect 'periwinkle' (configured in color.status.modified)
ignoring unknown color/effect 'periwinkle' (configured in color.status.modified)
M modified
\x1b[0;32;1mA \x1b[0m\x1b[0;32;1madded\x1b[0m (esc)
@@ -307,8 +309,8 @@
If result is not as expected, raise error
$ assert() {
- > hg status --color=always $1 > ../a
- > hg status --color=always $2 > ../b
+ > hg status $1 > ../a
+ > hg status $2 > ../b
> if diff ../a ../b > /dev/null; then
> out=0
> else
@@ -367,7 +369,7 @@
hg resolve with one unresolved, one resolved:
- $ hg resolve --color=always -l
+ $ hg resolve -l
\x1b[0;31;1mU \x1b[0m\x1b[0;31;1ma\x1b[0m (esc)
\x1b[0;32;1mR \x1b[0m\x1b[0;32;1mb\x1b[0m (esc)
--- a/tests/test-ui-color.py Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-ui-color.py Tue Feb 28 11:13:25 2017 -0800
@@ -1,16 +1,13 @@
from __future__ import absolute_import, print_function
import os
-from hgext import (
- color,
-)
from mercurial import (
dispatch,
ui as uimod,
)
# ensure errors aren't buffered
-testui = color.colorui()
+testui = uimod.ui()
testui.pushbuffer()
testui.write(('buffered\n'))
testui.warn(('warning\n'))
@@ -35,6 +32,7 @@
dispatch.dispatch(dispatch.request(['version', '-q'], ui_))
runcmd()
-print("colored? " + str(issubclass(ui_.__class__, color.colorui)))
+print("colored? %s" % (ui_._colormode is not None))
runcmd()
-print("colored? " + str(issubclass(ui_.__class__, color.colorui)))
+print("colored? %s" % (ui_._colormode is not None))
+
--- a/tests/test-update-branches.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-update-branches.t Tue Feb 28 11:13:25 2017 -0800
@@ -177,6 +177,28 @@
$ cd ..
+Test updating to null revision
+
+ $ hg init null-repo
+ $ cd null-repo
+ $ echo a > a
+ $ hg add a
+ $ hg ci -m a
+ $ hg up -qC 0
+ $ echo b > b
+ $ hg add b
+ $ hg up null
+ 0 files updated, 0 files merged, 1 files removed, 0 files unresolved
+ $ hg st
+ A b
+ $ hg up -q 0
+ $ hg st
+ A b
+ $ hg up -qC null
+ $ hg st
+ ? b
+ $ cd ..
+
Test updating with closed head
---------------------------------------------------------------------
--- a/tests/test-walk.t Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/test-walk.t Tue Feb 28 11:13:25 2017 -0800
@@ -112,6 +112,74 @@
f beans/navy ../beans/navy
f beans/pinto ../beans/pinto
f beans/turtle ../beans/turtle
+
+ $ hg debugwalk 'rootfilesin:'
+ f fennel ../fennel
+ f fenugreek ../fenugreek
+ f fiddlehead ../fiddlehead
+ $ hg debugwalk -I 'rootfilesin:'
+ f fennel ../fennel
+ f fenugreek ../fenugreek
+ f fiddlehead ../fiddlehead
+ $ hg debugwalk 'rootfilesin:.'
+ f fennel ../fennel
+ f fenugreek ../fenugreek
+ f fiddlehead ../fiddlehead
+ $ hg debugwalk -I 'rootfilesin:.'
+ f fennel ../fennel
+ f fenugreek ../fenugreek
+ f fiddlehead ../fiddlehead
+ $ hg debugwalk -X 'rootfilesin:'
+ f beans/black ../beans/black
+ f beans/borlotti ../beans/borlotti
+ f beans/kidney ../beans/kidney
+ f beans/navy ../beans/navy
+ f beans/pinto ../beans/pinto
+ f beans/turtle ../beans/turtle
+ f mammals/Procyonidae/cacomistle Procyonidae/cacomistle
+ f mammals/Procyonidae/coatimundi Procyonidae/coatimundi
+ f mammals/Procyonidae/raccoon Procyonidae/raccoon
+ f mammals/skunk skunk
+ $ hg debugwalk 'rootfilesin:fennel'
+ $ hg debugwalk -I 'rootfilesin:fennel'
+ $ hg debugwalk 'rootfilesin:skunk'
+ $ hg debugwalk -I 'rootfilesin:skunk'
+ $ hg debugwalk 'rootfilesin:beans'
+ f beans/black ../beans/black
+ f beans/borlotti ../beans/borlotti
+ f beans/kidney ../beans/kidney
+ f beans/navy ../beans/navy
+ f beans/pinto ../beans/pinto
+ f beans/turtle ../beans/turtle
+ $ hg debugwalk -I 'rootfilesin:beans'
+ f beans/black ../beans/black
+ f beans/borlotti ../beans/borlotti
+ f beans/kidney ../beans/kidney
+ f beans/navy ../beans/navy
+ f beans/pinto ../beans/pinto
+ f beans/turtle ../beans/turtle
+ $ hg debugwalk 'rootfilesin:mammals'
+ f mammals/skunk skunk
+ $ hg debugwalk -I 'rootfilesin:mammals'
+ f mammals/skunk skunk
+ $ hg debugwalk 'rootfilesin:mammals/'
+ f mammals/skunk skunk
+ $ hg debugwalk -I 'rootfilesin:mammals/'
+ f mammals/skunk skunk
+ $ hg debugwalk -X 'rootfilesin:mammals'
+ f beans/black ../beans/black
+ f beans/borlotti ../beans/borlotti
+ f beans/kidney ../beans/kidney
+ f beans/navy ../beans/navy
+ f beans/pinto ../beans/pinto
+ f beans/turtle ../beans/turtle
+ f fennel ../fennel
+ f fenugreek ../fenugreek
+ f fiddlehead ../fiddlehead
+ f mammals/Procyonidae/cacomistle Procyonidae/cacomistle
+ f mammals/Procyonidae/coatimundi Procyonidae/coatimundi
+ f mammals/Procyonidae/raccoon Procyonidae/raccoon
+
$ hg debugwalk .
f mammals/Procyonidae/cacomistle Procyonidae/cacomistle
f mammals/Procyonidae/coatimundi Procyonidae/coatimundi
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/test-xdg.t Tue Feb 28 11:13:25 2017 -0800
@@ -0,0 +1,11 @@
+#if no-windows no-osx
+
+ $ mkdir -p xdgconf/hg
+ $ echo '[ui]' > xdgconf/hg/hgrc
+ $ echo 'username = foobar' >> xdgconf/hg/hgrc
+ $ XDG_CONFIG_HOME="`pwd`/xdgconf" ; export XDG_CONFIG_HOME
+ $ unset HGRCPATH
+ $ hg config ui.username
+ foobar
+
+#endif
--- a/tests/tinyproxy.py Sat Feb 25 12:48:50 2017 +0900
+++ b/tests/tinyproxy.py Tue Feb 28 11:13:25 2017 -0800
@@ -26,6 +26,11 @@
urlparse = util.urlparse
socketserver = util.socketserver
+if os.environ.get('HGIPV6', '0') == '1':
+ family = socket.AF_INET6
+else:
+ family = socket.AF_INET
+
class ProxyHandler (httpserver.basehttprequesthandler):
__base = httpserver.basehttprequesthandler
__base_handle = __base.handle
@@ -65,7 +70,7 @@
return 1
def do_CONNECT(self):
- soc = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ soc = socket.socket(family, socket.SOCK_STREAM)
try:
if self._connect_to(self.path, soc):
self.log_request(200)
@@ -85,7 +90,7 @@
if scm != 'http' or fragment or not netloc:
self.send_error(400, "bad url %s" % self.path)
return
- soc = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ soc = socket.socket(family, socket.SOCK_STREAM)
try:
if self._connect_to(netloc, soc):
self.log_request()