mercurial-scm/hg: mercurial/util.py comparison

comparison mercurial/util.py @ 30761:7283719e2bfd

util: declare wire protocol support of compression engines This patch implements a new compression engine API allowing compression engines to declare support for the wire protocol. Support is declared by returning a compression format string identifier that will be added to payloads to signal the compression type of data that follows and default integer priorities of the engine. Accessor methods have been added to the compression engine manager class to facilitate use. Note that the "none" and "bz2" engines declare wire protocol support but aren't enabled by default due to their priorities being 0. It is essentially free from a coding perspective to support these compression formats, so we do it in case anyone may derive use from it.

author	Gregory Szorc <gregory.szorc@gmail.com>
date	Sat, 24 Dec 2016 13:51:12 -0700
parents	c1b7b2285522
children	c390b40fe1d7

comparison

equal deleted inserted replaced

-:753b9d43ca81
+:7283719e2bfd
 raise exc_val
 return received and suppressed
 # compression code
+SERVERROLE = 'server'
+CLIENTROLE = 'client'
+compewireprotosupport = collections.namedtuple(u'compenginewireprotosupport',
+(u'name', u'serverpriority',
+u'clientpriority'))
 class compressormanager(object):
 """Holds registrations of various compression engines.
 This class essentially abstracts the differences between compression
 engines to allow new compression formats to be added easily, possibly from
 self._engines = {}
 # Bundle spec human name to engine name.
 self._bundlenames = {}
 # Internal bundle identifier to engine name.
 self._bundletypes = {}
+# Wire proto identifier to engine name.
+self._wiretypes = {}
 def __getitem__(self, key):
 return self._engines[key]
 def __contains__(self, key):
 # No external facing name declared.
 if bundlename:
 self._bundlenames[bundlename] = name
 self._bundletypes[bundletype] = name
+wiresupport = engine.wireprotosupport()
+if wiresupport:
+wiretype = wiresupport.name
+if wiretype in self._wiretypes:
+raise error.Abort(_('wire protocol compression %s already '
+'registered by %s') %
+(wiretype, self._wiretypes[wiretype]))
+self._wiretypes[wiretype] = name
 self._engines[name] = engine
 @property
 def supportedbundlenames(self):
 if not engine.available():
 raise error.Abort(_('compression engine %s could not be loaded') %
 engine.name())
 return engine
+def supportedwireengines(self, role, onlyavailable=True):
+"""Obtain compression engines that support the wire protocol.
+Returns a list of engines in prioritized order, most desired first.
+If ``onlyavailable`` is set, filter out engines that can't be
+loaded.
+"""
+assert role in (SERVERROLE, CLIENTROLE)
+attr = 'serverpriority' if role == SERVERROLE else 'clientpriority'
+engines = [self._engines[e] for e in self._wiretypes.values()]
+if onlyavailable:
+engines = [e for e in engines if e.available()]
+def getkey(e):
+# Sort first by priority, highest first. In case of tie, sort
+# alphabetically. This is arbitrary, but ensures output is
+# stable.
+w = e.wireprotosupport()
+return -1 * getattr(w, attr), w.name
+return list(sorted(engines, key=getkey))
+def forwiretype(self, wiretype):
+engine = self._engines[self._wiretypes[wiretype]]
+if not engine.available():
+raise error.Abort(_('compression engine %s could not be loaded') %
+engine.name())
+return engine
 compengines = compressormanager()
 class compressionengine(object):
 """Base class for compression engines.
 If bundle compression is supported, the class must also implement
 ``compressstream`` and `decompressorreader``.
 """
 return None
+def wireprotosupport(self):
+"""Declare support for this compression format on the wire protocol.
+If this compression engine isn't supported for compressing wire
+protocol payloads, returns None.
+Otherwise, returns ``compenginewireprotosupport`` with the following
+fields:
+* String format identifier
+* Integer priority for the server
+* Integer priority for the client
+The integer priorities are used to order the advertisement of format
+support by server and client. The highest integer is advertised
+first. Integers with non-positive values aren't advertised.
+The priority values are somewhat arbitrary and only used for default
+ordering. The relative order can be changed via config options.
+If wire protocol compression is supported, the class must also implement
+``compressstream`` and ``decompressorreader``.
+"""
+return None
 def compressstream(self, it, opts=None):
 """Compress an iterator of chunks.
 The method receives an iterator (ideally a generator) of chunks of
 bytes to be compressed. It returns an iterator (ideally a generator)
 def name(self):
 return 'zlib'
 def bundletype(self):
 return 'gzip', 'GZ'
+def wireprotosupport(self):
+return compewireprotosupport('zlib', 20, 20)
 def compressstream(self, it, opts=None):
 opts = opts or {}
 z = zlib.compressobj(opts.get('level', -1))
 return 'bz2'
 def bundletype(self):
 return 'bzip2', 'BZ'
+# We declare a protocol name but don't advertise by default because
+# it is slow.
+def wireprotosupport(self):
+return compewireprotosupport('bzip2', 0, 0)
 def compressstream(self, it, opts=None):
 opts = opts or {}
 z = bz2.BZ2Compressor(opts.get('level', 9))
 for chunk in it:
 data = z.compress(chunk)
 def name(self):
 return 'none'
 def bundletype(self):
 return 'none', 'UN'
+# Clients always support uncompressed payloads. Servers don't because
+# unless you are on a fast network, uncompressed payloads can easily
+# saturate your network pipe.
+def wireprotosupport(self):
+return compewireprotosupport('none', 0, 10)
 def compressstream(self, it, opts=None):
 return it
 def decompressorreader(self, fh):
 return bool(self._module)
 def bundletype(self):
 return 'zstd', 'ZS'
+def wireprotosupport(self):
+return compewireprotosupport('zstd', 50, 50)
 def compressstream(self, it, opts=None):
 opts = opts or {}
 # zstd level 3 is almost always significantly faster than zlib
 # while providing no worse compression. It strikes a good balance
 # between speed and compression.

Mercurial > public > mercurial-scm > hg

comparison mercurial/util.py @ 30761:7283719e2bfd