mercurial-scm/hg: mercurial/wireprotov2server.py comparison

comparison mercurial/wireprotov2server.py @ 40178:46a40bce3ae0

wireprotov2: define and implement "filesdata" command Previously, the only way to access file revision data was the "filedata" command. This command is useful to have. But, it only allowed resolving revision data for a single file. This meant that clients needed to send 1 command for each tracked path they were seeking data on. Furthermore, those commands would need to enumerate the exact file nodes they wanted data for. This approach meant that clients were sending a lot of data to remotes in order to request file data. e.g. if there were 1M file revisions, we'd need at least 20,000,000 bytes just to encode file nodes! Many clients on the internet don't have that kind of upload capacity. In order to limit the amount of data that clients must send, we'll need more efficient ways to request repository data. This commit defines and implements a new "filesdata" command. This command allows the retrieval of data for multiple files by specifying changeset revisions and optional file patterns. The command figures out what file revisions are "relevant" and sends them in bulk. The logic around choosing which file revisions to send in the case of haveparents not being set is overly simple and will over-send files. We will need more smarts here eventually. (Specifically, the client will need to tell the server which revisions it knows about.) This work is deferred until a later time. Differential Revision: https://phab.mercurial-scm.org/D4981

author	Gregory Szorc <gregory.szorc@gmail.com>
date	Wed, 03 Oct 2018 12:54:39 -0700
parents	41e2633bcd00
children	ed55a0077490

comparison

equal deleted inserted replaced

-:41e2633bcd00
+:46a40bce3ae0
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 from __future__ import absolute_import
+import collections
 import contextlib
 import hashlib
 from .i18n import _
 from .node import (
 )
 from . import (
 discovery,
 encoding,
 error,
+match as matchmod,
 narrowspec,
 pycompat,
 wireprotoframing,
 wireprototypes,
 )
 yield d
 for extra in followingdata:
 yield extra
+def makefilematcher(repo, pathfilter):
+"""Construct a matcher from a path filter dict."""
+# Validate values.
+if pathfilter:
+for key in (b'include', b'exclude'):
+for pattern in pathfilter.get(key, []):
+if not pattern.startswith((b'path:', b'rootfilesin:')):
+raise error.WireprotoCommandError(
+'%s pattern must begin with `path:` or `rootfilesin:`; '
+'got %s', (key, pattern))
+if pathfilter:
+matcher = matchmod.match(repo.root, b'',
+include=pathfilter.get(b'include', []),
+exclude=pathfilter.get(b'exclude', []))
+else:
+matcher = matchmod.match(repo.root, b'')
+# Requested patterns could include files not in the local store. So
+# filter those out.
+return matchmod.intersectmatchers(repo.narrowmatch(), matcher)
 @wireprotocommand(
 'filedata',
 args={
 'haveparents': {
 'type': 'bool',
 # TODO censoring a file revision won't invalidate the cache.
 # Figure out a way to take censoring into account when deriving
 # the cache key.
 cachekeyfn=makecommandcachekeyfn('filedata', 1, allargs=True))
 def filedata(repo, proto, haveparents, nodes, fields, path):
+# TODO this API allows access to file revisions that are attached to
+# secret changesets. filesdata does not have this problem. Maybe this
+# API should be deleted?
 try:
 # Extensions may wish to access the protocol handler.
 store = getfilestore(repo, proto, path)
 except FileAccessError as e:
 raise error.WireprotoCommandError(e.msg, e.args)
 b'totalitems': len(nodes),
 }
 for o in emitfilerevisions(revisions, fields):
 yield o
+def filesdatacapabilities(repo, proto):
+batchsize = repo.ui.configint(
+b'experimental', b'server.filesdata.recommended-batch-size')
+return {
+b'recommendedbatchsize': batchsize,
+}
+@wireprotocommand(
+'filesdata',
+args={
+'haveparents': {
+'type': 'bool',
+'default': lambda: False,
+'example': True,
+},
+'fields': {
+'type': 'set',
+'default': set,
+'example': {b'parents', b'revision'},
+'validvalues': {b'firstchangeset', b'parents', b'revision'},
+},
+'pathfilter': {
+'type': 'dict',
+'default': lambda: None,
+'example': {b'include': [b'path:tests']},
+},
+'revisions': {
+'type': 'list',
+'example': [{
+b'type': b'changesetexplicit',
+b'nodes': [b'abcdef...'],
+}],
+},
+},
+permission='pull',
+# TODO censoring a file revision won't invalidate the cache.
+# Figure out a way to take censoring into account when deriving
+# the cache key.
+cachekeyfn=makecommandcachekeyfn('filesdata', 1, allargs=True),
+extracapabilitiesfn=filesdatacapabilities)
+def filesdata(repo, proto, haveparents, fields, pathfilter, revisions):
+# TODO This should operate on a repo that exposes obsolete changesets. There
+# is a race between a client making a push that obsoletes a changeset and
+# another client fetching files data for that changeset. If a client has a
+# changeset, it should probably be allowed to access files data for that
+# changeset.
+cl = repo.changelog
+outgoing = resolvenodes(repo, revisions)
+filematcher = makefilematcher(repo, pathfilter)
+# Figure out what needs to be emitted.
+changedpaths = set()
+fnodes = collections.defaultdict(set)
+for node in outgoing:
+ctx = repo[node]
+changedpaths.update(ctx.files())
+changedpaths = sorted(p for p in changedpaths if filematcher(p))
+# If ancestors are known, we send file revisions having a linkrev in the
+# outgoing set of changeset revisions.
+if haveparents:
+outgoingclrevs = set(cl.rev(n) for n in outgoing)
+for path in changedpaths:
+try:
+store = getfilestore(repo, proto, path)
+except FileAccessError as e:
+raise error.WireprotoCommandError(e.msg, e.args)
+for rev in store:
+linkrev = store.linkrev(rev)
+if linkrev in outgoingclrevs:
+fnodes[path].add(store.node(rev))
+# If ancestors aren't known, we walk the manifests and send all
+# encountered file revisions.
+else:
+for node in outgoing:
+mctx = repo[node].manifestctx()
+for path, fnode in mctx.read().items():
+if filematcher(path):
+fnodes[path].add(fnode)
+yield {
+b'totalpaths': len(fnodes),
+b'totalitems': sum(len(v) for v in fnodes.values())
+}
+for path, filenodes in sorted(fnodes.items()):
+try:
+store = getfilestore(repo, proto, path)
+except FileAccessError as e:
+raise error.WireprotoCommandError(e.msg, e.args)
+yield {
+b'path': path,
+b'totalitems': len(filenodes),
+}
+revisions = store.emitrevisions(filenodes,
+revisiondata=b'revision' in fields,
+assumehaveparentrevisions=haveparents)
+for o in emitfilerevisions(revisions, fields):
+yield o
 @wireprotocommand(
 'heads',
 args={
 'publiconly': {

Mercurial > public > mercurial-scm > hg

comparison mercurial/wireprotov2server.py @ 40178:46a40bce3ae0