view mercurial/policy.py @ 37060:2ec1fb9de638

wireproto: add request IDs to frames One of my primary goals with the new wire protocol is to make operations faster and enable both client and server-side operations to scale to multiple CPU cores. One of the ways we make server interactions faster is by reducing the number of round trips to that server. With the existing wire protocol, the "batch" command facilitates executing multiple commands from a single request payload. The way it works is the requests for multiple commands are serialized. The server executes those commands sequentially then serializes all their results. As an optimization for reducing round trips, this is very effective. The technical implementation, however, is pretty bad and suffers from a number of deficiencies. For example, it creates a new place where authorization to run a command must be checked. (The lack of this checking in older Mercurial releases was CVE-2018-1000132.) The principles behind the "batch" command are sound. However, the execution is not. Therefore, I want to ditch "batch" in the new wire protocol and have protocol level support for issuing multiple requests in a single round trip. This commit introduces support in the frame-based wire protocol to facilitate this. We do this by adding a "request ID" to each frame. If a server sees frames associated with different "request IDs," it handles them as separate requests. All of this happening possibly as part of the same message from client to server (the same request body in the case of HTTP). We /could/ model the exchange the way pipelined HTTP requests do, where the server processes requests in order they are issued and received. But this artifically constrains scalability. A better model is to allow multi-requests to be executed concurrently and for responses to be sent and handled concurrently. So the specification explicitly allows this. There is some work to be done around specifying dependencies between multi-requests. We take the easy road for now and punt on this problem, declaring that if order is important, clients must not issue the request until responses to dependent requests have been received. This commit focuses on the boilerplate of implementing the request ID. The server reactor still can't manage multiple, in-flight request IDs. This will be addressed in a subsequent commit. Because the wire semantics have changed, we bump the version of the media type. Differential Revision: https://phab.mercurial-scm.org/D2869
author Gregory Szorc <gregory.szorc@gmail.com>
date Wed, 14 Mar 2018 16:51:34 -0700
parents f3c314020beb
children 2025bf60adb2
line wrap: on
line source

# policy.py - module policy logic for Mercurial.
#
# Copyright 2015 Gregory Szorc <gregory.szorc@gmail.com>
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.

from __future__ import absolute_import

import os
import sys

# Rules for how modules can be loaded. Values are:
#
#    c - require C extensions
#    allow - allow pure Python implementation when C loading fails
#    cffi - required cffi versions (implemented within pure module)
#    cffi-allow - allow pure Python implementation if cffi version is missing
#    py - only load pure Python modules
#
# By default, fall back to the pure modules so the in-place build can
# run without recompiling the C extensions. This will be overridden by
# __modulepolicy__ generated by setup.py.
policy = b'allow'
_packageprefs = {
    # policy: (versioned package, pure package)
    b'c': (r'cext', None),
    b'allow': (r'cext', r'pure'),
    b'cffi': (r'cffi', None),
    b'cffi-allow': (r'cffi', r'pure'),
    b'py': (None, r'pure'),
}

try:
    from . import __modulepolicy__
    policy = __modulepolicy__.modulepolicy
except ImportError:
    pass

# PyPy doesn't load C extensions.
#
# The canonical way to do this is to test platform.python_implementation().
# But we don't import platform and don't bloat for it here.
if r'__pypy__' in sys.builtin_module_names:
    policy = b'cffi'

# Environment variable can always force settings.
if sys.version_info[0] >= 3:
    if r'HGMODULEPOLICY' in os.environ:
        policy = os.environ[r'HGMODULEPOLICY'].encode(r'utf-8')
else:
    policy = os.environ.get(r'HGMODULEPOLICY', policy)

def _importfrom(pkgname, modname):
    # from .<pkgname> import <modname> (where . is looked through this module)
    fakelocals = {}
    pkg = __import__(pkgname, globals(), fakelocals, [modname], level=1)
    try:
        fakelocals[modname] = mod = getattr(pkg, modname)
    except AttributeError:
        raise ImportError(r'cannot import name %s' % modname)
    # force import; fakelocals[modname] may be replaced with the real module
    getattr(mod, r'__doc__', None)
    return fakelocals[modname]

# keep in sync with "version" in C modules
_cextversions = {
    (r'cext', r'base85'): 1,
    (r'cext', r'bdiff'): 3,
    (r'cext', r'diffhelpers'): 1,
    (r'cext', r'mpatch'): 1,
    (r'cext', r'osutil'): 4,
    (r'cext', r'parsers'): 4,
}

# map import request to other package or module
_modredirects = {
    (r'cext', r'charencode'): (r'cext', r'parsers'),
    (r'cffi', r'base85'): (r'pure', r'base85'),
    (r'cffi', r'charencode'): (r'pure', r'charencode'),
    (r'cffi', r'diffhelpers'): (r'pure', r'diffhelpers'),
    (r'cffi', r'parsers'): (r'pure', r'parsers'),
}

def _checkmod(pkgname, modname, mod):
    expected = _cextversions.get((pkgname, modname))
    actual = getattr(mod, r'version', None)
    if actual != expected:
        raise ImportError(r'cannot import module %s.%s '
                          r'(expected version: %d, actual: %r)'
                          % (pkgname, modname, expected, actual))

def importmod(modname):
    """Import module according to policy and check API version"""
    try:
        verpkg, purepkg = _packageprefs[policy]
    except KeyError:
        raise ImportError(r'invalid HGMODULEPOLICY %r' % policy)
    assert verpkg or purepkg
    if verpkg:
        pn, mn = _modredirects.get((verpkg, modname), (verpkg, modname))
        try:
            mod = _importfrom(pn, mn)
            if pn == verpkg:
                _checkmod(pn, mn, mod)
            return mod
        except ImportError:
            if not purepkg:
                raise
    pn, mn = _modredirects.get((purepkg, modname), (purepkg, modname))
    return _importfrom(pn, mn)