# bundle2.py - generic container format to transmit arbitrary data.

#

# Copyright 2013 Facebook, Inc.

#

# This software may be used and distributed according to the terms of the

# GNU General Public License version 2 or any later version.

"""Handling of the new bundle2 format

The goal of bundle2 is to act as an atomically packet to transmit a set of

payloads in an application agnostic way. It consist in a sequence of "parts"

that will be handed to and processed by the application layer.

General format architecture

===========================

The format is architectured as follow

 - magic string

 - stream level parameters

 - payload parts (any number)

 - end of stream marker.

the Binary format

============================

All numbers are unsigned and big-endian.

stream level parameters

------------------------

Binary format is as follow

:params size: int32

  The total number of Bytes used by the parameters

:params value: arbitrary number of Bytes

  A blob of `params size` containing the serialized version of all stream level

  parameters.

  The blob contains a space separated list of parameters. Parameters with value

  are stored in the form `<name>=<value>`. Both name and value are urlquoted.

  Empty name are obviously forbidden.

  Name MUST start with a letter. If this first letter is lower case, the

  parameter is advisory and can be safely ignored. However when the first

  letter is capital, the parameter is mandatory and the bundling process MUST

  stop if he is not able to proceed it.

  Stream parameters use a simple textual format for two main reasons:

  - Stream level parameters should remain simple and we want to discourage any

    crazy usage.

  - Textual data allow easy human inspection of a bundle2 header in case of

    troubles.

  Any Applicative level options MUST go into a bundle2 part instead.

Payload part

------------------------

Binary format is as follow

:header size: int32

  The total number of Bytes used by the part header. When the header is empty

  (size = 0) this is interpreted as the end of stream marker.

:header:

    The header defines how to interpret the part. It contains two piece of

    data: the part type, and the part parameters.

    The part type is used to route an application level handler, that can

    interpret payload.

    Part parameters are passed to the application level handler.  They are

    meant to convey information that will help the application level object to

    interpret the part payload.

    The binary format of the header is has follow

    :typesize: (one byte)

    :parttype: alphanumerical part name (restricted to [a-zA-Z0-9_:-]*)

    :partid: A 32bits integer (unique in the bundle) that can be used to refer

             to this part.

    :parameters:

        Part's parameter may have arbitrary content, the binary structure is::

            <mandatory-count><advisory-count><param-sizes><param-data>

        :mandatory-count: 1 byte, number of mandatory parameters

        :advisory-count:  1 byte, number of advisory parameters

        :param-sizes:

            N couple of bytes, where N is the total number of parameters. Each

            couple contains (<size-of-key>, <size-of-value) for one parameter.

        :param-data:

            A blob of bytes from which each parameter key and value can be

            retrieved using the list of size couples stored in the previous

            field.

            Mandatory parameters comes first, then the advisory ones.

            Each parameter's key MUST be unique within the part.

:payload:

    payload is a series of `<chunksize><chunkdata>`.

    `chunksize` is an int32, `chunkdata` are plain bytes (as much as

    `chunksize` says)` The payload part is concluded by a zero size chunk.

    The current implementation always produces either zero or one chunk.

    This is an implementation limitation that will ultimately be lifted.

    `chunksize` can be negative to trigger special case processing. No such

    processing is in place yet.

Bundle processing

============================

Each part is processed in order using a "part handler". Handler are registered

for a certain part type.

The matching of a part to its handler is case insensitive. The case of the

part type is used to know if a part is mandatory or advisory. If the Part type

contains any uppercase char it is considered mandatory. When no handler is

known for a Mandatory part, the process is aborted and an exception is raised.

If the part is advisory and no handler is known, the part is ignored. When the

process is aborted, the full bundle is still read from the stream to keep the

channel usable. But none of the part read from an abort are processed. In the

future, dropping the stream may become an option for channel we do not care to

preserve.

"""

from __future__ import absolute_import, division

import errno

import os

import re

import string

import struct

import sys

from .i18n import _

from . import (

    bookmarks,

    changegroup,

    error,

    node as nodemod,

    obsolete,

    phases,

    pushkey,

    pycompat,

    tags,

    url,

    util,

)

urlerr = util.urlerr

urlreq = util.urlreq

_pack = struct.pack

_unpack = struct.unpack

_fstreamparamsize = '>i'

_fpartheadersize = '>i'

_fparttypesize = '>B'

_fpartid = '>I'

_fpayloadsize = '>i'

_fpartparamcount = '>BB'

preferedchunksize = 4096

_parttypeforbidden = re.compile('[^a-zA-Z0-9_:-]')

def outdebug(ui, message):

    """debug regarding output stream (bundling)"""

    if ui.configbool('devel', 'bundle2.debug'):

        ui.debug('bundle2-output: %s\n' % message)

def indebug(ui, message):

    """debug on input stream (unbundling)"""

    if ui.configbool('devel', 'bundle2.debug'):

        ui.debug('bundle2-input: %s\n' % message)

def validateparttype(parttype):

    """raise ValueError if a parttype contains invalid character"""

    if _parttypeforbidden.search(parttype):

        raise ValueError(parttype)

def _makefpartparamsizes(nbparams):

    """return a struct format to read part parameter sizes

    The number parameters is variable so we need to build that format

    dynamically.

    """

    return '>'+('BB'*nbparams)

parthandlermapping = {}

def parthandler(parttype, params=()):

    """decorator that register a function as a bundle2 part handler

    eg::

        @parthandler('myparttype', ('mandatory', 'param', 'handled'))

        def myparttypehandler(...):

            '''process a part of type "my part".'''

            ...

    """

    validateparttype(parttype)

    def _decorator(func):

        lparttype = parttype.lower() # enforce lower case matching.

        assert lparttype not in parthandlermapping

        parthandlermapping[lparttype] = func

        func.params = frozenset(params)

        return func

    return _decorator

class unbundlerecords(object):

    """keep record of what happens during and unbundle

    New records are added using `records.add('cat', obj)`. Where 'cat' is a

    category of record and obj is an arbitrary object.

    `records['cat']` will return all entries of this category 'cat'.

    Iterating on the object itself will yield `('category', obj)` tuples

    for all entries.

    All iterations happens in chronological order.

    """

    def __init__(self):

        self._categories = {}

        self._sequences = []

        self._replies = {}

    def add(self, category, entry, inreplyto=None):

        """add a new record of a given category.

        The entry can then be retrieved in the list returned by

        self['category']."""

        self._categories.setdefault(category, []).append(entry)

        self._sequences.append((category, entry))

        if inreplyto is not None:

            self.getreplies(inreplyto).add(category, entry)

    def getreplies(self, partid):

        """get the records that are replies to a specific part"""

        return self._replies.setdefault(partid, unbundlerecords())

    def __getitem__(self, cat):

        return tuple(self._categories.get(cat, ()))

    def __iter__(self):

        return iter(self._sequences)

    def __len__(self):

        return len(self._sequences)

    def __nonzero__(self):

        return bool(self._sequences)

    __bool__ = __nonzero__

class bundleoperation(object):

    """an object that represents a single bundling process

    Its purpose is to carry unbundle-related objects and states.

    A new object should be created at the beginning of each bundle processing.

    The object is to be returned by the processing function.

    The object has very little content now it will ultimately contain:

    * an access to the repo the bundle is applied to,

    * a ui object,

    * a way to retrieve a transaction to add changes to the repo,

    * a way to record the result of processing each part,

    * a way to construct a bundle response when applicable.

    """

    def __init__(self, repo, transactiongetter, captureoutput=True):

        self.repo = repo

        self.ui = repo.ui

        self.records = unbundlerecords()

        self.reply = None

        self.captureoutput = captureoutput

        self.hookargs = {}

        self._gettransaction = transactiongetter

    def gettransaction(self):

        transaction = self._gettransaction()

        if self.hookargs:

            # the ones added to the transaction supercede those added

            # to the operation.

            self.hookargs.update(transaction.hookargs)

            transaction.hookargs = self.hookargs

        # mark the hookargs as flushed.  further attempts to add to

        # hookargs will result in an abort.

        self.hookargs = None

        return transaction

    def addhookargs(self, hookargs):

        if self.hookargs is None:

            raise error.ProgrammingError('attempted to add hookargs to '

                                         'operation after transaction started')

        self.hookargs.update(hookargs)

class TransactionUnavailable(RuntimeError):

    pass

def _notransaction():

    """default method to get a transaction while processing a bundle

    Raise an exception to highlight the fact that no transaction was expected

    to be created"""

    raise TransactionUnavailable()

def applybundle(repo, unbundler, tr, source=None, url=None, **kwargs):

    # transform me into unbundler.apply() as soon as the freeze is lifted

    if isinstance(unbundler, unbundle20):

        tr.hookargs['bundle2'] = '1'

        if source is not None and 'source' not in tr.hookargs:

            tr.hookargs['source'] = source

        if url is not None and 'url' not in tr.hookargs:

            tr.hookargs['url'] = url

        return processbundle(repo, unbundler, lambda: tr)

    else:

        # the transactiongetter won't be used, but we might as well set it

        op = bundleoperation(repo, lambda: tr)

        _processchangegroup(op, unbundler, tr, source, url, **kwargs)

        return op

class partiterator(object):

    def __init__(self, repo, op, unbundler):

        self.repo = repo

        self.op = op

        self.unbundler = unbundler

        self.iterator = None

        self.count = 0

        self.current = None

    def __enter__(self):

        def func():

            itr = enumerate(self.unbundler.iterparts())

            for count, p in itr:

                self.count = count

                self.current = p

                yield p

                p.consume()

                self.current = None

        self.iterator = func()

        return self.iterator

    def __exit__(self, type, exc, tb):

        if not self.iterator:

            return

        # Only gracefully abort in a normal exception situation. User aborts

        # like Ctrl+C throw a KeyboardInterrupt which is not a base Exception,

        # and should not gracefully cleanup.

        if isinstance(exc, Exception):

            # Any exceptions seeking to the end of the bundle at this point are

            # almost certainly related to the underlying stream being bad.

            # And, chances are that the exception we're handling is related to

            # getting in that bad state. So, we swallow the seeking error and

            # re-raise the original error.

            seekerror = False

            try:

                if self.current:

                    # consume the part content to not corrupt the stream.

                    self.current.consume()

                for part in self.iterator:

                    # consume the bundle content

                    part.consume()

            except Exception:

                seekerror = True

            # Small hack to let caller code distinguish exceptions from bundle2

            # processing from processing the old format. This is mostly needed

            # to handle different return codes to unbundle according to the type

            # of bundle. We should probably clean up or drop this return code

            # craziness in a future version.

            exc.duringunbundle2 = True

            salvaged = []

            replycaps = None

            if self.op.reply is not None:

                salvaged = self.op.reply.salvageoutput()

                replycaps = self.op.reply.capabilities

            exc._replycaps = replycaps

            exc._bundle2salvagedoutput = salvaged

            # Re-raising from a variable loses the original stack. So only use

            # that form if we need to.

            if seekerror:

                raise exc

        self.repo.ui.debug('bundle2-input-bundle: %i parts total\n' %

                           self.count)

def processbundle(repo, unbundler, transactiongetter=None, op=None):

    """This function process a bundle, apply effect to/from a repo

    It iterates over each part then searches for and uses the proper handling

    code to process the part. Parts are processed in order.

    Unknown Mandatory part will abort the process.

    It is temporarily possible to provide a prebuilt bundleoperation to the

    function. This is used to ensure output is properly propagated in case of

    an error during the unbundling. This output capturing part will likely be

    reworked and this ability will probably go away in the process.

    """

    if op is None:

        if transactiongetter is None:

            transactiongetter = _notransaction

        op = bundleoperation(repo, transactiongetter)

    # todo:

    # - replace this is a init function soon.

    # - exception catching

    unbundler.params

    if repo.ui.debugflag:

        msg = ['bundle2-input-bundle:']

        if unbundler.params:

            msg.append(' %i params' % len(unbundler.params))

        if op._gettransaction is None or op._gettransaction is _notransaction:

            msg.append(' no-transaction')

        else:

            msg.append(' with-transaction')

        msg.append('\n')

        repo.ui.debug(''.join(msg))

    processparts(repo, op, unbundler)

    return op

def processparts(repo, op, unbundler):

    with partiterator(repo, op, unbundler) as parts:

        for part in parts:

            _processpart(op, part)

def _processchangegroup(op, cg, tr, source, url, **kwargs):

    ret = cg.apply(op.repo, tr, source, url, **kwargs)

    op.records.add('changegroup', {

        'return': ret,

    })

    return ret

def _gethandler(op, part):

    status = 'unknown' # used by debug output

    try:

        handler = parthandlermapping.get(part.type)

        if handler is None:

            status = 'unsupported-type'

            raise error.BundleUnknownFeatureError(parttype=part.type)

        indebug(op.ui, 'found a handler for part %s' % part.type)

        unknownparams = part.mandatorykeys - handler.params

        if unknownparams:

            unknownparams = list(unknownparams)

            unknownparams.sort()

            status = 'unsupported-params (%s)' % ', '.join(unknownparams)

            raise error.BundleUnknownFeatureError(parttype=part.type,

                                                  params=unknownparams)

        status = 'supported'

    except error.BundleUnknownFeatureError as exc:

        if part.mandatory: # mandatory parts

            raise

        indebug(op.ui, 'ignoring unsupported advisory part %s' % exc)

        return # skip to part processing