# phabricator.py - simple Phabricator integration

#

# Copyright 2017 Facebook, Inc.

#

# This software may be used and distributed according to the terms of the

# GNU General Public License version 2 or any later version.

"""simple Phabricator integration (EXPERIMENTAL)

This extension provides a ``phabsend`` command which sends a stack of

changesets to Phabricator, and a ``phabread`` command which prints a stack of

revisions in a format suitable for :hg:`import`, and a ``phabupdate`` command

to update statuses in batch.

By default, Phabricator requires ``Test Plan`` which might prevent some

changeset from being sent. The requirement could be disabled by changing

``differential.require-test-plan-field`` config server side.

Config::

    [phabricator]

    # Phabricator URL

    url = https://phab.example.com/

    # Repo callsign. If a repo has a URL https://$HOST/diffusion/FOO, then its

    # callsign is "FOO".

    callsign = FOO

    # curl command to use. If not set (default), use builtin HTTP library to

    # communicate. If set, use the specified curl command. This could be useful

    # if you need to specify advanced options that is not easily supported by

    # the internal library.

    curlcmd = curl --connect-timeout 2 --retry 3 --silent

    [auth]

    example.schemes = https

    example.prefix = phab.example.com

    # API token. Get it from https://$HOST/conduit/login/

    example.phabtoken = cli-xxxxxxxxxxxxxxxxxxxxxxxxxxxx

"""

from __future__ import absolute_import

import contextlib

import itertools

import json

import operator

import re

from mercurial.node import bin, nullid

from mercurial.i18n import _

from mercurial.pycompat import getattr

from mercurial import (

    cmdutil,

    context,

    encoding,

    error,

    exthelper,

    httpconnection as httpconnectionmod,

    mdiff,

    obsutil,

    parser,

    patch,

    phases,

    pycompat,

    scmutil,

    smartset,

    tags,

    templatefilters,

    templateutil,

    url as urlmod,

    util,

)

from mercurial.utils import (

    procutil,

    stringutil,

)

# Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for

# extensions which SHIP WITH MERCURIAL. Non-mainline extensions should

# be specifying the version(s) of Mercurial they are tested with, or

# leave the attribute unspecified.

testedwith = b'ships-with-hg-core'

eh = exthelper.exthelper()

cmdtable = eh.cmdtable

command = eh.command

configtable = eh.configtable

templatekeyword = eh.templatekeyword

# developer config: phabricator.batchsize

eh.configitem(

    b'phabricator', b'batchsize', default=12,

)

eh.configitem(

    b'phabricator', b'callsign', default=None,

)

eh.configitem(

    b'phabricator', b'curlcmd', default=None,

)

# developer config: phabricator.repophid

eh.configitem(

    b'phabricator', b'repophid', default=None,

)

eh.configitem(

    b'phabricator', b'url', default=None,

)

eh.configitem(

    b'phabsend', b'confirm', default=False,

)

colortable = {

    b'phabricator.action.created': b'green',

    b'phabricator.action.skipped': b'magenta',

    b'phabricator.action.updated': b'magenta',

    b'phabricator.desc': b'',

    b'phabricator.drev': b'bold',

    b'phabricator.node': b'',

}

_VCR_FLAGS = [

    (

        b'',

        b'test-vcr',

        b'',

        _(

            b'Path to a vcr file. If nonexistent, will record a new vcr transcript'

            b', otherwise will mock all http requests using the specified vcr file.'

            b' (ADVANCED)'

        ),

    ),

]

def vcrcommand(name, flags, spec, helpcategory=None, optionalrepo=False):

    fullflags = flags + _VCR_FLAGS

    def hgmatcher(r1, r2):

        if r1.uri != r2.uri or r1.method != r2.method:

            return False

        r1params = r1.body.split(b'&')

        r2params = r2.body.split(b'&')

        return set(r1params) == set(r2params)

    def sanitiserequest(request):

        request.body = re.sub(

            br'cli-[a-z0-9]+', br'cli-hahayouwish', request.body

        )

        return request

    def sanitiseresponse(response):

        if r'set-cookie' in response[r'headers']:

            del response[r'headers'][r'set-cookie']

        return response

    def decorate(fn):

        def inner(*args, **kwargs):

            cassette = pycompat.fsdecode(kwargs.pop(r'test_vcr', None))

            if cassette:

                import hgdemandimport

                with hgdemandimport.deactivated():

                    import vcr as vcrmod

                    import vcr.stubs as stubs

                    vcr = vcrmod.VCR(

                        serializer=r'json',

                        before_record_request=sanitiserequest,

                        before_record_response=sanitiseresponse,

                        custom_patches=[

                            (

                                urlmod,

                                r'httpconnection',

                                stubs.VCRHTTPConnection,

                            ),

                            (

                                urlmod,

                                r'httpsconnection',

                                stubs.VCRHTTPSConnection,

                            ),

                        ],

                    )

                    vcr.register_matcher(r'hgmatcher', hgmatcher)

                    with vcr.use_cassette(cassette, match_on=[r'hgmatcher']):

                        return fn(*args, **kwargs)

            return fn(*args, **kwargs)

        inner.__name__ = fn.__name__

        inner.__doc__ = fn.__doc__

        return command(

            name,

            fullflags,

            spec,

            helpcategory=helpcategory,

            optionalrepo=optionalrepo,

        )(inner)

    return decorate

def urlencodenested(params):

    """like urlencode, but works with nested parameters.

    For example, if params is {'a': ['b', 'c'], 'd': {'e': 'f'}}, it will be

    flattened to {'a[0]': 'b', 'a[1]': 'c', 'd[e]': 'f'} and then passed to

    urlencode. Note: the encoding is consistent with PHP's http_build_query.

    """

    flatparams = util.sortdict()

    def process(prefix, obj):

        if isinstance(obj, bool):

            obj = {True: b'true', False: b'false'}[obj]  # Python -> PHP form

        lister = lambda l: [(b'%d' % k, v) for k, v in enumerate(l)]

        items = {list: lister, dict: lambda x: x.items()}.get(type(obj))

        if items is None:

            flatparams[prefix] = obj

        else:

            for k, v in items(obj):

                if prefix:

                    process(b'%s[%s]' % (prefix, k), v)

                else:

                    process(k, v)

    process(b'', params)

    return util.urlreq.urlencode(flatparams)

def readurltoken(ui):

    """return conduit url, token and make sure they exist

    Currently read from [auth] config section. In the future, it might

    make sense to read from .arcconfig and .arcrc as well.

    """

    url = ui.config(b'phabricator', b'url')

    if not url:

        raise error.Abort(

            _(b'config %s.%s is required') % (b'phabricator', b'url')

        )

    res = httpconnectionmod.readauthforuri(ui, url, util.url(url).user)

    token = None

    if res:

        group, auth = res

        ui.debug(b"using auth.%s.* for authentication\n" % group)

        token = auth.get(b'phabtoken')

    if not token:

        raise error.Abort(

            _(b'Can\'t find conduit token associated to %s') % (url,)

        )

    return url, token

def callconduit(ui, name, params):

    """call Conduit API, params is a dict. return json.loads result, or None"""

    host, token = readurltoken(ui)

    url, authinfo = util.url(b'/'.join([host, b'api', name])).authinfo()

    ui.debug(b'Conduit Call: %s %s\n' % (url, pycompat.byterepr(params)))

    params = params.copy()

    params[b'api.token'] = token

    data = urlencodenested(params)

    curlcmd = ui.config(b'phabricator', b'curlcmd')

    if curlcmd:

        sin, sout = procutil.popen2(

            b'%s -d @- %s' % (curlcmd, procutil.shellquote(url))

        )

        sin.write(data)

        sin.close()

        body = sout.read()

    else:

        urlopener = urlmod.opener(ui, authinfo)

        request = util.urlreq.request(pycompat.strurl(url), data=data)

        with contextlib.closing(urlopener.open(request)) as rsp:

            body = rsp.read()

    ui.debug(b'Conduit Response: %s\n' % body)

    parsed = pycompat.rapply(

        lambda x: encoding.unitolocal(x)

        if isinstance(x, pycompat.unicode)

        else x,

        # json.loads only accepts bytes from py3.6+

        json.loads(encoding.unifromlocal(body)),

    )

    if parsed.get(b'error_code'):

        msg = _(b'Conduit Error (%s): %s') % (

            parsed[b'error_code'],

            parsed[b'error_info'],

        )

        raise error.Abort(msg)

    return parsed[b'result']

@vcrcommand(b'debugcallconduit', [], _(b'METHOD'), optionalrepo=True)

def debugcallconduit(ui, repo, name):

    """call Conduit API

    Call parameters are read from stdin as a JSON blob. Result will be written

    to stdout as a JSON blob.

    """

    # json.loads only accepts bytes from 3.6+

    rawparams = encoding.unifromlocal(ui.fin.read())

    # json.loads only returns unicode strings

    params = pycompat.rapply(

        lambda x: encoding.unitolocal(x)

        if isinstance(x, pycompat.unicode)

        else x,

        json.loads(rawparams),

    )

    # json.dumps only accepts unicode strings

    result = pycompat.rapply(

        lambda x: encoding.unifromlocal(x) if isinstance(x, bytes) else x,

        callconduit(ui, name, params),

    )

    s = json.dumps(result, sort_keys=True, indent=2, separators=(u',', u': '))

    ui.write(b'%s\n' % encoding.unitolocal(s))

def getrepophid(repo):

    """given callsign, return repository PHID or None"""

    # developer config: phabricator.repophid

    repophid = repo.ui.config(b'phabricator', b'repophid')

    if repophid:

        return repophid

    callsign = repo.ui.config(b'phabricator', b'callsign')

    if not callsign:

        return None

    query = callconduit(

        repo.ui,

        b'diffusion.repository.search',

        {b'constraints': {b'callsigns': [callsign]}},

    )

    if len(query[b'data']) == 0:

        return None

    repophid = query[b'data'][0][b'phid']

    repo.ui.setconfig(b'phabricator', b'repophid', repophid)

    return repophid

_differentialrevisiontagre = re.compile(br'\AD([1-9][0-9]*)\Z')

_differentialrevisiondescre = re.compile(

    br'^Differential Revision:\s*(?P<url>(?:.*)D(?P<id>[1-9][0-9]*))$', re.M

)

def getoldnodedrevmap(repo, nodelist):

    """find previous nodes that has been sent to Phabricator

    return {node: (oldnode, Differential diff, Differential Revision ID)}

    for node in nodelist with known previous sent versions, or associated

    Differential Revision IDs. ``oldnode`` and ``Differential diff`` could

    be ``None``.

    Examines commit messages like "Differential Revision:" to get the

    association information.

    If such commit message line is not found, examines all precursors and their

    tags. Tags with format like "D1234" are considered a match and the node

    with that tag, and the number after "D" (ex. 1234) will be returned.

    The ``old node``, if not None, is guaranteed to be the last diff of

    corresponding Differential Revision, and exist in the repo.

    """

    unfi = repo.unfiltered()

    nodemap = unfi.changelog.nodemap

    result = {}  # {node: (oldnode?, lastdiff?, drev)}

    toconfirm = {}  # {node: (force, {precnode}, drev)}

    for node in nodelist:

        ctx = unfi[node]

        # For tags like "D123", put them into "toconfirm" to verify later

        precnodes = list(obsutil.allpredecessors(unfi.obsstore, [node]))

        for n in precnodes:

            if n in nodemap:

                for tag in unfi.nodetags(n):

                    m = _differentialrevisiontagre.match(tag)

                    if m:

                        toconfirm[node] = (0, set(precnodes), int(m.group(1)))

                        continue

        # Check commit message

        m = _differentialrevisiondescre.search(ctx.description())

        if m:

            toconfirm[node] = (1, set(precnodes), int(m.group(r'id')))

    # Double check if tags are genuine by collecting all old nodes from

    # Phabricator, and expect precursors overlap with it.

    if toconfirm:

        drevs = [drev for force, precs, drev in toconfirm.values()]

        alldiffs = callconduit(

            unfi.ui, b'differential.querydiffs', {b'revisionIDs': drevs}

        )

        getnode = lambda d: bin(getdiffmeta(d).get(b'node', b'')) or None

        for newnode, (force, precset, drev) in toconfirm.items():

            diffs = [

                d for d in alldiffs.values() if int(d[b'revisionID']) == drev

            ]

            # "precursors" as known by Phabricator

            phprecset = set(getnode(d) for d in diffs)

            # Ignore if precursors (Phabricator and local repo) do not overlap,

            # and force is not set (when commit message says nothing)

            if not force and not bool(phprecset & precset):

                tagname = b'D%d' % drev

                tags.tag(

                    repo,

                    tagname,

                    nullid,

                    message=None,

                    user=None,

                    date=None,

                    local=True,

                )

                unfi.ui.warn(

                    _(

                        b'D%s: local tag removed - does not match '

                        b'Differential history\n'

                    )

                    % drev

                )

                continue

            # Find the last node using Phabricator metadata, and make sure it

            # exists in the repo

            oldnode = lastdiff = None

            if diffs:

                lastdiff = max(diffs, key=lambda d: int(d[b'id']))

                oldnode = getnode(lastdiff)

                if oldnode and oldnode not in nodemap:

                    oldnode = None

            result[newnode] = (oldnode, lastdiff, drev)

    return result

def getdiff(ctx, diffopts):

    """plain-text diff without header (user, commit message, etc)"""

    output = util.stringio()

    for chunk, _label in patch.diffui(

        ctx.repo(), ctx.p1().node(), ctx.node(), None, opts=diffopts

    ):

        output.write(chunk)

    return output.getvalue()

def creatediff(ctx):

    """create a Differential Diff"""

    repo = ctx.repo()

    repophid = getrepophid(repo)

    # Create a "Differential Diff" via "differential.createrawdiff" API

    params = {b'diff': getdiff(ctx, mdiff.diffopts(git=True, context=32767))}

    if repophid:

        params[b'repositoryPHID'] = repophid

    diff = callconduit(repo.ui, b'differential.createrawdiff', params)

    if not diff:

        raise error.Abort(_(b'cannot create diff for %s') % ctx)

    return diff

def writediffproperties(ctx, diff):

    """write metadata to diff so patches could be applied losslessly"""

    params = {

        b'diff_id': diff[b'id'],

        b'name': b'hg:meta',

        b'data': templatefilters.json(

            {

                b'user': ctx.user(),

                b'date': b'%d %d' % ctx.date(),

                b'branch': ctx.branch(),

                b'node': ctx.hex(),

                b'parent': ctx.p1().hex(),

            }

        ),

    }

    callconduit(ctx.repo().ui, b'differential.setdiffproperty', params)

    params = {

        b'diff_id': diff[b'id'],