# url.py - HTTP handling for mercurial

#

# Copyright 2005, 2006, 2007, 2008 Matt Mackall <mpm@selenic.com>

# Copyright 2006, 2007 Alexis S. L. Carvalho <alexis@cecm.usp.br>

# Copyright 2006 Vadim Gelfer <vadim.gelfer@gmail.com>

#

# This software may be used and distributed according to the terms of the

# GNU General Public License version 2 or any later version.

import urllib, urllib2, urlparse, httplib, os, re, socket, cStringIO

import __builtin__

from i18n import _

import keepalive, util

class url(object):

    """Reliable URL parser.

    This parses URLs and provides attributes for the following

    components:

    <scheme>://<user>:<passwd>@<host>:<port>/<path>?<query>#<fragment>

    Missing components are set to None. The only exception is

    fragment, which is set to '' if present but empty.

    If parse_fragment is False, fragment is included in query. If

    parse_query is False, query is included in path. If both are

    False, both fragment and query are included in path.

    See http://www.ietf.org/rfc/rfc2396.txt for more information.

    Examples:

    >>> url('http://www.ietf.org/rfc/rfc2396.txt')

    <url scheme: 'http', host: 'www.ietf.org', path: 'rfc/rfc2396.txt'>

    >>> url('ssh://[::1]:2200//home/joe/repo')

    <url scheme: 'ssh', host: '[::1]', port: '2200', path: '/home/joe/repo'>

    >>> url('file:///home/joe/repo')

    <url scheme: 'file', path: '/home/joe/repo'>

    >>> url('bundle:foo')

    <url scheme: 'bundle', path: 'foo'>

    >>> url('c:\\\\foo\\\\bar')

    <url path: 'c:\\\\foo\\\\bar'>

    Authentication credentials:

    >>> url('ssh://joe:xyz@x/repo')

    <url scheme: 'ssh', user: 'joe', passwd: 'xyz', host: 'x', path: 'repo'>

    >>> url('ssh://joe@x/repo')

    <url scheme: 'ssh', user: 'joe', host: 'x', path: 'repo'>

    Query strings and fragments:

    >>> url('http://host/a?b#c')

    <url scheme: 'http', host: 'host', path: 'a', query: 'b', fragment: 'c'>

    >>> url('http://host/a?b#c', parse_query=False, parse_fragment=False)

    <url scheme: 'http', host: 'host', path: 'a?b#c'>

    """

    _safechars = "!~*'()+"

    _safepchars = "/!~*'()+"

    def __init__(self, path, parse_query=True, parse_fragment=True):

        # We slowly chomp away at path until we have only the path left

        self.scheme = self.user = self.passwd = self.host = None

        self.port = self.path = self.query = self.fragment = None

        self._localpath = True

        # special case for Windows drive letters

        if has_drive_letter(path):

            self.path = path

            return

        if not path.startswith('/') and ':' in path:

            parts = path.split(':', 1)

            if parts[0]:

                self.scheme, path = parts

                self._localpath = False

        if not path:

            path = None

            if self._localpath:

                self.path = ''

                return

        else:

            if parse_fragment and '#' in path:

                path, self.fragment = path.split('#', 1)

                if not path:

                    path = None

            if self._localpath:

                self.path = path

                return

            if parse_query and '?' in path:

                path, self.query = path.split('?', 1)

                if not path:

                    path = None

                if not self.query:

                    self.query = None

            # // is required to specify a host/authority

            if path and path.startswith('//'):

                parts = path[2:].split('/', 1)

                if len(parts) > 1:

                    self.host, path = parts

                    path = path

                else:

                    self.host = parts[0]

                    path = None

                if not self.host:

                    self.host = None

                    if path:

                        path = '/' + path

            if self.host and '@' in self.host:

                self.user, self.host = self.host.rsplit('@', 1)

                if ':' in self.user:

                    self.user, self.passwd = self.user.split(':', 1)

                if not self.host:

                    self.host = None

            # Don't split on colons in IPv6 addresses without ports

            if (self.host and ':' in self.host and

                not (self.host.startswith('[') and self.host.endswith(']'))):

                self.host, self.port = self.host.rsplit(':', 1)

                if not self.host:

                    self.host = None

        self.path = path

        for a in ('user', 'passwd', 'host', 'port',

                  'path', 'query', 'fragment'):

            v = getattr(self, a)

            if v is not None:

                setattr(self, a, urllib.unquote(v))

    def __repr__(self):

        attrs = []

        for a in ('scheme', 'user', 'passwd', 'host', 'port', 'path',

                  'query', 'fragment'):

            v = getattr(self, a)

            if v is not None:

                attrs.append('%s: %r' % (a, v))

        return '<url %s>' % ', '.join(attrs)

    def __str__(self):

        """Join the URL's components back into a URL string.

        Examples:

        >>> str(url('http://user:pw@host:80/?foo#bar'))

        'http://user:pw@host:80/?foo#bar'

        >>> str(url('ssh://user:pw@[::1]:2200//home/joe#'))

        'ssh://user:pw@[::1]:2200//home/joe#'

        >>> str(url('http://localhost:80//'))

        'http://localhost:80//'

        >>> str(url('http://localhost:80/'))

        'http://localhost:80/'

        >>> str(url('http://localhost:80'))

        'http://localhost:80/'

        >>> str(url('bundle:foo'))

        'bundle:foo'

        >>> str(url('path'))

        'path'

        """

        if self._localpath:

            s = self.path

            if self.fragment:

                s += '#' + self.fragment

            return s

        s = self.scheme + ':'

        if (self.user or self.passwd or self.host or

            self.scheme and not self.path):

            s += '//'

        if self.user:

            s += urllib.quote(self.user, safe=self._safechars)

        if self.passwd:

            s += ':' + urllib.quote(self.passwd, safe=self._safechars)

        if self.user or self.passwd:

            s += '@'

        if self.host:

            if not (self.host.startswith('[') and self.host.endswith(']')):

                s += urllib.quote(self.host)

            else:

                s += self.host

        if self.port:

            s += ':' + urllib.quote(self.port)

        if self.host:

            s += '/'

        if self.path:

            s += urllib.quote(self.path, safe=self._safepchars)

        if self.query:

            s += '?' + urllib.quote(self.query, safe=self._safepchars)

        if self.fragment is not None:

            s += '#' + urllib.quote(self.fragment, safe=self._safepchars)

        return s

    def authinfo(self):

        user, passwd = self.user, self.passwd

        try:

            self.user, self.passwd = None, None

            s = str(self)

        finally:

            self.user, self.passwd = user, passwd

        if not self.user:

            return (s, None)

        return (s, (None, (str(self), self.host),

                    self.user, self.passwd or ''))

def has_scheme(path):

    return bool(url(path).scheme)

def has_drive_letter(path):

    return path[1:2] == ':' and path[0:1].isalpha()

def hidepassword(u):

    '''hide user credential in a url string'''

    u = url(u)

    if u.passwd:

        u.passwd = '***'

    return str(u)

def removeauth(u):

    '''remove all authentication information from a url string'''

    u = url(u)

    u.user = u.passwd = None

    return str(u)

def netlocsplit(netloc):

    '''split [user[:passwd]@]host[:port] into 4-tuple.'''

    a = netloc.find('@')

    if a == -1:

        user, passwd = None, None

    else:

        userpass, netloc = netloc[:a], netloc[a + 1:]

        c = userpass.find(':')

        if c == -1:

            user, passwd = urllib.unquote(userpass), None

        else:

            user = urllib.unquote(userpass[:c])

            passwd = urllib.unquote(userpass[c + 1:])

    c = netloc.find(':')

    if c == -1:

        host, port = netloc, None

    else:

        host, port = netloc[:c], netloc[c + 1:]

    return host, port, user, passwd

def netlocunsplit(host, port, user=None, passwd=None):

    '''turn host, port, user, passwd into [user[:passwd]@]host[:port].'''

    if port:

        hostport = host + ':' + port

    else:

        hostport = host

    if user:

        quote = lambda s: urllib.quote(s, safe='')

        if passwd:

            userpass = quote(user) + ':' + quote(passwd)

        else:

            userpass = quote(user)

        return userpass + '@' + hostport

    return hostport

def readauthforuri(ui, uri):

    # Read configuration

    config = dict()

    for key, val in ui.configitems('auth'):

        if '.' not in key:

            ui.warn(_("ignoring invalid [auth] key '%s'\n") % key)

            continue

        group, setting = key.rsplit('.', 1)

        gdict = config.setdefault(group, dict())

        if setting in ('username', 'cert', 'key'):

            val = util.expandpath(val)

        gdict[setting] = val

    # Find the best match

    scheme, hostpath = uri.split('://', 1)

    bestlen = 0

    bestauth = None

    for group, auth in config.iteritems():

        prefix = auth.get('prefix')

        if not prefix:

            continue

        p = prefix.split('://', 1)

        if len(p) > 1:

            schemes, prefix = [p[0]], p[1]

        else:

            schemes = (auth.get('schemes') or 'https').split()

        if (prefix == '*' or hostpath.startswith(prefix)) and \

            len(prefix) > bestlen and scheme in schemes:

            bestlen = len(prefix)

            bestauth = group, auth

    return bestauth

_safe = ('abcdefghijklmnopqrstuvwxyz'

         'ABCDEFGHIJKLMNOPQRSTUVWXYZ'

         '0123456789' '_.-/')

_safeset = None

_hex = None

def quotepath(path):

    '''quote the path part of a URL

    This is similar to urllib.quote, but it also tries to avoid

    quoting things twice (inspired by wget):

    >>> quotepath('abc def')

    'abc%20def'

    >>> quotepath('abc%20def')

    'abc%20def'

    >>> quotepath('abc%20 def')

    'abc%20%20def'

    >>> quotepath('abc def%20')

    'abc%20def%20'

    >>> quotepath('abc def%2')

    'abc%20def%252'

    >>> quotepath('abc def%')

    'abc%20def%25'

    '''

    global _safeset, _hex

    if _safeset is None:

        _safeset = set(_safe)

        _hex = set('abcdefABCDEF0123456789')

    l = list(path)

    for i in xrange(len(l)):

        c = l[i]

        if (c == '%' and i + 2 < len(l) and

            l[i + 1] in _hex and l[i + 2] in _hex):

            pass

        elif c not in _safeset:

            l[i] = '%%%02X' % ord(c)

    return ''.join(l)

class passwordmgr(urllib2.HTTPPasswordMgrWithDefaultRealm):

    def __init__(self, ui):

        urllib2.HTTPPasswordMgrWithDefaultRealm.__init__(self)

        self.ui = ui

    def find_user_password(self, realm, authuri):

        authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm.find_user_password(

            self, realm, authuri)

        user, passwd = authinfo

        if user and passwd:

            self._writedebug(user, passwd)

            return (user, passwd)

        if not user:

            res = readauthforuri(self.ui, authuri)

            if res:

                group, auth = res

                user, passwd = auth.get('username'), auth.get('password')

                self.ui.debug("using auth.%s.* for authentication\n" % group)

        if not user or not passwd:

            if not self.ui.interactive():

                raise util.Abort(_('http authorization required'))

            self.ui.write(_("http authorization required\n"))

            self.ui.write(_("realm: %s\n") % realm)

            if user:

                self.ui.write(_("user: %s\n") % user)

            else:

                user = self.ui.prompt(_("user:"), default=None)

            if not passwd:

                passwd = self.ui.getpass()

        self.add_password(realm, authuri, user, passwd)

        self._writedebug(user, passwd)

        return (user, passwd)

    def _writedebug(self, user, passwd):

        msg = _('http auth: user %s, password %s\n')

        self.ui.debug(msg % (user, passwd and '*' * len(passwd) or 'not set'))

class proxyhandler(urllib2.ProxyHandler):

    def __init__(self, ui):

        proxyurl = ui.config("http_proxy", "host") or os.getenv('http_proxy')

        # XXX proxyauthinfo = None

        if proxyurl:

            # proxy can be proper url or host[:port]

            if not (proxyurl.startswith('http:') or

                    proxyurl.startswith('https:')):

                proxyurl = 'http://' + proxyurl + '/'

            snpqf = urlparse.urlsplit(proxyurl)

            proxyscheme, proxynetloc, proxypath, proxyquery, proxyfrag = snpqf

            hpup = netlocsplit(proxynetloc)

            proxyhost, proxyport, proxyuser, proxypasswd = hpup

            if not proxyuser:

                proxyuser = ui.config("http_proxy", "user")

                proxypasswd = ui.config("http_proxy", "passwd")

            # see if we should use a proxy for this url

            no_list = ["localhost", "127.0.0.1"]

            no_list.extend([p.lower() for

                            p in ui.configlist("http_proxy", "no")])

            no_list.extend([p.strip().lower() for

                            p in os.getenv("no_proxy", '').split(',')

                            if p.strip()])

            # "http_proxy.always" config is for running tests on localhost

            if ui.configbool("http_proxy", "always"):

                self.no_list = []

            else:

                self.no_list = no_list

            proxyurl = urlparse.urlunsplit((

                proxyscheme, netlocunsplit(proxyhost, proxyport,

                                                proxyuser, proxypasswd or ''),

                proxypath, proxyquery, proxyfrag))

            proxies = {'http': proxyurl, 'https': proxyurl}

            ui.debug('proxying through http://%s:%s\n' %

                      (proxyhost, proxyport))

        else:

            proxies = {}

        # urllib2 takes proxy values from the environment and those

        # will take precedence if found, so drop them

        for env in ["HTTP_PROXY", "http_proxy", "no_proxy"]:

            try:

                if env in os.environ:

                    del os.environ[env]

            except OSError:

                pass

        urllib2.ProxyHandler.__init__(self, proxies)

        self.ui = ui

    def proxy_open(self, req, proxy, type_):

        host = req.get_host().split(':')[0]

        if host in self.no_list:

            return None

        # work around a bug in Python < 2.4.2

        # (it leaves a "\n" at the end of Proxy-authorization headers)

        baseclass = req.__class__

        class _request(baseclass):

            def add_header(self, key, val):

                if key.lower() == 'proxy-authorization':

                    val = val.strip()

                return baseclass.add_header(self, key, val)

        req.__class__ = _request

        return urllib2.ProxyHandler.proxy_open(self, req, proxy, type_)

class httpsendfile(object):

    """This is a wrapper around the objects returned by python's "open".

    Its purpose is to send file-like objects via HTTP and, to do so, it

    defines a __len__ attribute to feed the Content-Length header.

    """

    def __init__(self, ui, *args, **kwargs):

        # We can't just "self._data = open(*args, **kwargs)" here because there

        # is an "open" function defined in this module that shadows the global

        # one

        self.ui = ui

        self._data = __builtin__.open(*args, **kwargs)

        self.seek = self._data.seek

        self.close = self._data.close

        self.write = self._data.write

        self._len = os.fstat(self._data.fileno()).st_size

        self._pos = 0

        self._total = len(self) / 1024 * 2

    def read(self, *args, **kwargs):

        try:

            ret = self._data.read(*args, **kwargs)

        except EOFError:

            self.ui.progress(_('sending'), None)

        self._pos += len(ret)

        # We pass double the max for total because we currently have

        # to send the bundle twice in the case of a server that

        # requires authentication. Since we can't know until we try

        # once whether authentication will be required, just lie to

        # the user and maybe the push succeeds suddenly at 50%.

        self.ui.progress(_('sending'), self._pos / 1024,

                         unit=_('kb'), total=self._total)

        return ret