Mercurial > public > mercurial-scm > hg

comparison mercurial/url.py @ 43077:687b865b95ad

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
formatting: byteify all mercurial/ and hgext/ string literals Done with python3.7 contrib/byteify-strings.py -i $(hg files 'set:mercurial/**.py - mercurial/thirdparty/** + hgext/**.py - hgext/fsmonitor/pywatchman/** - mercurial/__init__.py') black -l 80 -t py33 -S $(hg files 'set:**.py - mercurial/thirdparty/** - "contrib/python-zstandard/**" - hgext/fsmonitor/pywatchman/**') # skip-blame mass-reformatting only Differential Revision: https://phab.mercurial-scm.org/D6972
author Augie Fackler <augie@google.com>
date Sun, 06 Oct 2019 09:48:39 -0400
parents 2372284d9457
children c59eb1560c44
comparison
equal deleted inserted replaced
43076:2372284d9457 43077:687b865b95ad
69 69
70 if not user or not passwd: 70 if not user or not passwd:
71 res = httpconnectionmod.readauthforuri(self.ui, authuri, user) 71 res = httpconnectionmod.readauthforuri(self.ui, authuri, user)
72 if res: 72 if res:
73 group, auth = res 73 group, auth = res
74 user, passwd = auth.get('username'), auth.get('password') 74 user, passwd = auth.get(b'username'), auth.get(b'password')
75 self.ui.debug("using auth.%s.* for authentication\n" % group) 75 self.ui.debug(b"using auth.%s.* for authentication\n" % group)
76 if not user or not passwd: 76 if not user or not passwd:
77 u = util.url(pycompat.bytesurl(authuri)) 77 u = util.url(pycompat.bytesurl(authuri))
78 u.query = None 78 u.query = None
79 if not self.ui.interactive(): 79 if not self.ui.interactive():
80 raise error.Abort( 80 raise error.Abort(
81 _('http authorization required for %s') 81 _(b'http authorization required for %s')
82 % util.hidepassword(bytes(u)) 82 % util.hidepassword(bytes(u))
83 ) 83 )
84 84
85 self.ui.write( 85 self.ui.write(
86 _("http authorization required for %s\n") 86 _(b"http authorization required for %s\n")
87 % util.hidepassword(bytes(u)) 87 % util.hidepassword(bytes(u))
88 ) 88 )
89 self.ui.write(_("realm: %s\n") % pycompat.bytesurl(realm)) 89 self.ui.write(_(b"realm: %s\n") % pycompat.bytesurl(realm))
90 if user: 90 if user:
91 self.ui.write(_("user: %s\n") % user) 91 self.ui.write(_(b"user: %s\n") % user)
92 else: 92 else:
93 user = self.ui.prompt(_("user:"), default=None) 93 user = self.ui.prompt(_(b"user:"), default=None)
94 94
95 if not passwd: 95 if not passwd:
96 passwd = self.ui.getpass() 96 passwd = self.ui.getpass()
97 97
98 self.passwddb.add_password(realm, authuri, user, passwd) 98 self.passwddb.add_password(realm, authuri, user, passwd)
99 self._writedebug(user, passwd) 99 self._writedebug(user, passwd)
100 return (pycompat.strurl(user), pycompat.strurl(passwd)) 100 return (pycompat.strurl(user), pycompat.strurl(passwd))
101 101
102 def _writedebug(self, user, passwd): 102 def _writedebug(self, user, passwd):
103 msg = _('http auth: user %s, password %s\n') 103 msg = _(b'http auth: user %s, password %s\n')
104 self.ui.debug(msg % (user, passwd and '*' * len(passwd) or 'not set')) 104 self.ui.debug(msg % (user, passwd and b'*' * len(passwd) or b'not set'))
105 105
106 def find_stored_password(self, authuri): 106 def find_stored_password(self, authuri):
107 return self.passwddb.find_user_password(None, authuri) 107 return self.passwddb.find_user_password(None, authuri)
108 108
109 109
110 class proxyhandler(urlreq.proxyhandler): 110 class proxyhandler(urlreq.proxyhandler):
111 def __init__(self, ui): 111 def __init__(self, ui):
112 proxyurl = ui.config("http_proxy", "host") or encoding.environ.get( 112 proxyurl = ui.config(b"http_proxy", b"host") or encoding.environ.get(
113 'http_proxy' 113 b'http_proxy'
114 ) 114 )
115 # XXX proxyauthinfo = None 115 # XXX proxyauthinfo = None
116 116
117 if proxyurl: 117 if proxyurl:
118 # proxy can be proper url or host[:port] 118 # proxy can be proper url or host[:port]
119 if not ( 119 if not (
120 proxyurl.startswith('http:') or proxyurl.startswith('https:') 120 proxyurl.startswith(b'http:') or proxyurl.startswith(b'https:')
121 ): 121 ):
122 proxyurl = 'http://' + proxyurl + '/' 122 proxyurl = b'http://' + proxyurl + b'/'
123 proxy = util.url(proxyurl) 123 proxy = util.url(proxyurl)
124 if not proxy.user: 124 if not proxy.user:
125 proxy.user = ui.config("http_proxy", "user") 125 proxy.user = ui.config(b"http_proxy", b"user")
126 proxy.passwd = ui.config("http_proxy", "passwd") 126 proxy.passwd = ui.config(b"http_proxy", b"passwd")
127 127
128 # see if we should use a proxy for this url 128 # see if we should use a proxy for this url
129 no_list = ["localhost", "127.0.0.1"] 129 no_list = [b"localhost", b"127.0.0.1"]
130 no_list.extend( 130 no_list.extend(
131 [p.lower() for p in ui.configlist("http_proxy", "no")] 131 [p.lower() for p in ui.configlist(b"http_proxy", b"no")]
132 ) 132 )
133 no_list.extend( 133 no_list.extend(
134 [ 134 [
135 p.strip().lower() 135 p.strip().lower()
136 for p in encoding.environ.get("no_proxy", '').split(',') 136 for p in encoding.environ.get(b"no_proxy", b'').split(b',')
137 if p.strip() 137 if p.strip()
138 ] 138 ]
139 ) 139 )
140 # "http_proxy.always" config is for running tests on localhost 140 # "http_proxy.always" config is for running tests on localhost
141 if ui.configbool("http_proxy", "always"): 141 if ui.configbool(b"http_proxy", b"always"):
142 self.no_list = [] 142 self.no_list = []
143 else: 143 else:
144 self.no_list = no_list 144 self.no_list = no_list
145 145
146 # Keys and values need to be str because the standard library 146 # Keys and values need to be str because the standard library
147 # expects them to be. 147 # expects them to be.
148 proxyurl = str(proxy) 148 proxyurl = str(proxy)
149 proxies = {r'http': proxyurl, r'https': proxyurl} 149 proxies = {r'http': proxyurl, r'https': proxyurl}
150 ui.debug('proxying through %s\n' % util.hidepassword(bytes(proxy))) 150 ui.debug(b'proxying through %s\n' % util.hidepassword(bytes(proxy)))
151 else: 151 else:
152 proxies = {} 152 proxies = {}
153 153
154 urlreq.proxyhandler.__init__(self, proxies) 154 urlreq.proxyhandler.__init__(self, proxies)
155 self.ui = ui 155 self.ui = ui
156 156
157 def proxy_open(self, req, proxy, type_): 157 def proxy_open(self, req, proxy, type_):
158 host = pycompat.bytesurl(urllibcompat.gethost(req)).split(':')[0] 158 host = pycompat.bytesurl(urllibcompat.gethost(req)).split(b':')[0]
159 for e in self.no_list: 159 for e in self.no_list:
160 if host == e: 160 if host == e:
161 return None 161 return None
162 if e.startswith('*.') and host.endswith(e[2:]): 162 if e.startswith(b'*.') and host.endswith(e[2:]):
163 return None 163 return None
164 if e.startswith('.') and host.endswith(e[1:]): 164 if e.startswith(b'.') and host.endswith(e[1:]):
165 return None 165 return None
166 166
167 return urlreq.proxyhandler.proxy_open(self, req, proxy, type_) 167 return urlreq.proxyhandler.proxy_open(self, req, proxy, type_)
168 168
169 169
179 orgsend(self, data) 179 orgsend(self, data)
180 180
181 return _sendfile 181 return _sendfile
182 182
183 183
184 has_https = util.safehasattr(urlreq, 'httpshandler') 184 has_https = util.safehasattr(urlreq, b'httpshandler')
185 185
186 186
187 class httpconnection(keepalive.HTTPConnection): 187 class httpconnection(keepalive.HTTPConnection):
188 # must be able to send big bundle as stream. 188 # must be able to send big bundle as stream.
189 send = _gen_sendfile(keepalive.HTTPConnection.send) 189 send = _gen_sendfile(keepalive.HTTPConnection.send)
210 tunnel_host = urllibcompat.getselector(req) 210 tunnel_host = urllibcompat.getselector(req)
211 new_tunnel = False 211 new_tunnel = False
212 212
213 if new_tunnel or tunnel_host == urllibcompat.getfullurl(req): # has proxy 213 if new_tunnel or tunnel_host == urllibcompat.getfullurl(req): # has proxy
214 u = util.url(pycompat.bytesurl(tunnel_host)) 214 u = util.url(pycompat.bytesurl(tunnel_host))
215 if new_tunnel or u.scheme == 'https': # only use CONNECT for HTTPS 215 if new_tunnel or u.scheme == b'https': # only use CONNECT for HTTPS
216 h.realhostport = ':'.join([u.host, (u.port or '443')]) 216 h.realhostport = b':'.join([u.host, (u.port or b'443')])
217 h.headers = req.headers.copy() 217 h.headers = req.headers.copy()
218 h.headers.update(handler.parent.addheaders) 218 h.headers.update(handler.parent.addheaders)
219 return 219 return
220 220
221 h.realhostport = None 221 h.realhostport = None
228 (x, self.headers[x]) 228 (x, self.headers[x])
229 for x in self.headers 229 for x in self.headers
230 if x.lower().startswith(r'proxy-') 230 if x.lower().startswith(r'proxy-')
231 ] 231 ]
232 ) 232 )
233 self.send('CONNECT %s HTTP/1.0\r\n' % self.realhostport) 233 self.send(b'CONNECT %s HTTP/1.0\r\n' % self.realhostport)
234 for header in proxyheaders.iteritems(): 234 for header in proxyheaders.iteritems():
235 self.send('%s: %s\r\n' % header) 235 self.send(b'%s: %s\r\n' % header)
236 self.send('\r\n') 236 self.send(b'\r\n')
237 237
238 # majority of the following code is duplicated from 238 # majority of the following code is duplicated from
239 # httplib.HTTPConnection as there are no adequate places to 239 # httplib.HTTPConnection as there are no adequate places to
240 # override functions to provide the needed functionality 240 # override functions to provide the needed functionality
241 # strict was removed in Python 3.4. 241 # strict was removed in Python 3.4.
242 kwargs = {} 242 kwargs = {}
243 if not pycompat.ispy3: 243 if not pycompat.ispy3:
244 kwargs['strict'] = self.strict 244 kwargs[b'strict'] = self.strict
245 245
246 res = self.response_class(self.sock, method=self._method, **kwargs) 246 res = self.response_class(self.sock, method=self._method, **kwargs)
247 247
248 while True: 248 while True:
249 version, status, reason = res._read_status() 249 version, status, reason = res._read_status()
250 if status != httplib.CONTINUE: 250 if status != httplib.CONTINUE:
251 break 251 break
252 # skip lines that are all whitespace 252 # skip lines that are all whitespace
253 list(iter(lambda: res.fp.readline().strip(), '')) 253 list(iter(lambda: res.fp.readline().strip(), b''))
254 res.status = status 254 res.status = status
255 res.reason = reason.strip() 255 res.reason = reason.strip()
256 256
257 if res.status == 200: 257 if res.status == 200:
258 # skip lines until we find a blank line 258 # skip lines until we find a blank line
259 list(iter(res.fp.readline, '\r\n')) 259 list(iter(res.fp.readline, b'\r\n'))
260 return True 260 return True
261 261
262 if version == 'HTTP/1.0': 262 if version == b'HTTP/1.0':
263 res.version = 10 263 res.version = 10
264 elif version.startswith('HTTP/1.'): 264 elif version.startswith(b'HTTP/1.'):
265 res.version = 11 265 res.version = 11
266 elif version == 'HTTP/0.9': 266 elif version == b'HTTP/0.9':
267 res.version = 9 267 res.version = 9
268 else: 268 else:
269 raise httplib.UnknownProtocol(version) 269 raise httplib.UnknownProtocol(version)
270 270
271 if res.version == 9: 271 if res.version == 9:
277 277
278 res.msg = httplib.HTTPMessage(res.fp) 278 res.msg = httplib.HTTPMessage(res.fp)
279 res.msg.fp = None 279 res.msg.fp = None
280 280
281 # are we using the chunked-style of transfer encoding? 281 # are we using the chunked-style of transfer encoding?
282 trenc = res.msg.getheader('transfer-encoding') 282 trenc = res.msg.getheader(b'transfer-encoding')
283 if trenc and trenc.lower() == "chunked": 283 if trenc and trenc.lower() == b"chunked":
284 res.chunked = 1 284 res.chunked = 1
285 res.chunk_left = None 285 res.chunk_left = None
286 else: 286 else:
287 res.chunked = 0 287 res.chunked = 0
288 288
290 res.will_close = res._check_close() 290 res.will_close = res._check_close()
291 291
292 # do we have a Content-Length? 292 # do we have a Content-Length?
293 # NOTE: RFC 2616, section 4.4, #3 says we ignore this if 293 # NOTE: RFC 2616, section 4.4, #3 says we ignore this if
294 # transfer-encoding is "chunked" 294 # transfer-encoding is "chunked"
295 length = res.msg.getheader('content-length') 295 length = res.msg.getheader(b'content-length')
296 if length and not res.chunked: 296 if length and not res.chunked:
297 try: 297 try:
298 res.length = int(length) 298 res.length = int(length)
299 except ValueError: 299 except ValueError:
300 res.length = None 300 res.length = None
307 # does the body have a fixed length? (of zero) 307 # does the body have a fixed length? (of zero)
308 if ( 308 if (
309 status == httplib.NO_CONTENT 309 status == httplib.NO_CONTENT
310 or status == httplib.NOT_MODIFIED 310 or status == httplib.NOT_MODIFIED
311 or 100 <= status < 200 311 or 100 <= status < 200
312 or res._method == 'HEAD' # 1xx codes 312 or res._method == b'HEAD' # 1xx codes
313 ): 313 ):
314 res.length = 0 314 res.length = 0
315 315
316 # if the connection remains open, and we aren't using chunked, and 316 # if the connection remains open, and we aren't using chunked, and
317 # a content-length was not provided, then assume that the connection 317 # a content-length was not provided, then assume that the connection
401 self.sock = socket.create_connection((self.host, self.port)) 401 self.sock = socket.create_connection((self.host, self.port))
402 402
403 host = self.host 403 host = self.host
404 if self.realhostport: # use CONNECT proxy 404 if self.realhostport: # use CONNECT proxy
405 _generic_proxytunnel(self) 405 _generic_proxytunnel(self)
406 host = self.realhostport.rsplit(':', 1)[0] 406 host = self.realhostport.rsplit(b':', 1)[0]
407 self.sock = sslutil.wrapsocket( 407 self.sock = sslutil.wrapsocket(
408 self.sock, 408 self.sock,
409 self.key_file, 409 self.key_file,
410 self.cert_file, 410 self.cert_file,
411 ui=self.ui, 411 ui=self.ui,
431 user, password = self.pwmgr.find_stored_password(url) 431 user, password = self.pwmgr.find_stored_password(url)
432 res = httpconnectionmod.readauthforuri(self.ui, url, user) 432 res = httpconnectionmod.readauthforuri(self.ui, url, user)
433 if res: 433 if res:
434 group, auth = res 434 group, auth = res
435 self.auth = auth 435 self.auth = auth
436 self.ui.debug("using auth.%s.* for authentication\n" % group) 436 self.ui.debug(b"using auth.%s.* for authentication\n" % group)
437 else: 437 else:
438 self.auth = None 438 self.auth = None
439 return self.do_open(self._makeconnection, req) 439 return self.do_open(self._makeconnection, req)
440 440
441 def _makeconnection(self, host, port=None, *args, **kwargs): 441 def _makeconnection(self, host, port=None, *args, **kwargs):
448 certfile = args[1] 448 certfile = args[1]
449 args = args[2:] 449 args = args[2:]
450 450
451 # if the user has specified different key/cert files in 451 # if the user has specified different key/cert files in
452 # hgrc, we prefer these 452 # hgrc, we prefer these
453 if self.auth and 'key' in self.auth and 'cert' in self.auth: 453 if self.auth and b'key' in self.auth and b'cert' in self.auth:
454 keyfile = self.auth['key'] 454 keyfile = self.auth[b'key']
455 certfile = self.auth['cert'] 455 certfile = self.auth[b'cert']
456 456
457 conn = httpsconnection( 457 conn = httpsconnection(
458 host, port, keyfile, certfile, *args, **kwargs 458 host, port, keyfile, certfile, *args, **kwargs
459 ) 459 )
460 conn.ui = self.ui 460 conn.ui = self.ui
518 def retry_http_basic_auth(self, host, req, realm): 518 def retry_http_basic_auth(self, host, req, realm):
519 user, pw = self.passwd.find_user_password( 519 user, pw = self.passwd.find_user_password(
520 realm, urllibcompat.getfullurl(req) 520 realm, urllibcompat.getfullurl(req)
521 ) 521 )
522 if pw is not None: 522 if pw is not None:
523 raw = "%s:%s" % (pycompat.bytesurl(user), pycompat.bytesurl(pw)) 523 raw = b"%s:%s" % (pycompat.bytesurl(user), pycompat.bytesurl(pw))
524 auth = r'Basic %s' % pycompat.strurl(base64.b64encode(raw).strip()) 524 auth = r'Basic %s' % pycompat.strurl(base64.b64encode(raw).strip())
525 if req.get_header(self.auth_header, None) == auth: 525 if req.get_header(self.auth_header, None) == auth:
526 return None 526 return None
527 self.auth = auth 527 self.auth = auth
528 req.add_unredirected_header(self.auth_header, auth) 528 req.add_unredirected_header(self.auth_header, auth)
533 533
534 class cookiehandler(urlreq.basehandler): 534 class cookiehandler(urlreq.basehandler):
535 def __init__(self, ui): 535 def __init__(self, ui):
536 self.cookiejar = None 536 self.cookiejar = None
537 537
538 cookiefile = ui.config('auth', 'cookiefile') 538 cookiefile = ui.config(b'auth', b'cookiefile')
539 if not cookiefile: 539 if not cookiefile:
540 return 540 return
541 541
542 cookiefile = util.expandpath(cookiefile) 542 cookiefile = util.expandpath(cookiefile)
543 try: 543 try:
547 cookiejar.load() 547 cookiejar.load()
548 self.cookiejar = cookiejar 548 self.cookiejar = cookiejar
549 except util.cookielib.LoadError as e: 549 except util.cookielib.LoadError as e:
550 ui.warn( 550 ui.warn(
551 _( 551 _(
552 '(error loading cookie file %s: %s; continuing without ' 552 b'(error loading cookie file %s: %s; continuing without '
553 'cookies)\n' 553 b'cookies)\n'
554 ) 554 )
555 % (cookiefile, stringutil.forcebytestr(e)) 555 % (cookiefile, stringutil.forcebytestr(e))
556 ) 556 )
557 557
558 def http_request(self, request): 558 def http_request(self, request):
593 ``util.socketobserver`` instance. 593 ``util.socketobserver`` instance.
594 594
595 ``sendaccept`` allows controlling whether the ``Accept`` request header 595 ``sendaccept`` allows controlling whether the ``Accept`` request header
596 is sent. The header is sent by default. 596 is sent. The header is sent by default.
597 ''' 597 '''
598 timeout = ui.configwith(float, 'http', 'timeout') 598 timeout = ui.configwith(float, b'http', b'timeout')
599 handlers = [] 599 handlers = []
600 600
601 if loggingfh: 601 if loggingfh:
602 handlers.append( 602 handlers.append(
603 logginghttphandler( 603 logginghttphandler(
619 realm, uris, user, passwd = authinfo 619 realm, uris, user, passwd = authinfo
620 saveduser, savedpass = passmgr.find_stored_password(uris[0]) 620 saveduser, savedpass = passmgr.find_stored_password(uris[0])
621 if user != saveduser or passwd: 621 if user != saveduser or passwd:
622 passmgr.add_password(realm, uris, user, passwd) 622 passmgr.add_password(realm, uris, user, passwd)
623 ui.debug( 623 ui.debug(
624 'http auth: user %s, password %s\n' 624 b'http auth: user %s, password %s\n'
625 % (user, passwd and '*' * len(passwd) or 'not set') 625 % (user, passwd and b'*' * len(passwd) or b'not set')
626 ) 626 )
627 627
628 handlers.extend( 628 handlers.extend(
629 (httpbasicauthhandler(passmgr), httpdigestauthhandler(passmgr)) 629 (httpbasicauthhandler(passmgr), httpdigestauthhandler(passmgr))
630 ) 630 )
651 # user agent they deem appropriate. 651 # user agent they deem appropriate.
652 # 652 #
653 # The custom user agent is for lfs, because unfortunately some servers 653 # The custom user agent is for lfs, because unfortunately some servers
654 # do look at this value. 654 # do look at this value.
655 if not useragent: 655 if not useragent:
656 agent = 'mercurial/proto-1.0 (Mercurial %s)' % util.version() 656 agent = b'mercurial/proto-1.0 (Mercurial %s)' % util.version()
657 opener.addheaders = [(r'User-agent', pycompat.sysstr(agent))] 657 opener.addheaders = [(r'User-agent', pycompat.sysstr(agent))]
658 else: 658 else:
659 opener.addheaders = [(r'User-agent', pycompat.sysstr(useragent))] 659 opener.addheaders = [(r'User-agent', pycompat.sysstr(useragent))]
660 660
661 # This header should only be needed by wire protocol requests. But it has 661 # This header should only be needed by wire protocol requests. But it has
673 if u.scheme: 673 if u.scheme:
674 u.scheme = u.scheme.lower() 674 u.scheme = u.scheme.lower()
675 url_, authinfo = u.authinfo() 675 url_, authinfo = u.authinfo()
676 else: 676 else:
677 path = util.normpath(os.path.abspath(url_)) 677 path = util.normpath(os.path.abspath(url_))
678 url_ = 'file://' + pycompat.bytesurl(urlreq.pathname2url(path)) 678 url_ = b'file://' + pycompat.bytesurl(urlreq.pathname2url(path))
679 authinfo = None 679 authinfo = None
680 return opener(ui, authinfo, sendaccept=sendaccept).open( 680 return opener(ui, authinfo, sendaccept=sendaccept).open(
681 pycompat.strurl(url_), data 681 pycompat.strurl(url_), data
682 ) 682 )
683 683
698 # e.expected is an integer if length known or None otherwise. 698 # e.expected is an integer if length known or None otherwise.
699 if e.expected: 699 if e.expected:
700 got = len(e.partial) 700 got = len(e.partial)
701 total = e.expected + got 701 total = e.expected + got
702 msg = _( 702 msg = _(
703 'HTTP request error (incomplete response; ' 703 b'HTTP request error (incomplete response; '
704 'expected %d bytes got %d)' 704 b'expected %d bytes got %d)'
705 ) % (total, got) 705 ) % (total, got)
706 else: 706 else:
707 msg = _('HTTP request error (incomplete response)') 707 msg = _(b'HTTP request error (incomplete response)')
708 708
709 raise error.PeerTransportError( 709 raise error.PeerTransportError(
710 msg, 710 msg,
711 hint=_( 711 hint=_(
712 'this may be an intermittent network failure; ' 712 b'this may be an intermittent network failure; '
713 'if the error persists, consider contacting the ' 713 b'if the error persists, consider contacting the '
714 'network or server operator' 714 b'network or server operator'
715 ), 715 ),
716 ) 716 )
717 except httplib.HTTPException as e: 717 except httplib.HTTPException as e:
718 raise error.PeerTransportError( 718 raise error.PeerTransportError(
719 _('HTTP request error (%s)') % e, 719 _(b'HTTP request error (%s)') % e,
720 hint=_( 720 hint=_(
721 'this may be an intermittent network failure; ' 721 b'this may be an intermittent network failure; '
722 'if the error persists, consider contacting the ' 722 b'if the error persists, consider contacting the '
723 'network or server operator' 723 b'network or server operator'
724 ), 724 ),
725 ) 725 )
726 726
727 resp.__class__ = readerproxy 727 resp.__class__ = readerproxy