Mercurial > public > mercurial-scm > hg

comparison mercurial/httppeer.py @ 43077:687b865b95ad

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
formatting: byteify all mercurial/ and hgext/ string literals Done with python3.7 contrib/byteify-strings.py -i $(hg files 'set:mercurial/**.py - mercurial/thirdparty/** + hgext/**.py - hgext/fsmonitor/pywatchman/** - mercurial/__init__.py') black -l 80 -t py33 -S $(hg files 'set:**.py - mercurial/thirdparty/** - "contrib/python-zstandard/**" - hgext/fsmonitor/pywatchman/**') # skip-blame mass-reformatting only Differential Revision: https://phab.mercurial-scm.org/D6972
author Augie Fackler <augie@google.com>
date Sun, 06 Oct 2019 09:48:39 -0400
parents 2372284d9457
children c59eb1560c44
comparison
equal deleted inserted replaced
43076:2372284d9457 43077:687b865b95ad
60 # Note: it is *NOT* a bug that the last bit here is a bytestring 60 # Note: it is *NOT* a bug that the last bit here is a bytestring
61 # and not a unicode: we're just getting the encoded length anyway, 61 # and not a unicode: we're just getting the encoded length anyway,
62 # and using an r-string to make it portable between Python 2 and 3 62 # and using an r-string to make it portable between Python 2 and 3
63 # doesn't work because then the \r is a literal backslash-r 63 # doesn't work because then the \r is a literal backslash-r
64 # instead of a carriage return. 64 # instead of a carriage return.
65 valuelen = limit - len(fmt % r'000') - len(': \r\n') 65 valuelen = limit - len(fmt % r'000') - len(b': \r\n')
66 result = [] 66 result = []
67 67
68 n = 0 68 n = 0
69 for i in pycompat.xrange(0, len(value), valuelen): 69 for i in pycompat.xrange(0, len(value), valuelen):
70 n += 1 70 n += 1
74 74
75 75
76 class _multifile(object): 76 class _multifile(object):
77 def __init__(self, *fileobjs): 77 def __init__(self, *fileobjs):
78 for f in fileobjs: 78 for f in fileobjs:
79 if not util.safehasattr(f, 'length'): 79 if not util.safehasattr(f, b'length'):
80 raise ValueError( 80 raise ValueError(
81 '_multifile only supports file objects that ' 81 b'_multifile only supports file objects that '
82 'have a length but this one does not:', 82 b'have a length but this one does not:',
83 type(f), 83 type(f),
84 f, 84 f,
85 ) 85 )
86 self._fileobjs = fileobjs 86 self._fileobjs = fileobjs
87 self._index = 0 87 self._index = 0
90 def length(self): 90 def length(self):
91 return sum(f.length for f in self._fileobjs) 91 return sum(f.length for f in self._fileobjs)
92 92
93 def read(self, amt=None): 93 def read(self, amt=None):
94 if amt <= 0: 94 if amt <= 0:
95 return ''.join(f.read() for f in self._fileobjs) 95 return b''.join(f.read() for f in self._fileobjs)
96 parts = [] 96 parts = []
97 while amt and self._index < len(self._fileobjs): 97 while amt and self._index < len(self._fileobjs):
98 parts.append(self._fileobjs[self._index].read(amt)) 98 parts.append(self._fileobjs[self._index].read(amt))
99 got = len(parts[-1]) 99 got = len(parts[-1])
100 if got < amt: 100 if got < amt:
101 self._index += 1 101 self._index += 1
102 amt -= got 102 amt -= got
103 return ''.join(parts) 103 return b''.join(parts)
104 104
105 def seek(self, offset, whence=os.SEEK_SET): 105 def seek(self, offset, whence=os.SEEK_SET):
106 if whence != os.SEEK_SET: 106 if whence != os.SEEK_SET:
107 raise NotImplementedError( 107 raise NotImplementedError(
108 '_multifile does not support anything other' 108 b'_multifile does not support anything other'
109 ' than os.SEEK_SET for whence on seek()' 109 b' than os.SEEK_SET for whence on seek()'
110 ) 110 )
111 if offset != 0: 111 if offset != 0:
112 raise NotImplementedError( 112 raise NotImplementedError(
113 '_multifile only supports seeking to start, but that ' 113 b'_multifile only supports seeking to start, but that '
114 'could be fixed if you need it' 114 b'could be fixed if you need it'
115 ) 115 )
116 for f in self._fileobjs: 116 for f in self._fileobjs:
117 f.seek(0) 117 f.seek(0)
118 self._index = 0 118 self._index = 0
119 119
129 ``capablefn`` is a function to evaluate a capability. 129 ``capablefn`` is a function to evaluate a capability.
130 130
131 ``cmd``, ``args``, and ``data`` define the command, its arguments, and 131 ``cmd``, ``args``, and ``data`` define the command, its arguments, and
132 raw data to pass to it. 132 raw data to pass to it.
133 """ 133 """
134 if cmd == 'pushkey': 134 if cmd == b'pushkey':
135 args['data'] = '' 135 args[b'data'] = b''
136 data = args.pop('data', None) 136 data = args.pop(b'data', None)
137 headers = args.pop('headers', {}) 137 headers = args.pop(b'headers', {})
138 138
139 ui.debug("sending %s command\n" % cmd) 139 ui.debug(b"sending %s command\n" % cmd)
140 q = [('cmd', cmd)] 140 q = [(b'cmd', cmd)]
141 headersize = 0 141 headersize = 0
142 # Important: don't use self.capable() here or else you end up 142 # Important: don't use self.capable() here or else you end up
143 # with infinite recursion when trying to look up capabilities 143 # with infinite recursion when trying to look up capabilities
144 # for the first time. 144 # for the first time.
145 postargsok = caps is not None and 'httppostargs' in caps 145 postargsok = caps is not None and b'httppostargs' in caps
146 146
147 # Send arguments via POST. 147 # Send arguments via POST.
148 if postargsok and args: 148 if postargsok and args:
149 strargs = urlreq.urlencode(sorted(args.items())) 149 strargs = urlreq.urlencode(sorted(args.items()))
150 if not data: 150 if not data:
160 headers[r'X-HgArgs-Post'] = len(strargs) 160 headers[r'X-HgArgs-Post'] = len(strargs)
161 elif args: 161 elif args:
162 # Calling self.capable() can infinite loop if we are calling 162 # Calling self.capable() can infinite loop if we are calling
163 # "capabilities". But that command should never accept wire 163 # "capabilities". But that command should never accept wire
164 # protocol arguments. So this should never happen. 164 # protocol arguments. So this should never happen.
165 assert cmd != 'capabilities' 165 assert cmd != b'capabilities'
166 httpheader = capablefn('httpheader') 166 httpheader = capablefn(b'httpheader')
167 if httpheader: 167 if httpheader:
168 headersize = int(httpheader.split(',', 1)[0]) 168 headersize = int(httpheader.split(b',', 1)[0])
169 169
170 # Send arguments via HTTP headers. 170 # Send arguments via HTTP headers.
171 if headersize > 0: 171 if headersize > 0:
172 # The headers can typically carry more data than the URL. 172 # The headers can typically carry more data than the URL.
173 encargs = urlreq.urlencode(sorted(args.items())) 173 encargs = urlreq.urlencode(sorted(args.items()))
174 for header, value in encodevalueinheaders( 174 for header, value in encodevalueinheaders(
175 encargs, 'X-HgArg', headersize 175 encargs, b'X-HgArg', headersize
176 ): 176 ):
177 headers[header] = value 177 headers[header] = value
178 # Send arguments via query string (Mercurial <1.9). 178 # Send arguments via query string (Mercurial <1.9).
179 else: 179 else:
180 q += sorted(args.items()) 180 q += sorted(args.items())
181 181
182 qs = '?%s' % urlreq.urlencode(q) 182 qs = b'?%s' % urlreq.urlencode(q)
183 cu = "%s%s" % (repobaseurl, qs) 183 cu = b"%s%s" % (repobaseurl, qs)
184 size = 0 184 size = 0
185 if util.safehasattr(data, 'length'): 185 if util.safehasattr(data, b'length'):
186 size = data.length 186 size = data.length
187 elif data is not None: 187 elif data is not None:
188 size = len(data) 188 size = len(data)
189 if data is not None and r'Content-Type' not in headers: 189 if data is not None and r'Content-Type' not in headers:
190 headers[r'Content-Type'] = r'application/mercurial-0.1' 190 headers[r'Content-Type'] = r'application/mercurial-0.1'
196 # protocol parameters should only occur after the handshake. 196 # protocol parameters should only occur after the handshake.
197 protoparams = set() 197 protoparams = set()
198 198
199 mediatypes = set() 199 mediatypes = set()
200 if caps is not None: 200 if caps is not None:
201 mt = capablefn('httpmediatype') 201 mt = capablefn(b'httpmediatype')
202 if mt: 202 if mt:
203 protoparams.add('0.1') 203 protoparams.add(b'0.1')
204 mediatypes = set(mt.split(',')) 204 mediatypes = set(mt.split(b','))
205 205
206 protoparams.add('partial-pull') 206 protoparams.add(b'partial-pull')
207 207
208 if '0.2tx' in mediatypes: 208 if b'0.2tx' in mediatypes:
209 protoparams.add('0.2') 209 protoparams.add(b'0.2')
210 210
211 if '0.2tx' in mediatypes and capablefn('compression'): 211 if b'0.2tx' in mediatypes and capablefn(b'compression'):
212 # We /could/ compare supported compression formats and prune 212 # We /could/ compare supported compression formats and prune
213 # non-mutually supported or error if nothing is mutually supported. 213 # non-mutually supported or error if nothing is mutually supported.
214 # For now, send the full list to the server and have it error. 214 # For now, send the full list to the server and have it error.
215 comps = [ 215 comps = [
216 e.wireprotosupport().name 216 e.wireprotosupport().name
217 for e in util.compengines.supportedwireengines(util.CLIENTROLE) 217 for e in util.compengines.supportedwireengines(util.CLIENTROLE)
218 ] 218 ]
219 protoparams.add('comp=%s' % ','.join(comps)) 219 protoparams.add(b'comp=%s' % b','.join(comps))
220 220
221 if protoparams: 221 if protoparams:
222 protoheaders = encodevalueinheaders( 222 protoheaders = encodevalueinheaders(
223 ' '.join(sorted(protoparams)), 'X-HgProto', headersize or 1024 223 b' '.join(sorted(protoparams)), b'X-HgProto', headersize or 1024
224 ) 224 )
225 for header, value in protoheaders: 225 for header, value in protoheaders:
226 headers[header] = value 226 headers[header] = value
227 227
228 varyheaders = [] 228 varyheaders = []
234 headers[r'Vary'] = r','.join(sorted(varyheaders)) 234 headers[r'Vary'] = r','.join(sorted(varyheaders))
235 235
236 req = requestbuilder(pycompat.strurl(cu), data, headers) 236 req = requestbuilder(pycompat.strurl(cu), data, headers)
237 237
238 if data is not None: 238 if data is not None:
239 ui.debug("sending %d bytes\n" % size) 239 ui.debug(b"sending %d bytes\n" % size)
240 req.add_unredirected_header(r'Content-Length', r'%d' % size) 240 req.add_unredirected_header(r'Content-Length', r'%d' % size)
241 241
242 return req, cu, qs 242 return req, cu, qs
243 243
244 244
255 """Send a prepared HTTP request. 255 """Send a prepared HTTP request.
256 256
257 Returns the response object. 257 Returns the response object.
258 """ 258 """
259 dbg = ui.debug 259 dbg = ui.debug
260 if ui.debugflag and ui.configbool('devel', 'debug.peer-request'): 260 if ui.debugflag and ui.configbool(b'devel', b'debug.peer-request'):
261 line = 'devel-peer-request: %s\n' 261 line = b'devel-peer-request: %s\n'
262 dbg( 262 dbg(
263 line 263 line
264 % '%s %s' 264 % b'%s %s'
265 % ( 265 % (
266 pycompat.bytesurl(req.get_method()), 266 pycompat.bytesurl(req.get_method()),
267 pycompat.bytesurl(req.get_full_url()), 267 pycompat.bytesurl(req.get_full_url()),
268 ) 268 )
269 ) 269 )
270 hgargssize = None 270 hgargssize = None
271 271
272 for header, value in sorted(req.header_items()): 272 for header, value in sorted(req.header_items()):
273 header = pycompat.bytesurl(header) 273 header = pycompat.bytesurl(header)
274 value = pycompat.bytesurl(value) 274 value = pycompat.bytesurl(value)
275 if header.startswith('X-hgarg-'): 275 if header.startswith(b'X-hgarg-'):
276 if hgargssize is None: 276 if hgargssize is None:
277 hgargssize = 0 277 hgargssize = 0
278 hgargssize += len(value) 278 hgargssize += len(value)
279 else: 279 else:
280 dbg(line % ' %s %s' % (header, value)) 280 dbg(line % b' %s %s' % (header, value))
281 281
282 if hgargssize is not None: 282 if hgargssize is not None:
283 dbg( 283 dbg(
284 line 284 line
285 % ' %d bytes of commands arguments in headers' 285 % b' %d bytes of commands arguments in headers'
286 % hgargssize 286 % hgargssize
287 ) 287 )
288 data = _reqdata(req) 288 data = _reqdata(req)
289 if data is not None: 289 if data is not None:
290 length = getattr(data, 'length', None) 290 length = getattr(data, 'length', None)
291 if length is None: 291 if length is None:
292 length = len(data) 292 length = len(data)
293 dbg(line % ' %d bytes of data' % length) 293 dbg(line % b' %d bytes of data' % length)
294 294
295 start = util.timer() 295 start = util.timer()
296 296
297 res = None 297 res = None
298 try: 298 try:
299 res = opener.open(req) 299 res = opener.open(req)
300 except urlerr.httperror as inst: 300 except urlerr.httperror as inst:
301 if inst.code == 401: 301 if inst.code == 401:
302 raise error.Abort(_('authorization failed')) 302 raise error.Abort(_(b'authorization failed'))
303 raise 303 raise
304 except httplib.HTTPException as inst: 304 except httplib.HTTPException as inst:
305 ui.debug( 305 ui.debug(
306 'http error requesting %s\n' % util.hidepassword(req.get_full_url()) 306 b'http error requesting %s\n'
307 % util.hidepassword(req.get_full_url())
307 ) 308 )
308 ui.traceback() 309 ui.traceback()
309 raise IOError(None, inst) 310 raise IOError(None, inst)
310 finally: 311 finally:
311 if ui.debugflag and ui.configbool('devel', 'debug.peer-request'): 312 if ui.debugflag and ui.configbool(b'devel', b'debug.peer-request'):
312 code = res.code if res else -1 313 code = res.code if res else -1
313 dbg( 314 dbg(
314 line 315 line
315 % ' finished in %.4f seconds (%d)' 316 % b' finished in %.4f seconds (%d)'
316 % (util.timer() - start, code) 317 % (util.timer() - start, code)
317 ) 318 )
318 319
319 # Insert error handlers for common I/O failures. 320 # Insert error handlers for common I/O failures.
320 urlmod.wrapresponse(res) 321 urlmod.wrapresponse(res)
338 respurl = respurl[: -len(qs)] 339 respurl = respurl[: -len(qs)]
339 qsdropped = False 340 qsdropped = False
340 else: 341 else:
341 qsdropped = True 342 qsdropped = True
342 343
343 if baseurl.rstrip('/') != respurl.rstrip('/'): 344 if baseurl.rstrip(b'/') != respurl.rstrip(b'/'):
344 redirected = True 345 redirected = True
345 if not ui.quiet: 346 if not ui.quiet:
346 ui.warn(_('real URL is %s\n') % respurl) 347 ui.warn(_(b'real URL is %s\n') % respurl)
347 348
348 try: 349 try:
349 proto = pycompat.bytesurl(resp.getheader(r'content-type', r'')) 350 proto = pycompat.bytesurl(resp.getheader(r'content-type', r''))
350 except AttributeError: 351 except AttributeError:
351 proto = pycompat.bytesurl(resp.headers.get(r'content-type', r'')) 352 proto = pycompat.bytesurl(resp.headers.get(r'content-type', r''))
352 353
353 safeurl = util.hidepassword(baseurl) 354 safeurl = util.hidepassword(baseurl)
354 if proto.startswith('application/hg-error'): 355 if proto.startswith(b'application/hg-error'):
355 raise error.OutOfBandError(resp.read()) 356 raise error.OutOfBandError(resp.read())
356 357
357 # Pre 1.0 versions of Mercurial used text/plain and 358 # Pre 1.0 versions of Mercurial used text/plain and
358 # application/hg-changegroup. We don't support such old servers. 359 # application/hg-changegroup. We don't support such old servers.
359 if not proto.startswith('application/mercurial-'): 360 if not proto.startswith(b'application/mercurial-'):
360 ui.debug("requested URL: '%s'\n" % util.hidepassword(requrl)) 361 ui.debug(b"requested URL: '%s'\n" % util.hidepassword(requrl))
361 msg = _( 362 msg = _(
362 "'%s' does not appear to be an hg repository:\n" 363 b"'%s' does not appear to be an hg repository:\n"
363 "---%%<--- (%s)\n%s\n---%%<---\n" 364 b"---%%<--- (%s)\n%s\n---%%<---\n"
364 ) % (safeurl, proto or 'no content-type', resp.read(1024)) 365 ) % (safeurl, proto or b'no content-type', resp.read(1024))
365 366
366 # Some servers may strip the query string from the redirect. We 367 # Some servers may strip the query string from the redirect. We
367 # raise a special error type so callers can react to this specially. 368 # raise a special error type so callers can react to this specially.
368 if redirected and qsdropped: 369 if redirected and qsdropped:
369 raise RedirectedRepoError(msg, respurl) 370 raise RedirectedRepoError(msg, respurl)
370 else: 371 else:
371 raise error.RepoError(msg) 372 raise error.RepoError(msg)
372 373
373 try: 374 try:
374 subtype = proto.split('-', 1)[1] 375 subtype = proto.split(b'-', 1)[1]
375 376
376 # Unless we end up supporting CBOR in the legacy wire protocol, 377 # Unless we end up supporting CBOR in the legacy wire protocol,
377 # this should ONLY be encountered for the initial capabilities 378 # this should ONLY be encountered for the initial capabilities
378 # request during handshake. 379 # request during handshake.
379 if subtype == 'cbor': 380 if subtype == b'cbor':
380 if allowcbor: 381 if allowcbor:
381 return respurl, proto, resp 382 return respurl, proto, resp
382 else: 383 else:
383 raise error.RepoError( 384 raise error.RepoError(
384 _('unexpected CBOR response from ' 'server') 385 _(b'unexpected CBOR response from ' b'server')
385 ) 386 )
386 387
387 version_info = tuple([int(n) for n in subtype.split('.')]) 388 version_info = tuple([int(n) for n in subtype.split(b'.')])
388 except ValueError: 389 except ValueError:
389 raise error.RepoError( 390 raise error.RepoError(
390 _("'%s' sent a broken Content-Type " "header (%s)") 391 _(b"'%s' sent a broken Content-Type " b"header (%s)")
391 % (safeurl, proto) 392 % (safeurl, proto)
392 ) 393 )
393 394
394 # TODO consider switching to a decompression reader that uses 395 # TODO consider switching to a decompression reader that uses
395 # generators. 396 # generators.
396 if version_info == (0, 1): 397 if version_info == (0, 1):
397 if compressible: 398 if compressible:
398 resp = util.compengines['zlib'].decompressorreader(resp) 399 resp = util.compengines[b'zlib'].decompressorreader(resp)
399 400
400 elif version_info == (0, 2): 401 elif version_info == (0, 2):
401 # application/mercurial-0.2 always identifies the compression 402 # application/mercurial-0.2 always identifies the compression
402 # engine in the payload header. 403 # engine in the payload header.
403 elen = struct.unpack('B', util.readexactly(resp, 1))[0] 404 elen = struct.unpack(b'B', util.readexactly(resp, 1))[0]
404 ename = util.readexactly(resp, elen) 405 ename = util.readexactly(resp, elen)
405 engine = util.compengines.forwiretype(ename) 406 engine = util.compengines.forwiretype(ename)
406 407
407 resp = engine.decompressorreader(resp) 408 resp = engine.decompressorreader(resp)
408 else: 409 else:
409 raise error.RepoError( 410 raise error.RepoError(
410 _("'%s' uses newer protocol %s") % (safeurl, subtype) 411 _(b"'%s' uses newer protocol %s") % (safeurl, subtype)
411 ) 412 )
412 413
413 return respurl, proto, resp 414 return respurl, proto, resp
414 415
415 416
417 def __init__(self, ui, path, url, opener, requestbuilder, caps): 418 def __init__(self, ui, path, url, opener, requestbuilder, caps):
418 self.ui = ui 419 self.ui = ui
419 self._path = path 420 self._path = path
420 self._url = url 421 self._url = url
421 self._caps = caps 422 self._caps = caps
422 self.limitedarguments = caps is not None and 'httppostargs' not in caps 423 self.limitedarguments = caps is not None and b'httppostargs' not in caps
423 self._urlopener = opener 424 self._urlopener = opener
424 self._requestbuilder = requestbuilder 425 self._requestbuilder = requestbuilder
425 426
426 def __del__(self): 427 def __del__(self):
427 for h in self._urlopener.handlers: 428 for h in self._urlopener.handlers:
451 ) 452 )
452 except AttributeError: 453 except AttributeError:
453 return 454 return
454 self.ui.note( 455 self.ui.note(
455 _( 456 _(
456 '(sent %d HTTP requests and %d bytes; ' 457 b'(sent %d HTTP requests and %d bytes; '
457 'received %d bytes in responses)\n' 458 b'received %d bytes in responses)\n'
458 ) 459 )
459 % (reqs, sent, recv) 460 % (reqs, sent, recv)
460 ) 461 )
461 462
462 # End of ipeerconnection interface. 463 # End of ipeerconnection interface.
499 500
500 def _callpush(self, cmd, cg, **args): 501 def _callpush(self, cmd, cg, **args):
501 # have to stream bundle to a temp file because we do not have 502 # have to stream bundle to a temp file because we do not have
502 # http 1.1 chunked transfer. 503 # http 1.1 chunked transfer.
503 504
504 types = self.capable('unbundle') 505 types = self.capable(b'unbundle')
505 try: 506 try:
506 types = types.split(',') 507 types = types.split(b',')
507 except AttributeError: 508 except AttributeError:
508 # servers older than d1b16a746db6 will send 'unbundle' as a 509 # servers older than d1b16a746db6 will send 'unbundle' as a
509 # boolean capability. They only support headerless/uncompressed 510 # boolean capability. They only support headerless/uncompressed
510 # bundles. 511 # bundles.
511 types = [""] 512 types = [b""]
512 for x in types: 513 for x in types:
513 if x in bundle2.bundletypes: 514 if x in bundle2.bundletypes:
514 type = x 515 type = x
515 break 516 break
516 517
517 tempname = bundle2.writebundle(self.ui, cg, None, type) 518 tempname = bundle2.writebundle(self.ui, cg, None, type)
518 fp = httpconnection.httpsendfile(self.ui, tempname, "rb") 519 fp = httpconnection.httpsendfile(self.ui, tempname, b"rb")
519 headers = {r'Content-Type': r'application/mercurial-0.1'} 520 headers = {r'Content-Type': r'application/mercurial-0.1'}
520 521
521 try: 522 try:
522 r = self._call(cmd, data=fp, headers=headers, **args) 523 r = self._call(cmd, data=fp, headers=headers, **args)
523 vals = r.split('\n', 1) 524 vals = r.split(b'\n', 1)
524 if len(vals) < 2: 525 if len(vals) < 2:
525 raise error.ResponseError(_("unexpected response:"), r) 526 raise error.ResponseError(_(b"unexpected response:"), r)
526 return vals 527 return vals
527 except urlerr.httperror: 528 except urlerr.httperror:
528 # Catch and re-raise these so we don't try and treat them 529 # Catch and re-raise these so we don't try and treat them
529 # like generic socket errors. They lack any values in 530 # like generic socket errors. They lack any values in
530 # .args on Python 3 which breaks our socket.error block. 531 # .args on Python 3 which breaks our socket.error block.
531 raise 532 raise
532 except socket.error as err: 533 except socket.error as err:
533 if err.args[0] in (errno.ECONNRESET, errno.EPIPE): 534 if err.args[0] in (errno.ECONNRESET, errno.EPIPE):
534 raise error.Abort(_('push failed: %s') % err.args[1]) 535 raise error.Abort(_(b'push failed: %s') % err.args[1])
535 raise error.Abort(err.args[1]) 536 raise error.Abort(err.args[1])
536 finally: 537 finally:
537 fp.close() 538 fp.close()
538 os.unlink(tempname) 539 os.unlink(tempname)
539 540
540 def _calltwowaystream(self, cmd, fp, **args): 541 def _calltwowaystream(self, cmd, fp, **args):
541 filename = None 542 filename = None
542 try: 543 try:
543 # dump bundle to disk 544 # dump bundle to disk
544 fd, filename = pycompat.mkstemp(prefix="hg-bundle-", suffix=".hg") 545 fd, filename = pycompat.mkstemp(prefix=b"hg-bundle-", suffix=b".hg")
545 with os.fdopen(fd, r"wb") as fh: 546 with os.fdopen(fd, r"wb") as fh:
546 d = fp.read(4096) 547 d = fp.read(4096)
547 while d: 548 while d:
548 fh.write(d) 549 fh.write(d)
549 d = fp.read(4096) 550 d = fp.read(4096)
550 # start http push 551 # start http push
551 with httpconnection.httpsendfile(self.ui, filename, "rb") as fp_: 552 with httpconnection.httpsendfile(self.ui, filename, b"rb") as fp_:
552 headers = {r'Content-Type': r'application/mercurial-0.1'} 553 headers = {r'Content-Type': r'application/mercurial-0.1'}
553 return self._callstream(cmd, data=fp_, headers=headers, **args) 554 return self._callstream(cmd, data=fp_, headers=headers, **args)
554 finally: 555 finally:
555 if filename is not None: 556 if filename is not None:
556 os.unlink(filename) 557 os.unlink(filename)
596 597
597 handler = wireprotov2peer.clienthandler( 598 handler = wireprotov2peer.clienthandler(
598 ui, reactor, opener=opener, requestbuilder=requestbuilder 599 ui, reactor, opener=opener, requestbuilder=requestbuilder
599 ) 600 )
600 601
601 url = '%s/%s' % (apiurl, permission) 602 url = b'%s/%s' % (apiurl, permission)
602 603
603 if len(requests) > 1: 604 if len(requests) > 1:
604 url += '/multirequest' 605 url += b'/multirequest'
605 else: 606 else:
606 url += '/%s' % requests[0][0] 607 url += b'/%s' % requests[0][0]
607 608
608 ui.debug('sending %d commands\n' % len(requests)) 609 ui.debug(b'sending %d commands\n' % len(requests))
609 for command, args, f in requests: 610 for command, args, f in requests:
610 ui.debug( 611 ui.debug(
611 'sending command %s: %s\n' 612 b'sending command %s: %s\n'
612 % (command, stringutil.pprint(args, indent=2)) 613 % (command, stringutil.pprint(args, indent=2))
613 ) 614 )
614 assert not list( 615 assert not list(
615 handler.callcommand(command, args, f, redirect=redirect) 616 handler.callcommand(command, args, f, redirect=redirect)
616 ) 617 )
629 630
630 try: 631 try:
631 res = opener.open(req) 632 res = opener.open(req)
632 except urlerr.httperror as e: 633 except urlerr.httperror as e:
633 if e.code == 401: 634 if e.code == 401:
634 raise error.Abort(_('authorization failed')) 635 raise error.Abort(_(b'authorization failed'))
635 636
636 raise 637 raise
637 except httplib.HTTPException as e: 638 except httplib.HTTPException as e:
638 ui.traceback() 639 ui.traceback()
639 raise IOError(None, e) 640 raise IOError(None, e)
681 self.close() 682 self.close()
682 683
683 def callcommand(self, command, args): 684 def callcommand(self, command, args):
684 if self._sent: 685 if self._sent:
685 raise error.ProgrammingError( 686 raise error.ProgrammingError(
686 'callcommand() cannot be used after ' 'commands are sent' 687 b'callcommand() cannot be used after ' b'commands are sent'
687 ) 688 )
688 689
689 if self._closed: 690 if self._closed:
690 raise error.ProgrammingError( 691 raise error.ProgrammingError(
691 'callcommand() cannot be used after ' 'close()' 692 b'callcommand() cannot be used after ' b'close()'
692 ) 693 )
693 694
694 # The service advertises which commands are available. So if we attempt 695 # The service advertises which commands are available. So if we attempt
695 # to call an unknown command or pass an unknown argument, we can screen 696 # to call an unknown command or pass an unknown argument, we can screen
696 # for this. 697 # for this.
697 if command not in self._descriptor['commands']: 698 if command not in self._descriptor[b'commands']:
698 raise error.ProgrammingError( 699 raise error.ProgrammingError(
699 'wire protocol command %s is not available' % command 700 b'wire protocol command %s is not available' % command
700 ) 701 )
701 702
702 cmdinfo = self._descriptor['commands'][command] 703 cmdinfo = self._descriptor[b'commands'][command]
703 unknownargs = set(args.keys()) - set(cmdinfo.get('args', {})) 704 unknownargs = set(args.keys()) - set(cmdinfo.get(b'args', {}))
704 705
705 if unknownargs: 706 if unknownargs:
706 raise error.ProgrammingError( 707 raise error.ProgrammingError(
707 'wire protocol command %s does not accept argument: %s' 708 b'wire protocol command %s does not accept argument: %s'
708 % (command, ', '.join(sorted(unknownargs))) 709 % (command, b', '.join(sorted(unknownargs)))
709 ) 710 )
710 711
711 self._neededpermissions |= set(cmdinfo['permissions']) 712 self._neededpermissions |= set(cmdinfo[b'permissions'])
712 713
713 # TODO we /could/ also validate types here, since the API descriptor 714 # TODO we /could/ also validate types here, since the API descriptor
714 # includes types... 715 # includes types...
715 716
716 f = pycompat.futures.Future() 717 f = pycompat.futures.Future()
754 if not calls: 755 if not calls:
755 return 756 return
756 757
757 permissions = set(self._neededpermissions) 758 permissions = set(self._neededpermissions)
758 759
759 if 'push' in permissions and 'pull' in permissions: 760 if b'push' in permissions and b'pull' in permissions:
760 permissions.remove('pull') 761 permissions.remove(b'pull')
761 762
762 if len(permissions) > 1: 763 if len(permissions) > 1:
763 raise error.RepoError( 764 raise error.RepoError(
764 _('cannot make request requiring multiple ' 'permissions: %s') 765 _(b'cannot make request requiring multiple ' b'permissions: %s')
765 % _(', ').join(sorted(permissions)) 766 % _(b', ').join(sorted(permissions))
766 ) 767 )
767 768
768 permission = {'push': 'rw', 'pull': 'ro',}[permissions.pop()] 769 permission = {b'push': b'rw', b'pull': b'ro',}[permissions.pop()]
769 770
770 handler, resp = sendv2request( 771 handler, resp = sendv2request(
771 self._ui, 772 self._ui,
772 self._opener, 773 self._opener,
773 self._requestbuilder, 774 self._requestbuilder,
807 # If any of our futures are still in progress, mark them as 808 # If any of our futures are still in progress, mark them as
808 # errored, otherwise a result() could wait indefinitely. 809 # errored, otherwise a result() could wait indefinitely.
809 for f in self._futures: 810 for f in self._futures:
810 if not f.done(): 811 if not f.done():
811 f.set_exception( 812 f.set_exception(
812 error.ResponseError(_('unfulfilled command response')) 813 error.ResponseError(_(b'unfulfilled command response'))
813 ) 814 )
814 815
815 self._futures = None 816 self._futures = None
816 817
817 def _handleresponse(self, handler, resp): 818 def _handleresponse(self, handler, resp):
830 self, ui, repourl, apipath, opener, requestbuilder, apidescriptor 831 self, ui, repourl, apipath, opener, requestbuilder, apidescriptor
831 ): 832 ):
832 self.ui = ui 833 self.ui = ui
833 self.apidescriptor = apidescriptor 834 self.apidescriptor = apidescriptor
834 835
835 if repourl.endswith('/'): 836 if repourl.endswith(b'/'):
836 repourl = repourl[:-1] 837 repourl = repourl[:-1]
837 838
838 self._url = repourl 839 self._url = repourl
839 self._apipath = apipath 840 self._apipath = apipath
840 self._apiurl = '%s/%s' % (repourl, apipath) 841 self._apiurl = b'%s/%s' % (repourl, apipath)
841 self._opener = opener 842 self._opener = opener
842 self._requestbuilder = requestbuilder 843 self._requestbuilder = requestbuilder
843 844
844 self._redirect = wireprotov2peer.supportedredirects(ui, apidescriptor) 845 self._redirect = wireprotov2peer.supportedredirects(ui, apidescriptor)
845 846
859 return False 860 return False
860 861
861 def close(self): 862 def close(self):
862 self.ui.note( 863 self.ui.note(
863 _( 864 _(
864 '(sent %d HTTP requests and %d bytes; ' 865 b'(sent %d HTTP requests and %d bytes; '
865 'received %d bytes in responses)\n' 866 b'received %d bytes in responses)\n'
866 ) 867 )
867 % ( 868 % (
868 self._opener.requestscount, 869 self._opener.requestscount,
869 self._opener.sentbytescount, 870 self._opener.sentbytescount,
870 self._opener.receivedbytescount, 871 self._opener.receivedbytescount,
879 # The capabilities used internally historically map to capabilities 880 # The capabilities used internally historically map to capabilities
880 # advertised from the "capabilities" wire protocol command. However, 881 # advertised from the "capabilities" wire protocol command. However,
881 # version 2 of that command works differently. 882 # version 2 of that command works differently.
882 883
883 # Maps to commands that are available. 884 # Maps to commands that are available.
884 if name in ('branchmap', 'getbundle', 'known', 'lookup', 'pushkey'): 885 if name in (
886 b'branchmap',
887 b'getbundle',
888 b'known',
889 b'lookup',
890 b'pushkey',
891 ):
885 return True 892 return True
886 893
887 # Other concepts. 894 # Other concepts.
888 if name in 'bundle2': 895 if name in b'bundle2':
889 return True 896 return True
890 897
891 # Alias command-* to presence of command of that name. 898 # Alias command-* to presence of command of that name.
892 if name.startswith('command-'): 899 if name.startswith(b'command-'):
893 return name[len('command-') :] in self.apidescriptor['commands'] 900 return name[len(b'command-') :] in self.apidescriptor[b'commands']
894 901
895 return False 902 return False
896 903
897 def requirecap(self, name, purpose): 904 def requirecap(self, name, purpose):
898 if self.capable(name): 905 if self.capable(name):
899 return 906 return
900 907
901 raise error.CapabilityError( 908 raise error.CapabilityError(
902 _( 909 _(
903 'cannot %s; client or remote repository does not support the ' 910 b'cannot %s; client or remote repository does not support the '
904 '\'%s\' capability' 911 b'\'%s\' capability'
905 ) 912 )
906 % (purpose, name) 913 % (purpose, name)
907 ) 914 )
908 915
909 # End of ipeercapabilities. 916 # End of ipeercapabilities.
933 # 940 #
934 # priority 941 # priority
935 # Integer priority for the service. If we could choose from multiple 942 # Integer priority for the service. If we could choose from multiple
936 # services, we choose the one with the highest priority. 943 # services, we choose the one with the highest priority.
937 API_PEERS = { 944 API_PEERS = {
938 wireprototypes.HTTP_WIREPROTO_V2: {'init': httpv2peer, 'priority': 50,}, 945 wireprototypes.HTTP_WIREPROTO_V2: {b'init': httpv2peer, b'priority': 50,},
939 } 946 }
940 947
941 948
942 def performhandshake(ui, url, opener, requestbuilder): 949 def performhandshake(ui, url, opener, requestbuilder):
943 # The handshake is a request to the capabilities command. 950 # The handshake is a request to the capabilities command.
944 951
945 caps = None 952 caps = None
946 953
947 def capable(x): 954 def capable(x):
948 raise error.ProgrammingError('should not be called') 955 raise error.ProgrammingError(b'should not be called')
949 956
950 args = {} 957 args = {}
951 958
952 # The client advertises support for newer protocols by adding an 959 # The client advertises support for newer protocols by adding an
953 # X-HgUpgrade-* header with a list of supported APIs and an 960 # X-HgUpgrade-* header with a list of supported APIs and an
954 # X-HgProto-* header advertising which serializing formats it supports. 961 # X-HgProto-* header advertising which serializing formats it supports.
955 # We only support the HTTP version 2 transport and CBOR responses for 962 # We only support the HTTP version 2 transport and CBOR responses for
956 # now. 963 # now.
957 advertisev2 = ui.configbool('experimental', 'httppeer.advertise-v2') 964 advertisev2 = ui.configbool(b'experimental', b'httppeer.advertise-v2')
958 965
959 if advertisev2: 966 if advertisev2:
960 args['headers'] = { 967 args[b'headers'] = {
961 r'X-HgProto-1': r'cbor', 968 r'X-HgProto-1': r'cbor',
962 } 969 }
963 970
964 args['headers'].update( 971 args[b'headers'].update(
965 encodevalueinheaders( 972 encodevalueinheaders(
966 ' '.join(sorted(API_PEERS)), 973 b' '.join(sorted(API_PEERS)),
967 'X-HgUpgrade', 974 b'X-HgUpgrade',
968 # We don't know the header limit this early. 975 # We don't know the header limit this early.
969 # So make it small. 976 # So make it small.
970 1024, 977 1024,
971 ) 978 )
972 ) 979 )
973 980
974 req, requrl, qs = makev1commandrequest( 981 req, requrl, qs = makev1commandrequest(
975 ui, requestbuilder, caps, capable, url, 'capabilities', args 982 ui, requestbuilder, caps, capable, url, b'capabilities', args
976 ) 983 )
977 resp = sendrequest(ui, opener, req) 984 resp = sendrequest(ui, opener, req)
978 985
979 # The server may redirect us to the repo root, stripping the 986 # The server may redirect us to the repo root, stripping the
980 # ?cmd=capabilities query string from the URL. The server would likely 987 # ?cmd=capabilities query string from the URL. The server would likely
992 respurl, ct, resp = parsev1commandresponse( 999 respurl, ct, resp = parsev1commandresponse(
993 ui, url, requrl, qs, resp, compressible=False, allowcbor=advertisev2 1000 ui, url, requrl, qs, resp, compressible=False, allowcbor=advertisev2
994 ) 1001 )
995 except RedirectedRepoError as e: 1002 except RedirectedRepoError as e:
996 req, requrl, qs = makev1commandrequest( 1003 req, requrl, qs = makev1commandrequest(
997 ui, requestbuilder, caps, capable, e.respurl, 'capabilities', args 1004 ui, requestbuilder, caps, capable, e.respurl, b'capabilities', args
998 ) 1005 )
999 resp = sendrequest(ui, opener, req) 1006 resp = sendrequest(ui, opener, req)
1000 respurl, ct, resp = parsev1commandresponse( 1007 respurl, ct, resp = parsev1commandresponse(
1001 ui, url, requrl, qs, resp, compressible=False, allowcbor=advertisev2 1008 ui, url, requrl, qs, resp, compressible=False, allowcbor=advertisev2
1002 ) 1009 )
1004 try: 1011 try:
1005 rawdata = resp.read() 1012 rawdata = resp.read()
1006 finally: 1013 finally:
1007 resp.close() 1014 resp.close()
1008 1015
1009 if not ct.startswith('application/mercurial-'): 1016 if not ct.startswith(b'application/mercurial-'):
1010 raise error.ProgrammingError('unexpected content-type: %s' % ct) 1017 raise error.ProgrammingError(b'unexpected content-type: %s' % ct)
1011 1018
1012 if advertisev2: 1019 if advertisev2:
1013 if ct == 'application/mercurial-cbor': 1020 if ct == b'application/mercurial-cbor':
1014 try: 1021 try:
1015 info = cborutil.decodeall(rawdata)[0] 1022 info = cborutil.decodeall(rawdata)[0]
1016 except cborutil.CBORDecodeError: 1023 except cborutil.CBORDecodeError:
1017 raise error.Abort( 1024 raise error.Abort(
1018 _('error decoding CBOR from remote server'), 1025 _(b'error decoding CBOR from remote server'),
1019 hint=_( 1026 hint=_(
1020 'try again and consider contacting ' 1027 b'try again and consider contacting '
1021 'the server operator' 1028 b'the server operator'
1022 ), 1029 ),
1023 ) 1030 )
1024 1031
1025 # We got a legacy response. That's fine. 1032 # We got a legacy response. That's fine.
1026 elif ct in ('application/mercurial-0.1', 'application/mercurial-0.2'): 1033 elif ct in (b'application/mercurial-0.1', b'application/mercurial-0.2'):
1027 info = {'v1capabilities': set(rawdata.split())} 1034 info = {b'v1capabilities': set(rawdata.split())}
1028 1035
1029 else: 1036 else:
1030 raise error.RepoError( 1037 raise error.RepoError(
1031 _('unexpected response type from server: %s') % ct 1038 _(b'unexpected response type from server: %s') % ct
1032 ) 1039 )
1033 else: 1040 else:
1034 info = {'v1capabilities': set(rawdata.split())} 1041 info = {b'v1capabilities': set(rawdata.split())}
1035 1042
1036 return respurl, info 1043 return respurl, info
1037 1044
1038 1045
1039 def makepeer(ui, path, opener=None, requestbuilder=urlreq.request): 1046 def makepeer(ui, path, opener=None, requestbuilder=urlreq.request):
1046 It exists as an argument so extensions can override the default. 1053 It exists as an argument so extensions can override the default.
1047 """ 1054 """
1048 u = util.url(path) 1055 u = util.url(path)
1049 if u.query or u.fragment: 1056 if u.query or u.fragment:
1050 raise error.Abort( 1057 raise error.Abort(
1051 _('unsupported URL component: "%s"') % (u.query or u.fragment) 1058 _(b'unsupported URL component: "%s"') % (u.query or u.fragment)
1052 ) 1059 )
1053 1060
1054 # urllib cannot handle URLs with embedded user or passwd. 1061 # urllib cannot handle URLs with embedded user or passwd.
1055 url, authinfo = u.authinfo() 1062 url, authinfo = u.authinfo()
1056 ui.debug('using %s\n' % url) 1063 ui.debug(b'using %s\n' % url)
1057 1064
1058 opener = opener or urlmod.opener(ui, authinfo) 1065 opener = opener or urlmod.opener(ui, authinfo)
1059 1066
1060 respurl, info = performhandshake(ui, url, opener, requestbuilder) 1067 respurl, info = performhandshake(ui, url, opener, requestbuilder)
1061 1068
1066 # example, the caller could say "I want a peer that does X." It's quite 1073 # example, the caller could say "I want a peer that does X." It's quite
1067 # possible that not all peers would do that. Since we know the service 1074 # possible that not all peers would do that. Since we know the service
1068 # capabilities, we could filter out services not meeting the 1075 # capabilities, we could filter out services not meeting the
1069 # requirements. Possibly by consulting the interfaces defined by the 1076 # requirements. Possibly by consulting the interfaces defined by the
1070 # peer type. 1077 # peer type.
1071 apipeerchoices = set(info.get('apis', {}).keys()) & set(API_PEERS.keys()) 1078 apipeerchoices = set(info.get(b'apis', {}).keys()) & set(API_PEERS.keys())
1072 1079
1073 preferredchoices = sorted( 1080 preferredchoices = sorted(
1074 apipeerchoices, key=lambda x: API_PEERS[x]['priority'], reverse=True 1081 apipeerchoices, key=lambda x: API_PEERS[x][b'priority'], reverse=True
1075 ) 1082 )
1076 1083
1077 for service in preferredchoices: 1084 for service in preferredchoices:
1078 apipath = '%s/%s' % (info['apibase'].rstrip('/'), service) 1085 apipath = b'%s/%s' % (info[b'apibase'].rstrip(b'/'), service)
1079 1086
1080 return API_PEERS[service]['init']( 1087 return API_PEERS[service][b'init'](
1081 ui, respurl, apipath, opener, requestbuilder, info['apis'][service] 1088 ui, respurl, apipath, opener, requestbuilder, info[b'apis'][service]
1082 ) 1089 )
1083 1090
1084 # Failed to construct an API peer. Fall back to legacy. 1091 # Failed to construct an API peer. Fall back to legacy.
1085 return httppeer( 1092 return httppeer(
1086 ui, path, respurl, opener, requestbuilder, info['v1capabilities'] 1093 ui, path, respurl, opener, requestbuilder, info[b'v1capabilities']
1087 ) 1094 )
1088 1095
1089 1096
1090 def instance(ui, path, create, intents=None, createopts=None): 1097 def instance(ui, path, create, intents=None, createopts=None):
1091 if create: 1098 if create:
1092 raise error.Abort(_('cannot create new http repository')) 1099 raise error.Abort(_(b'cannot create new http repository'))
1093 try: 1100 try:
1094 if path.startswith('https:') and not urlmod.has_https: 1101 if path.startswith(b'https:') and not urlmod.has_https:
1095 raise error.Abort( 1102 raise error.Abort(
1096 _('Python support for SSL and HTTPS ' 'is not installed') 1103 _(b'Python support for SSL and HTTPS ' b'is not installed')
1097 ) 1104 )
1098 1105
1099 inst = makepeer(ui, path) 1106 inst = makepeer(ui, path)
1100 1107
1101 return inst 1108 return inst
1102 except error.RepoError as httpexception: 1109 except error.RepoError as httpexception:
1103 try: 1110 try:
1104 r = statichttprepo.instance(ui, "static-" + path, create) 1111 r = statichttprepo.instance(ui, b"static-" + path, create)
1105 ui.note(_('(falling back to static-http)\n')) 1112 ui.note(_(b'(falling back to static-http)\n'))
1106 return r 1113 return r
1107 except error.RepoError: 1114 except error.RepoError:
1108 raise httpexception # use the original http RepoError instead 1115 raise httpexception # use the original http RepoError instead