Mercurial > public > mercurial-scm > hg

comparison mercurial/httppeer.py @ 43076:2372284d9457

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
formatting: blacken the codebase This is using my patch to black (https://github.com/psf/black/pull/826) so we don't un-wrap collection literals. Done with: hg files 'set:**.py - mercurial/thirdparty/** - "contrib/python-zstandard/**"' | xargs black -S # skip-blame mass-reformatting only # no-check-commit reformats foo_bar functions Differential Revision: https://phab.mercurial-scm.org/D6971
author Augie Fackler <augie@google.com>
date Sun, 06 Oct 2019 09:45:02 -0400
parents 37debb6771f5
children 687b865b95ad
comparison
equal deleted inserted replaced
43075:57875cf423c9 43076:2372284d9457
41 41
42 httplib = util.httplib 42 httplib = util.httplib
43 urlerr = util.urlerr 43 urlerr = util.urlerr
44 urlreq = util.urlreq 44 urlreq = util.urlreq
45 45
46
46 def encodevalueinheaders(value, header, limit): 47 def encodevalueinheaders(value, header, limit):
47 """Encode a string value into multiple HTTP headers. 48 """Encode a string value into multiple HTTP headers.
48 49
49 ``value`` will be encoded into 1 or more HTTP headers with the names 50 ``value`` will be encoded into 1 or more HTTP headers with the names
50 ``header-<N>`` where ``<N>`` is an integer starting at 1. Each header 51 ``header-<N>`` where ``<N>`` is an integer starting at 1. Each header
65 result = [] 66 result = []
66 67
67 n = 0 68 n = 0
68 for i in pycompat.xrange(0, len(value), valuelen): 69 for i in pycompat.xrange(0, len(value), valuelen):
69 n += 1 70 n += 1
70 result.append((fmt % str(n), pycompat.strurl(value[i:i + valuelen]))) 71 result.append((fmt % str(n), pycompat.strurl(value[i : i + valuelen])))
71 72
72 return result 73 return result
74
73 75
74 class _multifile(object): 76 class _multifile(object):
75 def __init__(self, *fileobjs): 77 def __init__(self, *fileobjs):
76 for f in fileobjs: 78 for f in fileobjs:
77 if not util.safehasattr(f, 'length'): 79 if not util.safehasattr(f, 'length'):
78 raise ValueError( 80 raise ValueError(
79 '_multifile only supports file objects that ' 81 '_multifile only supports file objects that '
80 'have a length but this one does not:', type(f), f) 82 'have a length but this one does not:',
83 type(f),
84 f,
85 )
81 self._fileobjs = fileobjs 86 self._fileobjs = fileobjs
82 self._index = 0 87 self._index = 0
83 88
84 @property 89 @property
85 def length(self): 90 def length(self):
99 104
100 def seek(self, offset, whence=os.SEEK_SET): 105 def seek(self, offset, whence=os.SEEK_SET):
101 if whence != os.SEEK_SET: 106 if whence != os.SEEK_SET:
102 raise NotImplementedError( 107 raise NotImplementedError(
103 '_multifile does not support anything other' 108 '_multifile does not support anything other'
104 ' than os.SEEK_SET for whence on seek()') 109 ' than os.SEEK_SET for whence on seek()'
110 )
105 if offset != 0: 111 if offset != 0:
106 raise NotImplementedError( 112 raise NotImplementedError(
107 '_multifile only supports seeking to start, but that ' 113 '_multifile only supports seeking to start, but that '
108 'could be fixed if you need it') 114 'could be fixed if you need it'
115 )
109 for f in self._fileobjs: 116 for f in self._fileobjs:
110 f.seek(0) 117 f.seek(0)
111 self._index = 0 118 self._index = 0
112 119
113 def makev1commandrequest(ui, requestbuilder, caps, capablefn, 120
114 repobaseurl, cmd, args): 121 def makev1commandrequest(
122 ui, requestbuilder, caps, capablefn, repobaseurl, cmd, args
123 ):
115 """Make an HTTP request to run a command for a version 1 client. 124 """Make an HTTP request to run a command for a version 1 client.
116 125
117 ``caps`` is a set of known server capabilities. The value may be 126 ``caps`` is a set of known server capabilities. The value may be
118 None if capabilities are not yet known. 127 None if capabilities are not yet known.
119 128
160 169
161 # Send arguments via HTTP headers. 170 # Send arguments via HTTP headers.
162 if headersize > 0: 171 if headersize > 0:
163 # The headers can typically carry more data than the URL. 172 # The headers can typically carry more data than the URL.
164 encargs = urlreq.urlencode(sorted(args.items())) 173 encargs = urlreq.urlencode(sorted(args.items()))
165 for header, value in encodevalueinheaders(encargs, 'X-HgArg', 174 for header, value in encodevalueinheaders(
166 headersize): 175 encargs, 'X-HgArg', headersize
176 ):
167 headers[header] = value 177 headers[header] = value
168 # Send arguments via query string (Mercurial <1.9). 178 # Send arguments via query string (Mercurial <1.9).
169 else: 179 else:
170 q += sorted(args.items()) 180 q += sorted(args.items())
171 181
200 210
201 if '0.2tx' in mediatypes and capablefn('compression'): 211 if '0.2tx' in mediatypes and capablefn('compression'):
202 # We /could/ compare supported compression formats and prune 212 # We /could/ compare supported compression formats and prune
203 # non-mutually supported or error if nothing is mutually supported. 213 # non-mutually supported or error if nothing is mutually supported.
204 # For now, send the full list to the server and have it error. 214 # For now, send the full list to the server and have it error.
205 comps = [e.wireprotosupport().name for e in 215 comps = [
206 util.compengines.supportedwireengines(util.CLIENTROLE)] 216 e.wireprotosupport().name
217 for e in util.compengines.supportedwireengines(util.CLIENTROLE)
218 ]
207 protoparams.add('comp=%s' % ','.join(comps)) 219 protoparams.add('comp=%s' % ','.join(comps))
208 220
209 if protoparams: 221 if protoparams:
210 protoheaders = encodevalueinheaders(' '.join(sorted(protoparams)), 222 protoheaders = encodevalueinheaders(
211 'X-HgProto', 223 ' '.join(sorted(protoparams)), 'X-HgProto', headersize or 1024
212 headersize or 1024) 224 )
213 for header, value in protoheaders: 225 for header, value in protoheaders:
214 headers[header] = value 226 headers[header] = value
215 227
216 varyheaders = [] 228 varyheaders = []
217 for header in headers: 229 for header in headers:
226 if data is not None: 238 if data is not None:
227 ui.debug("sending %d bytes\n" % size) 239 ui.debug("sending %d bytes\n" % size)
228 req.add_unredirected_header(r'Content-Length', r'%d' % size) 240 req.add_unredirected_header(r'Content-Length', r'%d' % size)
229 241
230 return req, cu, qs 242 return req, cu, qs
243
231 244
232 def _reqdata(req): 245 def _reqdata(req):
233 """Get request data, if any. If no data, returns None.""" 246 """Get request data, if any. If no data, returns None."""
234 if pycompat.ispy3: 247 if pycompat.ispy3:
235 return req.data 248 return req.data
236 if not req.has_data(): 249 if not req.has_data():
237 return None 250 return None
238 return req.get_data() 251 return req.get_data()
239 252
253
240 def sendrequest(ui, opener, req): 254 def sendrequest(ui, opener, req):
241 """Send a prepared HTTP request. 255 """Send a prepared HTTP request.
242 256
243 Returns the response object. 257 Returns the response object.
244 """ 258 """
245 dbg = ui.debug 259 dbg = ui.debug
246 if (ui.debugflag 260 if ui.debugflag and ui.configbool('devel', 'debug.peer-request'):
247 and ui.configbool('devel', 'debug.peer-request')):
248 line = 'devel-peer-request: %s\n' 261 line = 'devel-peer-request: %s\n'
249 dbg(line % '%s %s' % (pycompat.bytesurl(req.get_method()), 262 dbg(
250 pycompat.bytesurl(req.get_full_url()))) 263 line
264 % '%s %s'
265 % (
266 pycompat.bytesurl(req.get_method()),
267 pycompat.bytesurl(req.get_full_url()),
268 )
269 )
251 hgargssize = None 270 hgargssize = None
252 271
253 for header, value in sorted(req.header_items()): 272 for header, value in sorted(req.header_items()):
254 header = pycompat.bytesurl(header) 273 header = pycompat.bytesurl(header)
255 value = pycompat.bytesurl(value) 274 value = pycompat.bytesurl(value)
259 hgargssize += len(value) 278 hgargssize += len(value)
260 else: 279 else:
261 dbg(line % ' %s %s' % (header, value)) 280 dbg(line % ' %s %s' % (header, value))
262 281
263 if hgargssize is not None: 282 if hgargssize is not None:
264 dbg(line % ' %d bytes of commands arguments in headers' 283 dbg(
265 % hgargssize) 284 line
285 % ' %d bytes of commands arguments in headers'
286 % hgargssize
287 )
266 data = _reqdata(req) 288 data = _reqdata(req)
267 if data is not None: 289 if data is not None:
268 length = getattr(data, 'length', None) 290 length = getattr(data, 'length', None)
269 if length is None: 291 if length is None:
270 length = len(data) 292 length = len(data)
278 except urlerr.httperror as inst: 300 except urlerr.httperror as inst:
279 if inst.code == 401: 301 if inst.code == 401:
280 raise error.Abort(_('authorization failed')) 302 raise error.Abort(_('authorization failed'))
281 raise 303 raise
282 except httplib.HTTPException as inst: 304 except httplib.HTTPException as inst:
283 ui.debug('http error requesting %s\n' % 305 ui.debug(
284 util.hidepassword(req.get_full_url())) 306 'http error requesting %s\n' % util.hidepassword(req.get_full_url())
307 )
285 ui.traceback() 308 ui.traceback()
286 raise IOError(None, inst) 309 raise IOError(None, inst)
287 finally: 310 finally:
288 if ui.debugflag and ui.configbool('devel', 'debug.peer-request'): 311 if ui.debugflag and ui.configbool('devel', 'debug.peer-request'):
289 code = res.code if res else -1 312 code = res.code if res else -1
290 dbg(line % ' finished in %.4f seconds (%d)' 313 dbg(
291 % (util.timer() - start, code)) 314 line
315 % ' finished in %.4f seconds (%d)'
316 % (util.timer() - start, code)
317 )
292 318
293 # Insert error handlers for common I/O failures. 319 # Insert error handlers for common I/O failures.
294 urlmod.wrapresponse(res) 320 urlmod.wrapresponse(res)
295 321
296 return res 322 return res
323
297 324
298 class RedirectedRepoError(error.RepoError): 325 class RedirectedRepoError(error.RepoError):
299 def __init__(self, msg, respurl): 326 def __init__(self, msg, respurl):
300 super(RedirectedRepoError, self).__init__(msg) 327 super(RedirectedRepoError, self).__init__(msg)
301 self.respurl = respurl 328 self.respurl = respurl
302 329
303 def parsev1commandresponse(ui, baseurl, requrl, qs, resp, compressible, 330
304 allowcbor=False): 331 def parsev1commandresponse(
332 ui, baseurl, requrl, qs, resp, compressible, allowcbor=False
333 ):
305 # record the url we got redirected to 334 # record the url we got redirected to
306 redirected = False 335 redirected = False
307 respurl = pycompat.bytesurl(resp.geturl()) 336 respurl = pycompat.bytesurl(resp.geturl())
308 if respurl.endswith(qs): 337 if respurl.endswith(qs):
309 respurl = respurl[:-len(qs)] 338 respurl = respurl[: -len(qs)]
310 qsdropped = False 339 qsdropped = False
311 else: 340 else:
312 qsdropped = True 341 qsdropped = True
313 342
314 if baseurl.rstrip('/') != respurl.rstrip('/'): 343 if baseurl.rstrip('/') != respurl.rstrip('/'):
327 356
328 # Pre 1.0 versions of Mercurial used text/plain and 357 # Pre 1.0 versions of Mercurial used text/plain and
329 # application/hg-changegroup. We don't support such old servers. 358 # application/hg-changegroup. We don't support such old servers.
330 if not proto.startswith('application/mercurial-'): 359 if not proto.startswith('application/mercurial-'):
331 ui.debug("requested URL: '%s'\n" % util.hidepassword(requrl)) 360 ui.debug("requested URL: '%s'\n" % util.hidepassword(requrl))
332 msg = _("'%s' does not appear to be an hg repository:\n" 361 msg = _(
333 "---%%<--- (%s)\n%s\n---%%<---\n") % ( 362 "'%s' does not appear to be an hg repository:\n"
334 safeurl, proto or 'no content-type', resp.read(1024)) 363 "---%%<--- (%s)\n%s\n---%%<---\n"
364 ) % (safeurl, proto or 'no content-type', resp.read(1024))
335 365
336 # Some servers may strip the query string from the redirect. We 366 # Some servers may strip the query string from the redirect. We
337 # raise a special error type so callers can react to this specially. 367 # raise a special error type so callers can react to this specially.
338 if redirected and qsdropped: 368 if redirected and qsdropped:
339 raise RedirectedRepoError(msg, respurl) 369 raise RedirectedRepoError(msg, respurl)
348 # request during handshake. 378 # request during handshake.
349 if subtype == 'cbor': 379 if subtype == 'cbor':
350 if allowcbor: 380 if allowcbor:
351 return respurl, proto, resp 381 return respurl, proto, resp
352 else: 382 else:
353 raise error.RepoError(_('unexpected CBOR response from ' 383 raise error.RepoError(
354 'server')) 384 _('unexpected CBOR response from ' 'server')
385 )
355 386
356 version_info = tuple([int(n) for n in subtype.split('.')]) 387 version_info = tuple([int(n) for n in subtype.split('.')])
357 except ValueError: 388 except ValueError:
358 raise error.RepoError(_("'%s' sent a broken Content-Type " 389 raise error.RepoError(
359 "header (%s)") % (safeurl, proto)) 390 _("'%s' sent a broken Content-Type " "header (%s)")
391 % (safeurl, proto)
392 )
360 393
361 # TODO consider switching to a decompression reader that uses 394 # TODO consider switching to a decompression reader that uses
362 # generators. 395 # generators.
363 if version_info == (0, 1): 396 if version_info == (0, 1):
364 if compressible: 397 if compressible:
371 ename = util.readexactly(resp, elen) 404 ename = util.readexactly(resp, elen)
372 engine = util.compengines.forwiretype(ename) 405 engine = util.compengines.forwiretype(ename)
373 406
374 resp = engine.decompressorreader(resp) 407 resp = engine.decompressorreader(resp)
375 else: 408 else:
376 raise error.RepoError(_("'%s' uses newer protocol %s") % 409 raise error.RepoError(
377 (safeurl, subtype)) 410 _("'%s' uses newer protocol %s") % (safeurl, subtype)
411 )
378 412
379 return respurl, proto, resp 413 return respurl, proto, resp
414
380 415
381 class httppeer(wireprotov1peer.wirepeer): 416 class httppeer(wireprotov1peer.wirepeer):
382 def __init__(self, ui, path, url, opener, requestbuilder, caps): 417 def __init__(self, ui, path, url, opener, requestbuilder, caps):
383 self.ui = ui 418 self.ui = ui
384 self._path = path 419 self._path = path
407 def canpush(self): 442 def canpush(self):
408 return True 443 return True
409 444
410 def close(self): 445 def close(self):
411 try: 446 try:
412 reqs, sent, recv = (self._urlopener.requestscount, 447 reqs, sent, recv = (
413 self._urlopener.sentbytescount, 448 self._urlopener.requestscount,
414 self._urlopener.receivedbytescount) 449 self._urlopener.sentbytescount,
450 self._urlopener.receivedbytescount,
451 )
415 except AttributeError: 452 except AttributeError:
416 return 453 return
417 self.ui.note(_('(sent %d HTTP requests and %d bytes; ' 454 self.ui.note(
418 'received %d bytes in responses)\n') % 455 _(
419 (reqs, sent, recv)) 456 '(sent %d HTTP requests and %d bytes; '
457 'received %d bytes in responses)\n'
458 )
459 % (reqs, sent, recv)
460 )
420 461
421 # End of ipeerconnection interface. 462 # End of ipeerconnection interface.
422 463
423 # Begin of ipeercommands interface. 464 # Begin of ipeercommands interface.
424 465
428 # End of ipeercommands interface. 469 # End of ipeercommands interface.
429 470
430 def _callstream(self, cmd, _compressible=False, **args): 471 def _callstream(self, cmd, _compressible=False, **args):
431 args = pycompat.byteskwargs(args) 472 args = pycompat.byteskwargs(args)
432 473
433 req, cu, qs = makev1commandrequest(self.ui, self._requestbuilder, 474 req, cu, qs = makev1commandrequest(
434 self._caps, self.capable, 475 self.ui,
435 self._url, cmd, args) 476 self._requestbuilder,
477 self._caps,
478 self.capable,
479 self._url,
480 cmd,
481 args,
482 )
436 483
437 resp = sendrequest(self.ui, self._urlopener, req) 484 resp = sendrequest(self.ui, self._urlopener, req)
438 485
439 self._url, ct, resp = parsev1commandresponse(self.ui, self._url, cu, qs, 486 self._url, ct, resp = parsev1commandresponse(
440 resp, _compressible) 487 self.ui, self._url, cu, qs, resp, _compressible
488 )
441 489
442 return resp 490 return resp
443 491
444 def _call(self, cmd, **args): 492 def _call(self, cmd, **args):
445 fp = self._callstream(cmd, **args) 493 fp = self._callstream(cmd, **args)
511 return self._callstream(cmd, _compressible=True, **args) 559 return self._callstream(cmd, _compressible=True, **args)
512 560
513 def _abort(self, exception): 561 def _abort(self, exception):
514 raise exception 562 raise exception
515 563
516 def sendv2request(ui, opener, requestbuilder, apiurl, permission, requests, 564
517 redirect): 565 def sendv2request(
566 ui, opener, requestbuilder, apiurl, permission, requests, redirect
567 ):
518 wireprotoframing.populatestreamencoders() 568 wireprotoframing.populatestreamencoders()
519 569
520 uiencoders = ui.configlist(b'experimental', b'httppeer.v2-encoder-order') 570 uiencoders = ui.configlist(b'experimental', b'httppeer.v2-encoder-order')
521 571
522 if uiencoders: 572 if uiencoders:
523 encoders = [] 573 encoders = []
524 574
525 for encoder in uiencoders: 575 for encoder in uiencoders:
526 if encoder not in wireprotoframing.STREAM_ENCODERS: 576 if encoder not in wireprotoframing.STREAM_ENCODERS:
527 ui.warn(_(b'wire protocol version 2 encoder referenced in ' 577 ui.warn(
528 b'config (%s) is not known; ignoring\n') % encoder) 578 _(
579 b'wire protocol version 2 encoder referenced in '
580 b'config (%s) is not known; ignoring\n'
581 )
582 % encoder
583 )
529 else: 584 else:
530 encoders.append(encoder) 585 encoders.append(encoder)
531 586
532 else: 587 else:
533 encoders = wireprotoframing.STREAM_ENCODERS_ORDER 588 encoders = wireprotoframing.STREAM_ENCODERS_ORDER
534 589
535 reactor = wireprotoframing.clientreactor(ui, 590 reactor = wireprotoframing.clientreactor(
536 hasmultiplesend=False, 591 ui,
537 buffersends=True, 592 hasmultiplesend=False,
538 clientcontentencoders=encoders) 593 buffersends=True,
539 594 clientcontentencoders=encoders,
540 handler = wireprotov2peer.clienthandler(ui, reactor, 595 )
541 opener=opener, 596
542 requestbuilder=requestbuilder) 597 handler = wireprotov2peer.clienthandler(
598 ui, reactor, opener=opener, requestbuilder=requestbuilder
599 )
543 600
544 url = '%s/%s' % (apiurl, permission) 601 url = '%s/%s' % (apiurl, permission)
545 602
546 if len(requests) > 1: 603 if len(requests) > 1:
547 url += '/multirequest' 604 url += '/multirequest'
548 else: 605 else:
549 url += '/%s' % requests[0][0] 606 url += '/%s' % requests[0][0]
550 607
551 ui.debug('sending %d commands\n' % len(requests)) 608 ui.debug('sending %d commands\n' % len(requests))
552 for command, args, f in requests: 609 for command, args, f in requests:
553 ui.debug('sending command %s: %s\n' % ( 610 ui.debug(
554 command, stringutil.pprint(args, indent=2))) 611 'sending command %s: %s\n'
555 assert not list(handler.callcommand(command, args, f, 612 % (command, stringutil.pprint(args, indent=2))
556 redirect=redirect)) 613 )
614 assert not list(
615 handler.callcommand(command, args, f, redirect=redirect)
616 )
557 617
558 # TODO stream this. 618 # TODO stream this.
559 body = b''.join(map(bytes, handler.flushcommands())) 619 body = b''.join(map(bytes, handler.flushcommands()))
560 620
561 # TODO modify user-agent to reflect v2 621 # TODO modify user-agent to reflect v2
578 ui.traceback() 638 ui.traceback()
579 raise IOError(None, e) 639 raise IOError(None, e)
580 640
581 return handler, res 641 return handler, res
582 642
643
583 class queuedcommandfuture(pycompat.futures.Future): 644 class queuedcommandfuture(pycompat.futures.Future):
584 """Wraps result() on command futures to trigger submission on call.""" 645 """Wraps result() on command futures to trigger submission on call."""
585 646
586 def result(self, timeout=None): 647 def result(self, timeout=None):
587 if self.done(): 648 if self.done():
591 652
592 # sendcommands() will restore the original __class__ and self.result 653 # sendcommands() will restore the original __class__ and self.result
593 # will resolve to Future.result. 654 # will resolve to Future.result.
594 return self.result(timeout) 655 return self.result(timeout)
595 656
657
596 @interfaceutil.implementer(repository.ipeercommandexecutor) 658 @interfaceutil.implementer(repository.ipeercommandexecutor)
597 class httpv2executor(object): 659 class httpv2executor(object):
598 def __init__(self, ui, opener, requestbuilder, apiurl, descriptor, 660 def __init__(
599 redirect): 661 self, ui, opener, requestbuilder, apiurl, descriptor, redirect
662 ):
600 self._ui = ui 663 self._ui = ui
601 self._opener = opener 664 self._opener = opener
602 self._requestbuilder = requestbuilder 665 self._requestbuilder = requestbuilder
603 self._apiurl = apiurl 666 self._apiurl = apiurl
604 self._descriptor = descriptor 667 self._descriptor = descriptor
617 def __exit__(self, exctype, excvalue, exctb): 680 def __exit__(self, exctype, excvalue, exctb):
618 self.close() 681 self.close()
619 682
620 def callcommand(self, command, args): 683 def callcommand(self, command, args):
621 if self._sent: 684 if self._sent:
622 raise error.ProgrammingError('callcommand() cannot be used after ' 685 raise error.ProgrammingError(
623 'commands are sent') 686 'callcommand() cannot be used after ' 'commands are sent'
687 )
624 688
625 if self._closed: 689 if self._closed:
626 raise error.ProgrammingError('callcommand() cannot be used after ' 690 raise error.ProgrammingError(
627 'close()') 691 'callcommand() cannot be used after ' 'close()'
692 )
628 693
629 # The service advertises which commands are available. So if we attempt 694 # The service advertises which commands are available. So if we attempt
630 # to call an unknown command or pass an unknown argument, we can screen 695 # to call an unknown command or pass an unknown argument, we can screen
631 # for this. 696 # for this.
632 if command not in self._descriptor['commands']: 697 if command not in self._descriptor['commands']:
633 raise error.ProgrammingError( 698 raise error.ProgrammingError(
634 'wire protocol command %s is not available' % command) 699 'wire protocol command %s is not available' % command
700 )
635 701
636 cmdinfo = self._descriptor['commands'][command] 702 cmdinfo = self._descriptor['commands'][command]
637 unknownargs = set(args.keys()) - set(cmdinfo.get('args', {})) 703 unknownargs = set(args.keys()) - set(cmdinfo.get('args', {}))
638 704
639 if unknownargs: 705 if unknownargs:
640 raise error.ProgrammingError( 706 raise error.ProgrammingError(
641 'wire protocol command %s does not accept argument: %s' % ( 707 'wire protocol command %s does not accept argument: %s'
642 command, ', '.join(sorted(unknownargs)))) 708 % (command, ', '.join(sorted(unknownargs)))
709 )
643 710
644 self._neededpermissions |= set(cmdinfo['permissions']) 711 self._neededpermissions |= set(cmdinfo['permissions'])
645 712
646 # TODO we /could/ also validate types here, since the API descriptor 713 # TODO we /could/ also validate types here, since the API descriptor
647 # includes types... 714 # includes types...
673 if isinstance(f, queuedcommandfuture): 740 if isinstance(f, queuedcommandfuture):
674 f.__class__ = pycompat.futures.Future 741 f.__class__ = pycompat.futures.Future
675 f._peerexecutor = None 742 f._peerexecutor = None
676 743
677 # Mark the future as running and filter out cancelled futures. 744 # Mark the future as running and filter out cancelled futures.
678 calls = [(command, args, f) 745 calls = [
679 for command, args, f in self._calls 746 (command, args, f)
680 if f.set_running_or_notify_cancel()] 747 for command, args, f in self._calls
748 if f.set_running_or_notify_cancel()
749 ]
681 750
682 # Clear out references, prevent improper object usage. 751 # Clear out references, prevent improper object usage.
683 self._calls = None 752 self._calls = None
684 753
685 if not calls: 754 if not calls:
689 758
690 if 'push' in permissions and 'pull' in permissions: 759 if 'push' in permissions and 'pull' in permissions:
691 permissions.remove('pull') 760 permissions.remove('pull')
692 761
693 if len(permissions) > 1: 762 if len(permissions) > 1:
694 raise error.RepoError(_('cannot make request requiring multiple ' 763 raise error.RepoError(
695 'permissions: %s') % 764 _('cannot make request requiring multiple ' 'permissions: %s')
696 _(', ').join(sorted(permissions))) 765 % _(', ').join(sorted(permissions))
697 766 )
698 permission = { 767
699 'push': 'rw', 768 permission = {'push': 'rw', 'pull': 'ro',}[permissions.pop()]
700 'pull': 'ro',
701 }[permissions.pop()]
702 769
703 handler, resp = sendv2request( 770 handler, resp = sendv2request(
704 self._ui, self._opener, self._requestbuilder, self._apiurl, 771 self._ui,
705 permission, calls, self._redirect) 772 self._opener,
773 self._requestbuilder,
774 self._apiurl,
775 permission,
776 calls,
777 self._redirect,
778 )
706 779
707 # TODO we probably want to validate the HTTP code, media type, etc. 780 # TODO we probably want to validate the HTTP code, media type, etc.
708 781
709 self._responseexecutor = pycompat.futures.ThreadPoolExecutor(1) 782 self._responseexecutor = pycompat.futures.ThreadPoolExecutor(1)
710 self._responsef = self._responseexecutor.submit(self._handleresponse, 783 self._responsef = self._responseexecutor.submit(
711 handler, resp) 784 self._handleresponse, handler, resp
785 )
712 786
713 def close(self): 787 def close(self):
714 if self._closed: 788 if self._closed:
715 return 789 return
716 790
732 806
733 # If any of our futures are still in progress, mark them as 807 # If any of our futures are still in progress, mark them as
734 # errored, otherwise a result() could wait indefinitely. 808 # errored, otherwise a result() could wait indefinitely.
735 for f in self._futures: 809 for f in self._futures:
736 if not f.done(): 810 if not f.done():
737 f.set_exception(error.ResponseError( 811 f.set_exception(
738 _('unfulfilled command response'))) 812 error.ResponseError(_('unfulfilled command response'))
813 )
739 814
740 self._futures = None 815 self._futures = None
741 816
742 def _handleresponse(self, handler, resp): 817 def _handleresponse(self, handler, resp):
743 # Called in a thread to read the response. 818 # Called in a thread to read the response.
744 819
745 while handler.readdata(resp): 820 while handler.readdata(resp):
746 pass 821 pass
747 822
823
748 @interfaceutil.implementer(repository.ipeerv2) 824 @interfaceutil.implementer(repository.ipeerv2)
749 class httpv2peer(object): 825 class httpv2peer(object):
750 826
751 limitedarguments = False 827 limitedarguments = False
752 828
753 def __init__(self, ui, repourl, apipath, opener, requestbuilder, 829 def __init__(
754 apidescriptor): 830 self, ui, repourl, apipath, opener, requestbuilder, apidescriptor
831 ):
755 self.ui = ui 832 self.ui = ui
756 self.apidescriptor = apidescriptor 833 self.apidescriptor = apidescriptor
757 834
758 if repourl.endswith('/'): 835 if repourl.endswith('/'):
759 repourl = repourl[:-1] 836 repourl = repourl[:-1]
780 def canpush(self): 857 def canpush(self):
781 # TODO change once implemented. 858 # TODO change once implemented.
782 return False 859 return False
783 860
784 def close(self): 861 def close(self):
785 self.ui.note(_('(sent %d HTTP requests and %d bytes; ' 862 self.ui.note(
786 'received %d bytes in responses)\n') % 863 _(
787 (self._opener.requestscount, 864 '(sent %d HTTP requests and %d bytes; '
788 self._opener.sentbytescount, 865 'received %d bytes in responses)\n'
789 self._opener.receivedbytescount)) 866 )
867 % (
868 self._opener.requestscount,
869 self._opener.sentbytescount,
870 self._opener.receivedbytescount,
871 )
872 )
790 873
791 # End of ipeerconnection. 874 # End of ipeerconnection.
792 875
793 # Start of ipeercapabilities. 876 # Start of ipeercapabilities.
794 877
800 # Maps to commands that are available. 883 # Maps to commands that are available.
801 if name in ('branchmap', 'getbundle', 'known', 'lookup', 'pushkey'): 884 if name in ('branchmap', 'getbundle', 'known', 'lookup', 'pushkey'):
802 return True 885 return True
803 886
804 # Other concepts. 887 # Other concepts.
805 if name in ('bundle2'): 888 if name in 'bundle2':
806 return True 889 return True
807 890
808 # Alias command-* to presence of command of that name. 891 # Alias command-* to presence of command of that name.
809 if name.startswith('command-'): 892 if name.startswith('command-'):
810 return name[len('command-'):] in self.apidescriptor['commands'] 893 return name[len('command-') :] in self.apidescriptor['commands']
811 894
812 return False 895 return False
813 896
814 def requirecap(self, name, purpose): 897 def requirecap(self, name, purpose):
815 if self.capable(name): 898 if self.capable(name):
816 return 899 return
817 900
818 raise error.CapabilityError( 901 raise error.CapabilityError(
819 _('cannot %s; client or remote repository does not support the ' 902 _(
820 '\'%s\' capability') % (purpose, name)) 903 'cannot %s; client or remote repository does not support the '
904 '\'%s\' capability'
905 )
906 % (purpose, name)
907 )
821 908
822 # End of ipeercapabilities. 909 # End of ipeercapabilities.
823 910
824 def _call(self, name, **args): 911 def _call(self, name, **args):
825 with self.commandexecutor() as e: 912 with self.commandexecutor() as e:
826 return e.callcommand(name, args).result() 913 return e.callcommand(name, args).result()
827 914
828 def commandexecutor(self): 915 def commandexecutor(self):
829 return httpv2executor(self.ui, self._opener, self._requestbuilder, 916 return httpv2executor(
830 self._apiurl, self.apidescriptor, self._redirect) 917 self.ui,
918 self._opener,
919 self._requestbuilder,
920 self._apiurl,
921 self.apidescriptor,
922 self._redirect,
923 )
924
831 925
832 # Registry of API service names to metadata about peers that handle it. 926 # Registry of API service names to metadata about peers that handle it.
833 # 927 #
834 # The following keys are meaningful: 928 # The following keys are meaningful:
835 # 929 #
839 # 933 #
840 # priority 934 # priority
841 # Integer priority for the service. If we could choose from multiple 935 # Integer priority for the service. If we could choose from multiple
842 # services, we choose the one with the highest priority. 936 # services, we choose the one with the highest priority.
843 API_PEERS = { 937 API_PEERS = {
844 wireprototypes.HTTP_WIREPROTO_V2: { 938 wireprototypes.HTTP_WIREPROTO_V2: {'init': httpv2peer, 'priority': 50,},
845 'init': httpv2peer,
846 'priority': 50,
847 },
848 } 939 }
940
849 941
850 def performhandshake(ui, url, opener, requestbuilder): 942 def performhandshake(ui, url, opener, requestbuilder):
851 # The handshake is a request to the capabilities command. 943 # The handshake is a request to the capabilities command.
852 944
853 caps = None 945 caps = None
946
854 def capable(x): 947 def capable(x):
855 raise error.ProgrammingError('should not be called') 948 raise error.ProgrammingError('should not be called')
856 949
857 args = {} 950 args = {}
858 951
867 args['headers'] = { 960 args['headers'] = {
868 r'X-HgProto-1': r'cbor', 961 r'X-HgProto-1': r'cbor',
869 } 962 }
870 963
871 args['headers'].update( 964 args['headers'].update(
872 encodevalueinheaders(' '.join(sorted(API_PEERS)), 965 encodevalueinheaders(
873 'X-HgUpgrade', 966 ' '.join(sorted(API_PEERS)),
874 # We don't know the header limit this early. 967 'X-HgUpgrade',
875 # So make it small. 968 # We don't know the header limit this early.
876 1024)) 969 # So make it small.
877 970 1024,
878 req, requrl, qs = makev1commandrequest(ui, requestbuilder, caps, 971 )
879 capable, url, 'capabilities', 972 )
880 args) 973
974 req, requrl, qs = makev1commandrequest(
975 ui, requestbuilder, caps, capable, url, 'capabilities', args
976 )
881 resp = sendrequest(ui, opener, req) 977 resp = sendrequest(ui, opener, req)
882 978
883 # The server may redirect us to the repo root, stripping the 979 # The server may redirect us to the repo root, stripping the
884 # ?cmd=capabilities query string from the URL. The server would likely 980 # ?cmd=capabilities query string from the URL. The server would likely
885 # return HTML in this case and ``parsev1commandresponse()`` would raise. 981 # return HTML in this case and ``parsev1commandresponse()`` would raise.
891 # However, Mercurial clients for several years appeared to handle this 987 # However, Mercurial clients for several years appeared to handle this
892 # issue without behavior degradation. And according to issue 5860, it may 988 # issue without behavior degradation. And according to issue 5860, it may
893 # be a longstanding bug in some server implementations. So we allow a 989 # be a longstanding bug in some server implementations. So we allow a
894 # redirect that drops the query string to "just work." 990 # redirect that drops the query string to "just work."
895 try: 991 try:
896 respurl, ct, resp = parsev1commandresponse(ui, url, requrl, qs, resp, 992 respurl, ct, resp = parsev1commandresponse(
897 compressible=False, 993 ui, url, requrl, qs, resp, compressible=False, allowcbor=advertisev2
898 allowcbor=advertisev2) 994 )
899 except RedirectedRepoError as e: 995 except RedirectedRepoError as e:
900 req, requrl, qs = makev1commandrequest(ui, requestbuilder, caps, 996 req, requrl, qs = makev1commandrequest(
901 capable, e.respurl, 997 ui, requestbuilder, caps, capable, e.respurl, 'capabilities', args
902 'capabilities', args) 998 )
903 resp = sendrequest(ui, opener, req) 999 resp = sendrequest(ui, opener, req)
904 respurl, ct, resp = parsev1commandresponse(ui, url, requrl, qs, resp, 1000 respurl, ct, resp = parsev1commandresponse(
905 compressible=False, 1001 ui, url, requrl, qs, resp, compressible=False, allowcbor=advertisev2
906 allowcbor=advertisev2) 1002 )
907 1003
908 try: 1004 try:
909 rawdata = resp.read() 1005 rawdata = resp.read()
910 finally: 1006 finally:
911 resp.close() 1007 resp.close()
916 if advertisev2: 1012 if advertisev2:
917 if ct == 'application/mercurial-cbor': 1013 if ct == 'application/mercurial-cbor':
918 try: 1014 try:
919 info = cborutil.decodeall(rawdata)[0] 1015 info = cborutil.decodeall(rawdata)[0]
920 except cborutil.CBORDecodeError: 1016 except cborutil.CBORDecodeError:
921 raise error.Abort(_('error decoding CBOR from remote server'), 1017 raise error.Abort(
922 hint=_('try again and consider contacting ' 1018 _('error decoding CBOR from remote server'),
923 'the server operator')) 1019 hint=_(
1020 'try again and consider contacting '
1021 'the server operator'
1022 ),
1023 )
924 1024
925 # We got a legacy response. That's fine. 1025 # We got a legacy response. That's fine.
926 elif ct in ('application/mercurial-0.1', 'application/mercurial-0.2'): 1026 elif ct in ('application/mercurial-0.1', 'application/mercurial-0.2'):
927 info = { 1027 info = {'v1capabilities': set(rawdata.split())}
928 'v1capabilities': set(rawdata.split())
929 }
930 1028
931 else: 1029 else:
932 raise error.RepoError( 1030 raise error.RepoError(
933 _('unexpected response type from server: %s') % ct) 1031 _('unexpected response type from server: %s') % ct
1032 )
934 else: 1033 else:
935 info = { 1034 info = {'v1capabilities': set(rawdata.split())}
936 'v1capabilities': set(rawdata.split())
937 }
938 1035
939 return respurl, info 1036 return respurl, info
1037
940 1038
941 def makepeer(ui, path, opener=None, requestbuilder=urlreq.request): 1039 def makepeer(ui, path, opener=None, requestbuilder=urlreq.request):
942 """Construct an appropriate HTTP peer instance. 1040 """Construct an appropriate HTTP peer instance.
943 1041
944 ``opener`` is an ``url.opener`` that should be used to establish 1042 ``opener`` is an ``url.opener`` that should be used to establish
947 ``requestbuilder`` is the type used for constructing HTTP requests. 1045 ``requestbuilder`` is the type used for constructing HTTP requests.
948 It exists as an argument so extensions can override the default. 1046 It exists as an argument so extensions can override the default.
949 """ 1047 """
950 u = util.url(path) 1048 u = util.url(path)
951 if u.query or u.fragment: 1049 if u.query or u.fragment:
952 raise error.Abort(_('unsupported URL component: "%s"') % 1050 raise error.Abort(
953 (u.query or u.fragment)) 1051 _('unsupported URL component: "%s"') % (u.query or u.fragment)
1052 )
954 1053
955 # urllib cannot handle URLs with embedded user or passwd. 1054 # urllib cannot handle URLs with embedded user or passwd.
956 url, authinfo = u.authinfo() 1055 url, authinfo = u.authinfo()
957 ui.debug('using %s\n' % url) 1056 ui.debug('using %s\n' % url)
958 1057
969 # capabilities, we could filter out services not meeting the 1068 # capabilities, we could filter out services not meeting the
970 # requirements. Possibly by consulting the interfaces defined by the 1069 # requirements. Possibly by consulting the interfaces defined by the
971 # peer type. 1070 # peer type.
972 apipeerchoices = set(info.get('apis', {}).keys()) & set(API_PEERS.keys()) 1071 apipeerchoices = set(info.get('apis', {}).keys()) & set(API_PEERS.keys())
973 1072
974 preferredchoices = sorted(apipeerchoices, 1073 preferredchoices = sorted(
975 key=lambda x: API_PEERS[x]['priority'], 1074 apipeerchoices, key=lambda x: API_PEERS[x]['priority'], reverse=True
976 reverse=True) 1075 )
977 1076
978 for service in preferredchoices: 1077 for service in preferredchoices:
979 apipath = '%s/%s' % (info['apibase'].rstrip('/'), service) 1078 apipath = '%s/%s' % (info['apibase'].rstrip('/'), service)
980 1079
981 return API_PEERS[service]['init'](ui, respurl, apipath, opener, 1080 return API_PEERS[service]['init'](
982 requestbuilder, 1081 ui, respurl, apipath, opener, requestbuilder, info['apis'][service]
983 info['apis'][service]) 1082 )
984 1083
985 # Failed to construct an API peer. Fall back to legacy. 1084 # Failed to construct an API peer. Fall back to legacy.
986 return httppeer(ui, path, respurl, opener, requestbuilder, 1085 return httppeer(
987 info['v1capabilities']) 1086 ui, path, respurl, opener, requestbuilder, info['v1capabilities']
1087 )
1088
988 1089
989 def instance(ui, path, create, intents=None, createopts=None): 1090 def instance(ui, path, create, intents=None, createopts=None):
990 if create: 1091 if create:
991 raise error.Abort(_('cannot create new http repository')) 1092 raise error.Abort(_('cannot create new http repository'))
992 try: 1093 try:
993 if path.startswith('https:') and not urlmod.has_https: 1094 if path.startswith('https:') and not urlmod.has_https:
994 raise error.Abort(_('Python support for SSL and HTTPS ' 1095 raise error.Abort(
995 'is not installed')) 1096 _('Python support for SSL and HTTPS ' 'is not installed')
1097 )
996 1098
997 inst = makepeer(ui, path) 1099 inst = makepeer(ui, path)
998 1100
999 return inst 1101 return inst
1000 except error.RepoError as httpexception: 1102 except error.RepoError as httpexception:
1001 try: 1103 try:
1002 r = statichttprepo.instance(ui, "static-" + path, create) 1104 r = statichttprepo.instance(ui, "static-" + path, create)
1003 ui.note(_('(falling back to static-http)\n')) 1105 ui.note(_('(falling back to static-http)\n'))
1004 return r 1106 return r
1005 except error.RepoError: 1107 except error.RepoError:
1006 raise httpexception # use the original http RepoError instead 1108 raise httpexception # use the original http RepoError instead