Mercurial > public > mercurial-scm > hg

comparison mercurial/keepalive.py @ 28883:032c4c2f802a

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
pycompat: switch to util.urlreq/util.urlerr for py3 compat
author timeless <timeless@mozdev.org>
date Wed, 06 Apr 2016 23:22:12 +0000
parents b1b22185c764
children 0d83ad967bf8
comparison
equal deleted inserted replaced
28882:800ec7c048b0 28883:032c4c2f802a
26 """An HTTP handler for urllib2 that supports HTTP 1.1 and keepalive. 26 """An HTTP handler for urllib2 that supports HTTP 1.1 and keepalive.
27 27
28 >>> import urllib2 28 >>> import urllib2
29 >>> from keepalive import HTTPHandler 29 >>> from keepalive import HTTPHandler
30 >>> keepalive_handler = HTTPHandler() 30 >>> keepalive_handler = HTTPHandler()
31 >>> opener = urllib2.build_opener(keepalive_handler) 31 >>> opener = urlreq.buildopener(keepalive_handler)
32 >>> urllib2.install_opener(opener) 32 >>> urlreq.installopener(opener)
33 >>> 33 >>>
34 >>> fo = urllib2.urlopen('http://www.python.org') 34 >>> fo = urlreq.urlopen('http://www.python.org')
35 35
36 If a connection to a given host is requested, and all of the existing 36 If a connection to a given host is requested, and all of the existing
37 connections are still in use, another connection will be opened. If 37 connections are still in use, another connection will be opened. If
38 the handler tries to use an existing connection but it fails in some 38 the handler tries to use an existing connection but it fails in some
39 way, it will be closed and removed from the pool. 39 way, it will be closed and removed from the pool.
112 import errno 112 import errno
113 import httplib 113 import httplib
114 import socket 114 import socket
115 import sys 115 import sys
116 import thread 116 import thread
117 import urllib2 117
118 from . import (
119 util,
120 )
121
122 urlerr = util.urlerr
123 urlreq = util.urlreq
118 124
119 DEBUG = None 125 DEBUG = None
120 126
121 if sys.version_info < (2, 4): 127 if sys.version_info < (2, 4):
122 HANDLE_ERRORS = 1 128 HANDLE_ERRORS = 1
225 return self.do_open(HTTPConnection, req) 231 return self.do_open(HTTPConnection, req)
226 232
227 def do_open(self, http_class, req): 233 def do_open(self, http_class, req):
228 host = req.get_host() 234 host = req.get_host()
229 if not host: 235 if not host:
230 raise urllib2.URLError('no host given') 236 raise urlerr.urlerror('no host given')
231 237
232 try: 238 try:
233 h = self._cm.get_ready_conn(host) 239 h = self._cm.get_ready_conn(host)
234 while h: 240 while h:
235 r = self._reuse_connection(h, req, host) 241 r = self._reuse_connection(h, req, host)
252 host, id(h)) 258 host, id(h))
253 self._cm.add(host, h, 0) 259 self._cm.add(host, h, 0)
254 self._start_transaction(h, req) 260 self._start_transaction(h, req)
255 r = h.getresponse() 261 r = h.getresponse()
256 except (socket.error, httplib.HTTPException) as err: 262 except (socket.error, httplib.HTTPException) as err:
257 raise urllib2.URLError(err) 263 raise urlerr.urlerror(err)
258 264
259 # if not a persistent connection, don't try to reuse it 265 # if not a persistent connection, don't try to reuse it
260 if r.will_close: 266 if r.will_close:
261 self._cm.remove(h) 267 self._cm.remove(h)
262 268
344 if 'content-length' not in headers: 350 if 'content-length' not in headers:
345 h.putheader('Content-length', '%d' % len(data)) 351 h.putheader('Content-length', '%d' % len(data))
346 else: 352 else:
347 h.putrequest('GET', req.get_selector(), **skipheaders) 353 h.putrequest('GET', req.get_selector(), **skipheaders)
348 except socket.error as err: 354 except socket.error as err:
349 raise urllib2.URLError(err) 355 raise urlerr.urlerror(err)
350 for k, v in headers.items(): 356 for k, v in headers.items():
351 h.putheader(k, v) 357 h.putheader(k, v)
352 h.endheaders() 358 h.endheaders()
353 if req.has_data(): 359 if req.has_data():
354 h.send(data) 360 h.send(data)
355 361
356 class HTTPHandler(KeepAliveHandler, urllib2.HTTPHandler): 362 class HTTPHandler(KeepAliveHandler, urlreq.httphandler):
357 pass 363 pass
358 364
359 class HTTPResponse(httplib.HTTPResponse): 365 class HTTPResponse(httplib.HTTPResponse):
360 # we need to subclass HTTPResponse in order to 366 # we need to subclass HTTPResponse in order to
361 # 1) add readline() and readlines() methods 367 # 1) add readline() and readlines() methods
591 597
592 def error_handler(url): 598 def error_handler(url):
593 global HANDLE_ERRORS 599 global HANDLE_ERRORS
594 orig = HANDLE_ERRORS 600 orig = HANDLE_ERRORS
595 keepalive_handler = HTTPHandler() 601 keepalive_handler = HTTPHandler()
596 opener = urllib2.build_opener(keepalive_handler) 602 opener = urlreq.buildopener(keepalive_handler)
597 urllib2.install_opener(opener) 603 urlreq.installopener(opener)
598 pos = {0: 'off', 1: 'on'} 604 pos = {0: 'off', 1: 'on'}
599 for i in (0, 1): 605 for i in (0, 1):
600 print(" fancy error handling %s (HANDLE_ERRORS = %i)" % (pos[i], i)) 606 print(" fancy error handling %s (HANDLE_ERRORS = %i)" % (pos[i], i))
601 HANDLE_ERRORS = i 607 HANDLE_ERRORS = i
602 try: 608 try:
603 fo = urllib2.urlopen(url) 609 fo = urlreq.urlopen(url)
604 fo.read() 610 fo.read()
605 fo.close() 611 fo.close()
606 try: 612 try:
607 status, reason = fo.status, fo.reason 613 status, reason = fo.status, fo.reason
608 except AttributeError: 614 except AttributeError:
621 from . import util 627 from . import util
622 md5 = util.md5 628 md5 = util.md5
623 format = '%25s: %s' 629 format = '%25s: %s'
624 630
625 # first fetch the file with the normal http handler 631 # first fetch the file with the normal http handler
626 opener = urllib2.build_opener() 632 opener = urlreq.buildopener()
627 urllib2.install_opener(opener) 633 urlreq.installopener(opener)
628 fo = urllib2.urlopen(url) 634 fo = urlreq.urlopen(url)
629 foo = fo.read() 635 foo = fo.read()
630 fo.close() 636 fo.close()
631 m = md5(foo) 637 m = md5(foo)
632 print(format % ('normal urllib', m.hexdigest())) 638 print(format % ('normal urllib', m.hexdigest()))
633 639
634 # now install the keepalive handler and try again 640 # now install the keepalive handler and try again
635 opener = urllib2.build_opener(HTTPHandler()) 641 opener = urlreq.buildopener(HTTPHandler())
636 urllib2.install_opener(opener) 642 urlreq.installopener(opener)
637 643
638 fo = urllib2.urlopen(url) 644 fo = urlreq.urlopen(url)
639 foo = fo.read() 645 foo = fo.read()
640 fo.close() 646 fo.close()
641 m = md5(foo) 647 m = md5(foo)
642 print(format % ('keepalive read', m.hexdigest())) 648 print(format % ('keepalive read', m.hexdigest()))
643 649
644 fo = urllib2.urlopen(url) 650 fo = urlreq.urlopen(url)
645 foo = '' 651 foo = ''
646 while True: 652 while True:
647 f = fo.readline() 653 f = fo.readline()
648 if f: 654 if f:
649 foo = foo + f 655 foo = foo + f
655 def comp(N, url): 661 def comp(N, url):
656 print(' making %i connections to:\n %s' % (N, url)) 662 print(' making %i connections to:\n %s' % (N, url))
657 663
658 sys.stdout.write(' first using the normal urllib handlers') 664 sys.stdout.write(' first using the normal urllib handlers')
659 # first use normal opener 665 # first use normal opener
660 opener = urllib2.build_opener() 666 opener = urlreq.buildopener()
661 urllib2.install_opener(opener) 667 urlreq.installopener(opener)
662 t1 = fetch(N, url) 668 t1 = fetch(N, url)
663 print(' TIME: %.3f s' % t1) 669 print(' TIME: %.3f s' % t1)
664 670
665 sys.stdout.write(' now using the keepalive handler ') 671 sys.stdout.write(' now using the keepalive handler ')
666 # now install the keepalive handler and try again 672 # now install the keepalive handler and try again
667 opener = urllib2.build_opener(HTTPHandler()) 673 opener = urlreq.buildopener(HTTPHandler())
668 urllib2.install_opener(opener) 674 urlreq.installopener(opener)
669 t2 = fetch(N, url) 675 t2 = fetch(N, url)
670 print(' TIME: %.3f s' % t2) 676 print(' TIME: %.3f s' % t2)
671 print(' improvement factor: %.2f' % (t1 / t2)) 677 print(' improvement factor: %.2f' % (t1 / t2))
672 678
673 def fetch(N, url, delay=0): 679 def fetch(N, url, delay=0):
675 lens = [] 681 lens = []
676 starttime = time.time() 682 starttime = time.time()
677 for i in range(N): 683 for i in range(N):
678 if delay and i > 0: 684 if delay and i > 0:
679 time.sleep(delay) 685 time.sleep(delay)
680 fo = urllib2.urlopen(url) 686 fo = urlreq.urlopen(url)
681 foo = fo.read() 687 foo = fo.read()
682 fo.close() 688 fo.close()
683 lens.append(len(foo)) 689 lens.append(len(foo))
684 diff = time.time() - starttime 690 diff = time.time() - starttime
685 691
698 def debug(self, msg, *args): 704 def debug(self, msg, *args):
699 print(msg % args) 705 print(msg % args)
700 info = warning = error = debug 706 info = warning = error = debug
701 DEBUG = FakeLogger() 707 DEBUG = FakeLogger()
702 print(" fetching the file to establish a connection") 708 print(" fetching the file to establish a connection")
703 fo = urllib2.urlopen(url) 709 fo = urlreq.urlopen(url)
704 data1 = fo.read() 710 data1 = fo.read()
705 fo.close() 711 fo.close()
706 712
707 i = 20 713 i = 20
708 print(" waiting %i seconds for the server to close the connection" % i) 714 print(" waiting %i seconds for the server to close the connection" % i)
712 time.sleep(1) 718 time.sleep(1)
713 i -= 1 719 i -= 1
714 sys.stderr.write('\r') 720 sys.stderr.write('\r')
715 721
716 print(" fetching the file a second time") 722 print(" fetching the file a second time")
717 fo = urllib2.urlopen(url) 723 fo = urlreq.urlopen(url)
718 data2 = fo.read() 724 data2 = fo.read()
719 fo.close() 725 fo.close()
720 726
721 if data1 == data2: 727 if data1 == data2:
722 print(' data are identical') 728 print(' data are identical')