Mercurial > public > mercurial-scm > hg

comparison mercurial/statprof.py @ 43077:687b865b95ad

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
formatting: byteify all mercurial/ and hgext/ string literals Done with python3.7 contrib/byteify-strings.py -i $(hg files 'set:mercurial/**.py - mercurial/thirdparty/** + hgext/**.py - hgext/fsmonitor/pywatchman/** - mercurial/__init__.py') black -l 80 -t py33 -S $(hg files 'set:**.py - mercurial/thirdparty/** - "contrib/python-zstandard/**" - hgext/fsmonitor/pywatchman/**') # skip-blame mass-reformatting only Differential Revision: https://phab.mercurial-scm.org/D6972
author Augie Fackler <augie@google.com>
date Sun, 06 Oct 2019 09:48:39 -0400
parents 2372284d9457
children eef9a2d67051
comparison
equal deleted inserted replaced
43076:2372284d9457 43077:687b865b95ad
120 ) 120 )
121 121
122 defaultdict = collections.defaultdict 122 defaultdict = collections.defaultdict
123 contextmanager = contextlib.contextmanager 123 contextmanager = contextlib.contextmanager
124 124
125 __all__ = ['start', 'stop', 'reset', 'display', 'profile'] 125 __all__ = [b'start', b'stop', b'reset', b'display', b'profile']
126 126
127 skips = { 127 skips = {
128 r"util.py:check", 128 r"util.py:check",
129 r"extensions.py:closure", 129 r"extensions.py:closure",
130 r"color.py:colorcmd", 130 r"color.py:colorcmd",
155 155
156 156
157 class ProfileState(object): 157 class ProfileState(object):
158 def __init__(self, frequency=None): 158 def __init__(self, frequency=None):
159 self.reset(frequency) 159 self.reset(frequency)
160 self.track = 'cpu' 160 self.track = b'cpu'
161 161
162 def reset(self, frequency=None): 162 def reset(self, frequency=None):
163 # total so far 163 # total so far
164 self.accumulated_time = (0.0, 0.0) 164 self.accumulated_time = (0.0, 0.0)
165 # start_time when timer is active 165 # start_time when timer is active
192 def seconds_per_sample(self): 192 def seconds_per_sample(self):
193 return self.accumulated_time[self.timeidx] / len(self.samples) 193 return self.accumulated_time[self.timeidx] / len(self.samples)
194 194
195 @property 195 @property
196 def timeidx(self): 196 def timeidx(self):
197 if self.track == 'real': 197 if self.track == b'real':
198 return 1 198 return 1
199 return 0 199 return 0
200 200
201 201
202 state = ProfileState() 202 state = ProfileState()
236 236
237 def getsource(self, length): 237 def getsource(self, length):
238 if self.source is None: 238 if self.source is None:
239 lineno = self.lineno - 1 239 lineno = self.lineno - 1
240 try: 240 try:
241 with open(self.path, 'rb') as fp: 241 with open(self.path, b'rb') as fp:
242 for i, line in enumerate(fp): 242 for i, line in enumerate(fp):
243 if i == lineno: 243 if i == lineno:
244 self.source = line.strip() 244 self.source = line.strip()
245 break 245 break
246 except: 246 except:
247 pass 247 pass
248 if self.source is None: 248 if self.source is None:
249 self.source = '' 249 self.source = b''
250 250
251 source = self.source 251 source = self.source
252 if len(source) > length: 252 if len(source) > length:
253 source = source[: (length - 3)] + "..." 253 source = source[: (length - 3)] + b"..."
254 return source 254 return source
255 255
256 def filename(self): 256 def filename(self):
257 return os.path.basename(self.path) 257 return os.path.basename(self.path)
258 258
328 328
329 329
330 lastmechanism = None 330 lastmechanism = None
331 331
332 332
333 def start(mechanism='thread', track='cpu'): 333 def start(mechanism=b'thread', track=b'cpu'):
334 '''Install the profiling signal handler, and start profiling.''' 334 '''Install the profiling signal handler, and start profiling.'''
335 state.track = track # note: nesting different mode won't work 335 state.track = track # note: nesting different mode won't work
336 state.profile_level += 1 336 state.profile_level += 1
337 if state.profile_level == 1: 337 if state.profile_level == 1:
338 state.last_start_time = clock() 338 state.last_start_time = clock()
340 state.remaining_prof_time = None 340 state.remaining_prof_time = None
341 341
342 global lastmechanism 342 global lastmechanism
343 lastmechanism = mechanism 343 lastmechanism = mechanism
344 344
345 if mechanism == 'signal': 345 if mechanism == b'signal':
346 signal.signal(signal.SIGPROF, profile_signal_handler) 346 signal.signal(signal.SIGPROF, profile_signal_handler)
347 signal.setitimer( 347 signal.setitimer(
348 signal.ITIMER_PROF, rpt or state.sample_interval, 0.0 348 signal.ITIMER_PROF, rpt or state.sample_interval, 0.0
349 ) 349 )
350 elif mechanism == 'thread': 350 elif mechanism == b'thread':
351 frame = inspect.currentframe() 351 frame = inspect.currentframe()
352 tid = [k for k, f in sys._current_frames().items() if f == frame][0] 352 tid = [k for k, f in sys._current_frames().items() if f == frame][0]
353 state.thread = threading.Thread( 353 state.thread = threading.Thread(
354 target=samplerthread, args=(tid,), name="samplerthread" 354 target=samplerthread, args=(tid,), name=b"samplerthread"
355 ) 355 )
356 state.thread.start() 356 state.thread.start()
357 357
358 358
359 def stop(): 359 def stop():
360 '''Stop profiling, and uninstall the profiling signal handler.''' 360 '''Stop profiling, and uninstall the profiling signal handler.'''
361 state.profile_level -= 1 361 state.profile_level -= 1
362 if state.profile_level == 0: 362 if state.profile_level == 0:
363 if lastmechanism == 'signal': 363 if lastmechanism == b'signal':
364 rpt = signal.setitimer(signal.ITIMER_PROF, 0.0, 0.0) 364 rpt = signal.setitimer(signal.ITIMER_PROF, 0.0, 0.0)
365 signal.signal(signal.SIGPROF, signal.SIG_IGN) 365 signal.signal(signal.SIGPROF, signal.SIG_IGN)
366 state.remaining_prof_time = rpt[0] 366 state.remaining_prof_time = rpt[0]
367 elif lastmechanism == 'thread': 367 elif lastmechanism == b'thread':
368 stopthread.set() 368 stopthread.set()
369 state.thread.join() 369 state.thread.join()
370 370
371 state.accumulate_time(clock()) 371 state.accumulate_time(clock())
372 state.last_start_time = None 372 state.last_start_time = None
373 statprofpath = encoding.environ.get('STATPROF_DEST') 373 statprofpath = encoding.environ.get(b'STATPROF_DEST')
374 if statprofpath: 374 if statprofpath:
375 save_data(statprofpath) 375 save_data(statprofpath)
376 376
377 return state 377 return state
378 378
379 379
380 def save_data(path): 380 def save_data(path):
381 with open(path, 'w+') as file: 381 with open(path, b'w+') as file:
382 file.write("%f %f\n" % state.accumulated_time) 382 file.write(b"%f %f\n" % state.accumulated_time)
383 for sample in state.samples: 383 for sample in state.samples:
384 time = sample.time 384 time = sample.time
385 stack = sample.stack 385 stack = sample.stack
386 sites = [ 386 sites = [
387 '\1'.join([s.path, b'%d' % s.lineno, s.function]) for s in stack 387 b'\1'.join([s.path, b'%d' % s.lineno, s.function])
388 for s in stack
388 ] 389 ]
389 file.write("%d\0%s\n" % (time, '\0'.join(sites))) 390 file.write(b"%d\0%s\n" % (time, b'\0'.join(sites)))
390 391
391 392
392 def load_data(path): 393 def load_data(path):
393 lines = open(path, 'rb').read().splitlines() 394 lines = open(path, b'rb').read().splitlines()
394 395
395 state.accumulated_time = [float(value) for value in lines[0].split()] 396 state.accumulated_time = [float(value) for value in lines[0].split()]
396 state.samples = [] 397 state.samples = []
397 for line in lines[1:]: 398 for line in lines[1:]:
398 parts = line.split('\0') 399 parts = line.split(b'\0')
399 time = float(parts[0]) 400 time = float(parts[0])
400 rawsites = parts[1:] 401 rawsites = parts[1:]
401 sites = [] 402 sites = []
402 for rawsite in rawsites: 403 for rawsite in rawsites:
403 siteparts = rawsite.split('\1') 404 siteparts = rawsite.split(b'\1')
404 sites.append( 405 sites.append(
405 CodeSite.get(siteparts[0], int(siteparts[1]), siteparts[2]) 406 CodeSite.get(siteparts[0], int(siteparts[1]), siteparts[2])
406 ) 407 )
407 408
408 state.samples.append(Sample(sites, time)) 409 state.samples.append(Sample(sites, time))
412 '''Clear out the state of the profiler. Do not call while the 413 '''Clear out the state of the profiler. Do not call while the
413 profiler is running. 414 profiler is running.
414 415
415 The optional frequency argument specifies the number of samples to 416 The optional frequency argument specifies the number of samples to
416 collect per second.''' 417 collect per second.'''
417 assert state.profile_level == 0, "Can't reset() while statprof is running" 418 assert state.profile_level == 0, b"Can't reset() while statprof is running"
418 CodeSite.cache.clear() 419 CodeSite.cache.clear()
419 state.reset(frequency) 420 state.reset(frequency)
420 421
421 422
422 @contextmanager 423 @contextmanager
512 elif format == DisplayFormats.Json: 513 elif format == DisplayFormats.Json:
513 write_to_json(data, fp) 514 write_to_json(data, fp)
514 elif format == DisplayFormats.Chrome: 515 elif format == DisplayFormats.Chrome:
515 write_to_chrome(data, fp, **kwargs) 516 write_to_chrome(data, fp, **kwargs)
516 else: 517 else:
517 raise Exception("Invalid display format") 518 raise Exception(b"Invalid display format")
518 519
519 if format not in (DisplayFormats.Json, DisplayFormats.Chrome): 520 if format not in (DisplayFormats.Json, DisplayFormats.Chrome):
520 fp.write(b'---\n') 521 fp.write(b'---\n')
521 fp.write(b'Sample count: %d\n' % len(data.samples)) 522 fp.write(b'Sample count: %d\n' % len(data.samples))
522 fp.write(b'Total time: %f seconds (%f wall)\n' % data.accumulated_time) 523 fp.write(b'Total time: %f seconds (%f wall)\n' % data.accumulated_time)
537 % (b"time", b"seconds", b"seconds", b"name") 538 % (b"time", b"seconds", b"seconds", b"name")
538 ) 539 )
539 540
540 for stat in stats: 541 for stat in stats:
541 site = stat.site 542 site = stat.site
542 sitelabel = '%s:%d:%s' % (site.filename(), site.lineno, site.function) 543 sitelabel = b'%s:%d:%s' % (site.filename(), site.lineno, site.function)
543 fp.write( 544 fp.write(
544 b'%6.2f %9.2f %9.2f %s\n' 545 b'%6.2f %9.2f %9.2f %s\n'
545 % ( 546 % (
546 stat.selfpercent(), 547 stat.selfpercent(),
547 stat.totalseconds(), 548 stat.totalseconds(),
554 def display_by_method(data, fp): 555 def display_by_method(data, fp):
555 '''Print the profiler data with each sample function represented 556 '''Print the profiler data with each sample function represented
556 as one row in a table. Important lines within that function are 557 as one row in a table. Important lines within that function are
557 output as nested rows. Sorted by self-time per line.''' 558 output as nested rows. Sorted by self-time per line.'''
558 fp.write( 559 fp.write(
559 b'%5.5s %10.10s %7.7s %-8.8s\n' % ('% ', 'cumulative', 'self', '') 560 b'%5.5s %10.10s %7.7s %-8.8s\n'
561 % (b'% ', b'cumulative', b'self', b'')
560 ) 562 )
561 fp.write( 563 fp.write(
562 b'%5.5s %9.9s %8.8s %-8.8s\n' 564 b'%5.5s %9.9s %8.8s %-8.8s\n'
563 % ("time", "seconds", "seconds", "name") 565 % (b"time", b"seconds", b"seconds", b"name")
564 ) 566 )
565 567
566 stats = SiteStats.buildstats(data.samples) 568 stats = SiteStats.buildstats(data.samples)
567 569
568 grouped = defaultdict(list) 570 grouped = defaultdict(list)
620 fp.write(b'%33.0f%% %6.2f line %d: %s\n' % stattuple) 622 fp.write(b'%33.0f%% %6.2f line %d: %s\n' % stattuple)
621 623
622 624
623 def display_about_method(data, fp, function=None, **kwargs): 625 def display_about_method(data, fp, function=None, **kwargs):
624 if function is None: 626 if function is None:
625 raise Exception("Invalid function") 627 raise Exception(b"Invalid function")
626 628
627 filename = None 629 filename = None
628 if ':' in function: 630 if b':' in function:
629 filename, function = function.split(':') 631 filename, function = function.split(b':')
630 632
631 relevant_samples = 0 633 relevant_samples = 0
632 parents = {} 634 parents = {}
633 children = {} 635 children = {}
634 636
683 total_cum_percent += stat.totalpercent() 685 total_cum_percent += stat.totalpercent()
684 686
685 fp.write( 687 fp.write(
686 b'\n %s:%s Total: %0.2fs (%0.2f%%) Self: %0.2fs (%0.2f%%)\n\n' 688 b'\n %s:%s Total: %0.2fs (%0.2f%%) Self: %0.2fs (%0.2f%%)\n\n'
687 % ( 689 % (
688 pycompat.sysbytes(filename or '___'), 690 pycompat.sysbytes(filename or b'___'),
689 pycompat.sysbytes(function), 691 pycompat.sysbytes(function),
690 total_cum_sec, 692 total_cum_sec,
691 total_cum_percent, 693 total_cum_percent,
692 total_self_sec, 694 total_self_sec,
693 total_self_percent, 695 total_self_percent,
744 for c in node.children.itervalues() 746 for c in node.children.itervalues()
745 if c.count >= (limit * root.count) 747 if c.count >= (limit * root.count)
746 ] 748 ]
747 if site: 749 if site:
748 indent = depth * 2 - 1 750 indent = depth * 2 - 1
749 filename = '' 751 filename = b''
750 function = '' 752 function = b''
751 if len(node.children) > 0: 753 if len(node.children) > 0:
752 childsite = list(node.children.itervalues())[0].site 754 childsite = list(node.children.itervalues())[0].site
753 filename = (childsite.filename() + ':').ljust(15) 755 filename = (childsite.filename() + b':').ljust(15)
754 function = childsite.function 756 function = childsite.function
755 757
756 # lots of string formatting 758 # lots of string formatting
757 listpattern = ( 759 listpattern = (
758 ''.ljust(indent) 760 b''.ljust(indent)
759 + ('\\' if multiple_siblings else '|') 761 + (b'\\' if multiple_siblings else b'|')
760 + ' %4.1f%%' 762 + b' %4.1f%%'
761 + (' %5.2fs' % node.count if showtime else '') 763 + (b' %5.2fs' % node.count if showtime else b'')
762 + ' %s %s' 764 + b' %s %s'
763 ) 765 )
764 liststring = listpattern % ( 766 liststring = listpattern % (
765 node.count / root.count * 100, 767 node.count / root.count * 100,
766 filename, 768 filename,
767 function, 769 function,
768 ) 770 )
769 codepattern = '%' + ('%d' % (55 - len(liststring))) + 's %d: %s' 771 codepattern = b'%' + (b'%d' % (55 - len(liststring))) + b's %d: %s'
770 codestring = codepattern % ('line', site.lineno, site.getsource(30)) 772 codestring = codepattern % (
773 b'line',
774 site.lineno,
775 site.getsource(30),
776 )
771 777
772 finalstring = liststring + codestring 778 finalstring = liststring + codestring
773 childrensamples = sum([c.count for c in node.children.itervalues()]) 779 childrensamples = sum([c.count for c in node.children.itervalues()])
774 # Make frames that performed more than 10% of the operation red 780 # Make frames that performed more than 10% of the operation red
775 if node.count - childrensamples > (0.1 * root.count): 781 if node.count - childrensamples > (0.1 * root.count):
776 finalstring = '\033[91m' + finalstring + '\033[0m' 782 finalstring = b'\033[91m' + finalstring + b'\033[0m'
777 # Make frames that didn't actually perform work dark grey 783 # Make frames that didn't actually perform work dark grey
778 elif node.count - childrensamples == 0: 784 elif node.count - childrensamples == 0:
779 finalstring = '\033[90m' + finalstring + '\033[0m' 785 finalstring = b'\033[90m' + finalstring + b'\033[0m'
780 fp.write(finalstring + b'\n') 786 fp.write(finalstring + b'\n')
781 787
782 newdepth = depth 788 newdepth = depth
783 if len(visiblechildren) > 1 or multiple_siblings: 789 if len(visiblechildren) > 1 or multiple_siblings:
784 newdepth += 1 790 newdepth += 1
791 _write(root, 0, False) 797 _write(root, 0, False)
792 798
793 799
794 def write_to_flame(data, fp, scriptpath=None, outputfile=None, **kwargs): 800 def write_to_flame(data, fp, scriptpath=None, outputfile=None, **kwargs):
795 if scriptpath is None: 801 if scriptpath is None:
796 scriptpath = encoding.environ['HOME'] + '/flamegraph.pl' 802 scriptpath = encoding.environ[b'HOME'] + b'/flamegraph.pl'
797 if not os.path.exists(scriptpath): 803 if not os.path.exists(scriptpath):
798 fp.write(b'error: missing %s\n' % scriptpath) 804 fp.write(b'error: missing %s\n' % scriptpath)
799 fp.write(b'get it here: https://github.com/brendangregg/FlameGraph\n') 805 fp.write(b'get it here: https://github.com/brendangregg/FlameGraph\n')
800 return 806 return
801 807
802 lines = {} 808 lines = {}
803 for sample in data.samples: 809 for sample in data.samples:
804 sites = [s.function for s in sample.stack] 810 sites = [s.function for s in sample.stack]
805 sites.reverse() 811 sites.reverse()
806 line = ';'.join(sites) 812 line = b';'.join(sites)
807 if line in lines: 813 if line in lines:
808 lines[line] = lines[line] + 1 814 lines[line] = lines[line] + 1
809 else: 815 else:
810 lines[line] = 1 816 lines[line] = 1
811 817
812 fd, path = pycompat.mkstemp() 818 fd, path = pycompat.mkstemp()
813 819
814 with open(path, "w+") as file: 820 with open(path, b"w+") as file:
815 for line, count in lines.iteritems(): 821 for line, count in lines.iteritems():
816 file.write("%s %d\n" % (line, count)) 822 file.write(b"%s %d\n" % (line, count))
817 823
818 if outputfile is None: 824 if outputfile is None:
819 outputfile = '~/flamegraph.svg' 825 outputfile = b'~/flamegraph.svg'
820 826
821 os.system("perl ~/flamegraph.pl %s > %s" % (path, outputfile)) 827 os.system(b"perl ~/flamegraph.pl %s > %s" % (path, outputfile))
822 fp.write(b'Written to %s\n' % outputfile) 828 fp.write(b'Written to %s\n' % outputfile)
823 829
824 830
825 _pathcache = {} 831 _pathcache = {}
826 832
981 ) 987 )
982 data = json.dumps(dict(traceEvents=events, stackFrames=frames), indent=1) 988 data = json.dumps(dict(traceEvents=events, stackFrames=frames), indent=1)
983 if not isinstance(data, bytes): 989 if not isinstance(data, bytes):
984 data = data.encode('utf-8') 990 data = data.encode('utf-8')
985 fp.write(data) 991 fp.write(data)
986 fp.write('\n') 992 fp.write(b'\n')
987 993
988 994
989 def printusage(): 995 def printusage():
990 print( 996 print(
991 r""" 997 r"""
1018 return 0 1024 return 0
1019 1025
1020 displayargs = {} 1026 displayargs = {}
1021 1027
1022 optstart = 2 1028 optstart = 2
1023 displayargs['function'] = None 1029 displayargs[b'function'] = None
1024 if argv[1] == r'hotpath': 1030 if argv[1] == r'hotpath':
1025 displayargs['format'] = DisplayFormats.Hotpath 1031 displayargs[b'format'] = DisplayFormats.Hotpath
1026 elif argv[1] == r'lines': 1032 elif argv[1] == r'lines':
1027 displayargs['format'] = DisplayFormats.ByLine 1033 displayargs[b'format'] = DisplayFormats.ByLine
1028 elif argv[1] == r'functions': 1034 elif argv[1] == r'functions':
1029 displayargs['format'] = DisplayFormats.ByMethod 1035 displayargs[b'format'] = DisplayFormats.ByMethod
1030 elif argv[1] == r'function': 1036 elif argv[1] == r'function':
1031 displayargs['format'] = DisplayFormats.AboutMethod 1037 displayargs[b'format'] = DisplayFormats.AboutMethod
1032 displayargs['function'] = argv[2] 1038 displayargs[b'function'] = argv[2]
1033 optstart = 3 1039 optstart = 3
1034 elif argv[1] == r'flame': 1040 elif argv[1] == r'flame':
1035 displayargs['format'] = DisplayFormats.FlameGraph 1041 displayargs[b'format'] = DisplayFormats.FlameGraph
1036 else: 1042 else:
1037 printusage() 1043 printusage()
1038 return 0 1044 return 0
1039 1045
1040 # process options 1046 # process options
1041 try: 1047 try:
1042 opts, args = pycompat.getoptb( 1048 opts, args = pycompat.getoptb(
1043 sys.argv[optstart:], 1049 sys.argv[optstart:],
1044 "hl:f:o:p:", 1050 b"hl:f:o:p:",
1045 ["help", "limit=", "file=", "output-file=", "script-path="], 1051 [b"help", b"limit=", b"file=", b"output-file=", b"script-path="],
1046 ) 1052 )
1047 except getopt.error as msg: 1053 except getopt.error as msg:
1048 print(msg) 1054 print(msg)
1049 printusage() 1055 printusage()
1050 return 2 1056 return 2
1051 1057
1052 displayargs['limit'] = 0.05 1058 displayargs[b'limit'] = 0.05
1053 path = None 1059 path = None
1054 for o, value in opts: 1060 for o, value in opts:
1055 if o in (r"-l", r"--limit"): 1061 if o in (r"-l", r"--limit"):
1056 displayargs['limit'] = float(value) 1062 displayargs[b'limit'] = float(value)
1057 elif o in (r"-f", r"--file"): 1063 elif o in (r"-f", r"--file"):
1058 path = value 1064 path = value
1059 elif o in (r"-o", r"--output-file"): 1065 elif o in (r"-o", r"--output-file"):
1060 displayargs['outputfile'] = value 1066 displayargs[b'outputfile'] = value
1061 elif o in (r"-p", r"--script-path"): 1067 elif o in (r"-p", r"--script-path"):
1062 displayargs['scriptpath'] = value 1068 displayargs[b'scriptpath'] = value
1063 elif o in (r"-h", r"help"): 1069 elif o in (r"-h", r"help"):
1064 printusage() 1070 printusage()
1065 return 0 1071 return 0
1066 else: 1072 else:
1067 assert False, "unhandled option %s" % o 1073 assert False, b"unhandled option %s" % o
1068 1074
1069 if not path: 1075 if not path:
1070 print(r'must specify --file to load') 1076 print(r'must specify --file to load')
1071 return 1 1077 return 1
1072 1078