Mercurial > public > mercurial-scm > hg

comparison contrib/testparseutil.py @ 43076:2372284d9457

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
formatting: blacken the codebase This is using my patch to black (https://github.com/psf/black/pull/826) so we don't un-wrap collection literals. Done with: hg files 'set:**.py - mercurial/thirdparty/** - "contrib/python-zstandard/**"' | xargs black -S # skip-blame mass-reformatting only # no-check-commit reformats foo_bar functions Differential Revision: https://phab.mercurial-scm.org/D6971
author Augie Fackler <augie@google.com>
date Sun, 06 Oct 2019 09:45:02 -0400
parents c2deb2512823
children 127cc1f72e70
comparison
equal deleted inserted replaced
43075:57875cf423c9 43076:2372284d9457
12 import sys 12 import sys
13 13
14 #################### 14 ####################
15 # for Python3 compatibility (almost comes from mercurial/pycompat.py) 15 # for Python3 compatibility (almost comes from mercurial/pycompat.py)
16 16
17 ispy3 = (sys.version_info[0] >= 3) 17 ispy3 = sys.version_info[0] >= 3
18
18 19
19 def identity(a): 20 def identity(a):
20 return a 21 return a
22
21 23
22 def _rapply(f, xs): 24 def _rapply(f, xs):
23 if xs is None: 25 if xs is None:
24 # assume None means non-value of optional data 26 # assume None means non-value of optional data
25 return xs 27 return xs
27 return type(xs)(_rapply(f, x) for x in xs) 29 return type(xs)(_rapply(f, x) for x in xs)
28 if isinstance(xs, dict): 30 if isinstance(xs, dict):
29 return type(xs)((_rapply(f, k), _rapply(f, v)) for k, v in xs.items()) 31 return type(xs)((_rapply(f, k), _rapply(f, v)) for k, v in xs.items())
30 return f(xs) 32 return f(xs)
31 33
34
32 def rapply(f, xs): 35 def rapply(f, xs):
33 if f is identity: 36 if f is identity:
34 # fast path mainly for py2 37 # fast path mainly for py2
35 return xs 38 return xs
36 return _rapply(f, xs) 39 return _rapply(f, xs)
37 40
41
38 if ispy3: 42 if ispy3:
39 import builtins 43 import builtins
40 44
41 def bytestr(s): 45 def bytestr(s):
42 # tiny version of pycompat.bytestr 46 # tiny version of pycompat.bytestr
47 return s 51 return s
48 return s.decode(u'latin-1') 52 return s.decode(u'latin-1')
49 53
50 def opentext(f): 54 def opentext(f):
51 return open(f, 'r') 55 return open(f, 'r')
56
57
52 else: 58 else:
53 bytestr = str 59 bytestr = str
54 sysstr = identity 60 sysstr = identity
55 61
56 opentext = open 62 opentext = open
57 63
64
58 def b2s(x): 65 def b2s(x):
59 # convert BYTES elements in "x" to SYSSTR recursively 66 # convert BYTES elements in "x" to SYSSTR recursively
60 return rapply(sysstr, x) 67 return rapply(sysstr, x)
61 68
69
62 def writeout(data): 70 def writeout(data):
63 # write "data" in BYTES into stdout 71 # write "data" in BYTES into stdout
64 sys.stdout.write(data) 72 sys.stdout.write(data)
65 73
74
66 def writeerr(data): 75 def writeerr(data):
67 # write "data" in BYTES into stderr 76 # write "data" in BYTES into stderr
68 sys.stderr.write(data) 77 sys.stderr.write(data)
69 78
79
70 #################### 80 ####################
81
71 82
72 class embeddedmatcher(object): 83 class embeddedmatcher(object):
73 """Base class to detect embedded code fragments in *.t test script 84 """Base class to detect embedded code fragments in *.t test script
74 """ 85 """
86
75 __metaclass__ = abc.ABCMeta 87 __metaclass__ = abc.ABCMeta
76 88
77 def __init__(self, desc): 89 def __init__(self, desc):
78 self.desc = desc 90 self.desc = desc
79 91
123 """ 135 """
124 136
125 @abc.abstractmethod 137 @abc.abstractmethod
126 def codeinside(self, ctx, line): 138 def codeinside(self, ctx, line):
127 """Return actual code at line inside embedded code""" 139 """Return actual code at line inside embedded code"""
140
128 141
129 def embedded(basefile, lines, errors, matchers): 142 def embedded(basefile, lines, errors, matchers):
130 """pick embedded code fragments up from given lines 143 """pick embedded code fragments up from given lines
131 144
132 This is common parsing logic, which examines specified matchers on 145 This is common parsing logic, which examines specified matchers on
166 >>> b2s(errors) 179 >>> b2s(errors)
167 ['<dummy>:1: ambiguous line for "ambiguous #1", "ambiguous #2"'] 180 ['<dummy>:1: ambiguous line for "ambiguous #1", "ambiguous #2"']
168 181
169 """ 182 """
170 matcher = None 183 matcher = None
171 ctx = filename = code = startline = None # for pyflakes 184 ctx = filename = code = startline = None # for pyflakes
172 185
173 for lineno, line in enumerate(lines, 1): 186 for lineno, line in enumerate(lines, 1):
174 if not line.endswith('\n'): 187 if not line.endswith('\n'):
175 line += '\n' # to normalize EOF line 188 line += '\n' # to normalize EOF line
176 if matcher: # now, inside embedded code 189 if matcher: # now, inside embedded code
177 if matcher.endsat(ctx, line): 190 if matcher.endsat(ctx, line):
178 codeatend = matcher.codeatend(ctx, line) 191 codeatend = matcher.codeatend(ctx, line)
179 if codeatend is not None: 192 if codeatend is not None:
180 code.append(codeatend) 193 code.append(codeatend)
181 if not matcher.ignores(ctx): 194 if not matcher.ignores(ctx):
183 matcher = None 196 matcher = None
184 # DO NOT "continue", because line might start next fragment 197 # DO NOT "continue", because line might start next fragment
185 elif not matcher.isinside(ctx, line): 198 elif not matcher.isinside(ctx, line):
186 # this is an error of basefile 199 # this is an error of basefile
187 # (if matchers are implemented correctly) 200 # (if matchers are implemented correctly)
188 errors.append('%s:%d: unexpected line for "%s"' 201 errors.append(
189 % (basefile, lineno, matcher.desc)) 202 '%s:%d: unexpected line for "%s"'
203 % (basefile, lineno, matcher.desc)
204 )
190 # stop extracting embedded code by current 'matcher', 205 # stop extracting embedded code by current 'matcher',
191 # because appearance of unexpected line might mean 206 # because appearance of unexpected line might mean
192 # that expected end-of-embedded-code line might never 207 # that expected end-of-embedded-code line might never
193 # appear 208 # appear
194 matcher = None 209 matcher = None
206 if ctx: 221 if ctx:
207 matched.append((m, ctx)) 222 matched.append((m, ctx))
208 if matched: 223 if matched:
209 if len(matched) > 1: 224 if len(matched) > 1:
210 # this is an error of matchers, maybe 225 # this is an error of matchers, maybe
211 errors.append('%s:%d: ambiguous line for %s' % 226 errors.append(
212 (basefile, lineno, 227 '%s:%d: ambiguous line for %s'
213 ', '.join(['"%s"' % m.desc 228 % (
214 for m, c in matched]))) 229 basefile,
230 lineno,
231 ', '.join(['"%s"' % m.desc for m, c in matched]),
232 )
233 )
215 # omit extracting embedded code, because choosing 234 # omit extracting embedded code, because choosing
216 # arbitrary matcher from matched ones might fail to 235 # arbitrary matcher from matched ones might fail to
217 # detect the end of embedded code as expected. 236 # detect the end of embedded code as expected.
218 continue 237 continue
219 matcher, ctx = matched[0] 238 matcher, ctx = matched[0]
236 if not matcher.ignores(ctx): 255 if not matcher.ignores(ctx):
237 yield (filename, startline, lineno + 1, ''.join(code)) 256 yield (filename, startline, lineno + 1, ''.join(code))
238 else: 257 else:
239 # this is an error of basefile 258 # this is an error of basefile
240 # (if matchers are implemented correctly) 259 # (if matchers are implemented correctly)
241 errors.append('%s:%d: unexpected end of file for "%s"' 260 errors.append(
242 % (basefile, lineno, matcher.desc)) 261 '%s:%d: unexpected end of file for "%s"'
262 % (basefile, lineno, matcher.desc)
263 )
264
243 265
244 # heredoc limit mark to ignore embedded code at check-code.py or so 266 # heredoc limit mark to ignore embedded code at check-code.py or so
245 heredocignorelimit = 'NO_CHECK_EOF' 267 heredocignorelimit = 'NO_CHECK_EOF'
246 268
247 # the pattern to match against cases below, and to return a limit mark 269 # the pattern to match against cases below, and to return a limit mark
249 # 271 #
250 # - << LIMITMARK 272 # - << LIMITMARK
251 # - << "LIMITMARK" 273 # - << "LIMITMARK"
252 # - << 'LIMITMARK' 274 # - << 'LIMITMARK'
253 heredoclimitpat = r'\s*<<\s*(?P<lquote>["\']?)(?P<limit>\w+)(?P=lquote)' 275 heredoclimitpat = r'\s*<<\s*(?P<lquote>["\']?)(?P<limit>\w+)(?P=lquote)'
276
254 277
255 class fileheredocmatcher(embeddedmatcher): 278 class fileheredocmatcher(embeddedmatcher):
256 """Detect "cat > FILE << LIMIT" style embedded code 279 """Detect "cat > FILE << LIMIT" style embedded code
257 280
258 >>> matcher = fileheredocmatcher('heredoc .py file', r'[^<]+\\.py') 281 >>> matcher = fileheredocmatcher('heredoc .py file', r'[^<]+\\.py')
288 False 311 False
289 >>> ctx = matcher.startsat(' $ cat > file.py << NO_CHECK_EOF\\n') 312 >>> ctx = matcher.startsat(' $ cat > file.py << NO_CHECK_EOF\\n')
290 >>> matcher.ignores(ctx) 313 >>> matcher.ignores(ctx)
291 True 314 True
292 """ 315 """
316
293 _prefix = ' > ' 317 _prefix = ' > '
294 318
295 def __init__(self, desc, namepat): 319 def __init__(self, desc, namepat):
296 super(fileheredocmatcher, self).__init__(desc) 320 super(fileheredocmatcher, self).__init__(desc)
297 321
300 # group 324 # group
301 # 325 #
302 # - > NAMEPAT 326 # - > NAMEPAT
303 # - > "NAMEPAT" 327 # - > "NAMEPAT"
304 # - > 'NAMEPAT' 328 # - > 'NAMEPAT'
305 namepat = (r'\s*>>?\s*(?P<nquote>["\']?)(?P<name>%s)(?P=nquote)' 329 namepat = (
306 % namepat) 330 r'\s*>>?\s*(?P<nquote>["\']?)(?P<name>%s)(?P=nquote)' % namepat
331 )
307 self._fileres = [ 332 self._fileres = [
308 # "cat > NAME << LIMIT" case 333 # "cat > NAME << LIMIT" case
309 re.compile(r' \$ \s*cat' + namepat + heredoclimitpat), 334 re.compile(r' \$ \s*cat' + namepat + heredoclimitpat),
310 # "cat << LIMIT > NAME" case 335 # "cat << LIMIT > NAME" case
311 re.compile(r' \$ \s*cat' + heredoclimitpat + namepat), 336 re.compile(r' \$ \s*cat' + heredoclimitpat + namepat),
314 def startsat(self, line): 339 def startsat(self, line):
315 # ctx is (filename, END-LINE-OF-EMBEDDED-CODE) tuple 340 # ctx is (filename, END-LINE-OF-EMBEDDED-CODE) tuple
316 for filere in self._fileres: 341 for filere in self._fileres:
317 matched = filere.match(line) 342 matched = filere.match(line)
318 if matched: 343 if matched:
319 return (matched.group('name'), 344 return (
320 ' > %s\n' % matched.group('limit')) 345 matched.group('name'),
346 ' > %s\n' % matched.group('limit'),
347 )
321 348
322 def endsat(self, ctx, line): 349 def endsat(self, ctx, line):
323 return ctx[1] == line 350 return ctx[1] == line
324 351
325 def isinside(self, ctx, line): 352 def isinside(self, ctx, line):
330 357
331 def filename(self, ctx): 358 def filename(self, ctx):
332 return ctx[0] 359 return ctx[0]
333 360
334 def codeatstart(self, ctx, line): 361 def codeatstart(self, ctx, line):
335 return None # no embedded code at start line 362 return None # no embedded code at start line
336 363
337 def codeatend(self, ctx, line): 364 def codeatend(self, ctx, line):
338 return None # no embedded code at end line 365 return None # no embedded code at end line
339 366
340 def codeinside(self, ctx, line): 367 def codeinside(self, ctx, line):
341 return line[len(self._prefix):] # strip prefix 368 return line[len(self._prefix) :] # strip prefix
369
342 370
343 #### 371 ####
344 # for embedded python script 372 # for embedded python script
373
345 374
346 class pydoctestmatcher(embeddedmatcher): 375 class pydoctestmatcher(embeddedmatcher):
347 """Detect ">>> code" style embedded python code 376 """Detect ">>> code" style embedded python code
348 377
349 >>> matcher = pydoctestmatcher() 378 >>> matcher = pydoctestmatcher()
393 >>> end = '\\n' 422 >>> end = '\\n'
394 >>> matcher.endsat(ctx, end) 423 >>> matcher.endsat(ctx, end)
395 True 424 True
396 >>> matcher.codeatend(ctx, end) 425 >>> matcher.codeatend(ctx, end)
397 """ 426 """
427
398 _prefix = ' >>> ' 428 _prefix = ' >>> '
399 _prefixre = re.compile(r' (>>>|\.\.\.) ') 429 _prefixre = re.compile(r' (>>>|\.\.\.) ')
400 430
401 # If a line matches against not _prefixre but _outputre, that line 431 # If a line matches against not _prefixre but _outputre, that line
402 # is "an expected output line" (= not a part of code fragment). 432 # is "an expected output line" (= not a part of code fragment).
417 447
418 def endsat(self, ctx, line): 448 def endsat(self, ctx, line):
419 return not (self._prefixre.match(line) or self._outputre.match(line)) 449 return not (self._prefixre.match(line) or self._outputre.match(line))
420 450
421 def isinside(self, ctx, line): 451 def isinside(self, ctx, line):
422 return True # always true, if not yet ended 452 return True # always true, if not yet ended
423 453
424 def ignores(self, ctx): 454 def ignores(self, ctx):
425 return False # should be checked always 455 return False # should be checked always
426 456
427 def filename(self, ctx): 457 def filename(self, ctx):
428 return None # no filename 458 return None # no filename
429 459
430 def codeatstart(self, ctx, line): 460 def codeatstart(self, ctx, line):
431 return line[len(self._prefix):] # strip prefix ' >>> '/' ... ' 461 return line[len(self._prefix) :] # strip prefix ' >>> '/' ... '
432 462
433 def codeatend(self, ctx, line): 463 def codeatend(self, ctx, line):
434 return None # no embedded code at end line 464 return None # no embedded code at end line
435 465
436 def codeinside(self, ctx, line): 466 def codeinside(self, ctx, line):
437 if self._prefixre.match(line): 467 if self._prefixre.match(line):
438 return line[len(self._prefix):] # strip prefix ' >>> '/' ... ' 468 return line[len(self._prefix) :] # strip prefix ' >>> '/' ... '
439 return '\n' # an expected output line is treated as an empty line 469 return '\n' # an expected output line is treated as an empty line
470
440 471
441 class pyheredocmatcher(embeddedmatcher): 472 class pyheredocmatcher(embeddedmatcher):
442 """Detect "python << LIMIT" style embedded python code 473 """Detect "python << LIMIT" style embedded python code
443 474
444 >>> matcher = pyheredocmatcher() 475 >>> matcher = pyheredocmatcher()
472 False 503 False
473 >>> ctx = matcher.startsat(' $ python << NO_CHECK_EOF\\n') 504 >>> ctx = matcher.startsat(' $ python << NO_CHECK_EOF\\n')
474 >>> matcher.ignores(ctx) 505 >>> matcher.ignores(ctx)
475 True 506 True
476 """ 507 """
508
477 _prefix = ' > ' 509 _prefix = ' > '
478 510
479 _startre = re.compile(r' \$ (\$PYTHON|"\$PYTHON"|python).*' + 511 _startre = re.compile(
480 heredoclimitpat) 512 r' \$ (\$PYTHON|"\$PYTHON"|python).*' + heredoclimitpat
513 )
481 514
482 def __init__(self): 515 def __init__(self):
483 super(pyheredocmatcher, self).__init__("heredoc python invocation") 516 super(pyheredocmatcher, self).__init__("heredoc python invocation")
484 517
485 def startsat(self, line): 518 def startsat(self, line):
496 529
497 def ignores(self, ctx): 530 def ignores(self, ctx):
498 return ' > %s\n' % heredocignorelimit == ctx 531 return ' > %s\n' % heredocignorelimit == ctx
499 532
500 def filename(self, ctx): 533 def filename(self, ctx):
501 return None # no filename 534 return None # no filename
502 535
503 def codeatstart(self, ctx, line): 536 def codeatstart(self, ctx, line):
504 return None # no embedded code at start line 537 return None # no embedded code at start line
505 538
506 def codeatend(self, ctx, line): 539 def codeatend(self, ctx, line):
507 return None # no embedded code at end line 540 return None # no embedded code at end line
508 541
509 def codeinside(self, ctx, line): 542 def codeinside(self, ctx, line):
510 return line[len(self._prefix):] # strip prefix 543 return line[len(self._prefix) :] # strip prefix
544
511 545
512 _pymatchers = [ 546 _pymatchers = [
513 pydoctestmatcher(), 547 pydoctestmatcher(),
514 pyheredocmatcher(), 548 pyheredocmatcher(),
515 # use '[^<]+' instead of '\S+', in order to match against 549 # use '[^<]+' instead of '\S+', in order to match against
516 # paths including whitespaces 550 # paths including whitespaces
517 fileheredocmatcher('heredoc .py file', r'[^<]+\.py'), 551 fileheredocmatcher('heredoc .py file', r'[^<]+\.py'),
518 ] 552 ]
519 553
554
520 def pyembedded(basefile, lines, errors): 555 def pyembedded(basefile, lines, errors):
521 return embedded(basefile, lines, errors, _pymatchers) 556 return embedded(basefile, lines, errors, _pymatchers)
557
522 558
523 #### 559 ####
524 # for embedded shell script 560 # for embedded shell script
525 561
526 _shmatchers = [ 562 _shmatchers = [
527 # use '[^<]+' instead of '\S+', in order to match against 563 # use '[^<]+' instead of '\S+', in order to match against
528 # paths including whitespaces 564 # paths including whitespaces
529 fileheredocmatcher('heredoc .sh file', r'[^<]+\.sh'), 565 fileheredocmatcher('heredoc .sh file', r'[^<]+\.sh'),
530 ] 566 ]
531 567
568
532 def shembedded(basefile, lines, errors): 569 def shembedded(basefile, lines, errors):
533 return embedded(basefile, lines, errors, _shmatchers) 570 return embedded(basefile, lines, errors, _shmatchers)
571
534 572
535 #### 573 ####
536 # for embedded hgrc configuration 574 # for embedded hgrc configuration
537 575
538 _hgrcmatchers = [ 576 _hgrcmatchers = [
539 # use '[^<]+' instead of '\S+', in order to match against 577 # use '[^<]+' instead of '\S+', in order to match against
540 # paths including whitespaces 578 # paths including whitespaces
541 fileheredocmatcher('heredoc hgrc file', 579 fileheredocmatcher(
542 r'(([^/<]+/)+hgrc|\$HGRCPATH|\${HGRCPATH})'), 580 'heredoc hgrc file', r'(([^/<]+/)+hgrc|\$HGRCPATH|\${HGRCPATH})'
581 ),
543 ] 582 ]
583
544 584
545 def hgrcembedded(basefile, lines, errors): 585 def hgrcembedded(basefile, lines, errors):
546 return embedded(basefile, lines, errors, _hgrcmatchers) 586 return embedded(basefile, lines, errors, _hgrcmatchers)
587
547 588
548 #### 589 ####
549 590
550 if __name__ == "__main__": 591 if __name__ == "__main__":
551 import optparse 592 import optparse
556 for name, starts, ends, code in embeddedfunc(basefile, lines, errors): 597 for name, starts, ends, code in embeddedfunc(basefile, lines, errors):
557 if not name: 598 if not name:
558 name = '<anonymous>' 599 name = '<anonymous>'
559 writeout("%s:%d: %s starts\n" % (basefile, starts, name)) 600 writeout("%s:%d: %s starts\n" % (basefile, starts, name))
560 if opts.verbose and code: 601 if opts.verbose and code:
561 writeout(" |%s\n" % 602 writeout(" |%s\n" % "\n |".join(l for l in code.splitlines()))
562 "\n |".join(l for l in code.splitlines()))
563 writeout("%s:%d: %s ends\n" % (basefile, ends, name)) 603 writeout("%s:%d: %s ends\n" % (basefile, ends, name))
564 for e in errors: 604 for e in errors:
565 writeerr("%s\n" % e) 605 writeerr("%s\n" % e)
566 return len(errors) 606 return len(errors)
567 607
577 if showembedded('<stdin>', lines, embeddedfunc, opts): 617 if showembedded('<stdin>', lines, embeddedfunc, opts):
578 ret = 1 618 ret = 1
579 return ret 619 return ret
580 620
581 commands = {} 621 commands = {}
622
582 def command(name, desc): 623 def command(name, desc):
583 def wrap(func): 624 def wrap(func):
584 commands[name] = (desc, func) 625 commands[name] = (desc, func)
626
585 return wrap 627 return wrap
586 628
587 @command("pyembedded", "detect embedded python script") 629 @command("pyembedded", "detect embedded python script")
588 def pyembeddedcmd(args, opts): 630 def pyembeddedcmd(args, opts):
589 return applyembedded(args, pyembedded, opts) 631 return applyembedded(args, pyembedded, opts)
594 636
595 @command("hgrcembedded", "detect embedded hgrc configuration") 637 @command("hgrcembedded", "detect embedded hgrc configuration")
596 def hgrcembeddedcmd(args, opts): 638 def hgrcembeddedcmd(args, opts):
597 return applyembedded(args, hgrcembedded, opts) 639 return applyembedded(args, hgrcembedded, opts)
598 640
599 availablecommands = "\n".join([" - %s: %s" % (key, value[0]) 641 availablecommands = "\n".join(
600 for key, value in commands.items()]) 642 [" - %s: %s" % (key, value[0]) for key, value in commands.items()]
601 643 )
602 parser = optparse.OptionParser("""%prog COMMAND [file ...] 644
645 parser = optparse.OptionParser(
646 """%prog COMMAND [file ...]
603 647
604 Pick up embedded code fragments from given file(s) or stdin, and list 648 Pick up embedded code fragments from given file(s) or stdin, and list
605 up start/end lines of them in standard compiler format 649 up start/end lines of them in standard compiler format
606 ("FILENAME:LINENO:"). 650 ("FILENAME:LINENO:").
607 651
608 Available commands are: 652 Available commands are:
609 """ + availablecommands + """ 653 """
610 """) 654 + availablecommands
611 parser.add_option("-v", "--verbose", 655 + """
612 help="enable additional output (e.g. actual code)", 656 """
613 action="store_true") 657 )
658 parser.add_option(
659 "-v",
660 "--verbose",
661 help="enable additional output (e.g. actual code)",
662 action="store_true",
663 )
614 (opts, args) = parser.parse_args() 664 (opts, args) = parser.parse_args()
615 665
616 if not args or args[0] not in commands: 666 if not args or args[0] not in commands:
617 parser.print_help() 667 parser.print_help()
618 sys.exit(255) 668 sys.exit(255)