Mercurial > public > mercurial-scm > hg-stable

comparison mercurial/revsetlang.py @ 31450:80c8a6db450d

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
py3: use bytestr wrapper in revsetlang.tokenize() This backs out 77270ec0cdd9 and wraps program by bytestr() instead.
author Yuya Nishihara <yuya@tcha.org>
date Thu, 16 Mar 2017 21:36:21 +0900
parents f784ba187089
children 553ad16b274f
comparison
equal deleted inserted replaced
31449:f784ba187089 31450:80c8a6db450d
76 Check that @ is a valid unquoted token character (issue3686): 76 Check that @ is a valid unquoted token character (issue3686):
77 >>> list(tokenize("@::")) 77 >>> list(tokenize("@::"))
78 [('symbol', '@', 0), ('::', None, 1), ('end', None, 3)] 78 [('symbol', '@', 0), ('::', None, 1), ('end', None, 3)]
79 79
80 ''' 80 '''
81 program = pycompat.bytestr(program)
81 if syminitletters is None: 82 if syminitletters is None:
82 syminitletters = _syminitletters 83 syminitletters = _syminitletters
83 if symletters is None: 84 if symletters is None:
84 symletters = _symletters 85 symletters = _symletters
85 86
98 yield ('end', None, len(program)) 99 yield ('end', None, len(program))
99 return 100 return
100 101
101 pos, l = 0, len(program) 102 pos, l = 0, len(program)
102 while pos < l: 103 while pos < l:
103 c = program[pos:pos + 1] 104 c = program[pos]
104 if c.isspace(): # skip inter-token whitespace 105 if c.isspace(): # skip inter-token whitespace
105 pass 106 pass
106 elif c == ':' and program[pos:pos + 2] == '::': # look ahead carefully 107 elif c == ':' and program[pos:pos + 2] == '::': # look ahead carefully
107 yield ('::', None, pos) 108 yield ('::', None, pos)
108 pos += 1 # skip ahead 109 pos += 1 # skip ahead
116 yield (c, None, pos) 117 yield (c, None, pos)
117 elif (c in _quoteletters or c == 'r' and 118 elif (c in _quoteletters or c == 'r' and
118 program[pos:pos + 2] in ("r'", 'r"')): # handle quoted strings 119 program[pos:pos + 2] in ("r'", 'r"')): # handle quoted strings
119 if c == 'r': 120 if c == 'r':
120 pos += 1 121 pos += 1
121 c = program[pos:pos + 1] 122 c = program[pos]
122 decode = lambda x: x 123 decode = lambda x: x
123 else: 124 else:
124 decode = parser.unescapestr 125 decode = parser.unescapestr
125 pos += 1 126 pos += 1
126 s = pos 127 s = pos
127 while pos < l: # find closing quote 128 while pos < l: # find closing quote
128 d = program[pos:pos + 1] 129 d = program[pos]
129 if d == '\\': # skip over escaped characters 130 if d == '\\': # skip over escaped characters
130 pos += 2 131 pos += 2
131 continue 132 continue
132 if d == c: 133 if d == c:
133 yield ('string', decode(program[s:pos]), s) 134 yield ('string', decode(program[s:pos]), s)
138 # gather up a symbol/keyword 139 # gather up a symbol/keyword
139 elif c in syminitletters: 140 elif c in syminitletters:
140 s = pos 141 s = pos
141 pos += 1 142 pos += 1
142 while pos < l: # find end of symbol 143 while pos < l: # find end of symbol
143 d = program[pos:pos + 1] 144 d = program[pos]
144 if d not in symletters: 145 if d not in symletters:
145 break 146 break
146 if (d == '.' 147 if d == '.' and program[pos - 1] == '.': # special case for ..
147 and program[pos - 1:pos] == '.'): # special case for ..
148 pos -= 1 148 pos -= 1
149 break 149 break
150 pos += 1 150 pos += 1
151 sym = program[s:pos] 151 sym = program[s:pos]
152 if sym in keywords: # operator keywords 152 if sym in keywords: # operator keywords