Mercurial > public > mercurial-scm > hg

comparison contrib/byteify-strings.py @ 38388:f701bc936e7f

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
byteify-strings: do not rewrite iteritems() and itervalues() by default We can't do that automatically due to performance concerns.
author Yuya Nishihara <yuya@tcha.org>
date Thu, 31 May 2018 22:34:23 +0900
parents b704da9a9dda
children 1d68fd5f614a
comparison
equal deleted inserted replaced
38387:b704da9a9dda 38388:f701bc936e7f
17 import tempfile 17 import tempfile
18 import token 18 import token
19 import tokenize 19 import tokenize
20 20
21 if True: 21 if True:
22 def replacetokens(tokens): 22 def replacetokens(tokens, opts):
23 """Transform a stream of tokens from raw to Python 3. 23 """Transform a stream of tokens from raw to Python 3.
24 24
25 Returns a generator of possibly rewritten tokens. 25 Returns a generator of possibly rewritten tokens.
26 26
27 The input token list may be mutated as part of processing. However, 27 The input token list may be mutated as part of processing. However,
127 if argidx is not None: 127 if argidx is not None:
128 _ensureunicode(argidx) 128 _ensureunicode(argidx)
129 129
130 # It changes iteritems/values to items/values as they are not 130 # It changes iteritems/values to items/values as they are not
131 # present in Python 3 world. 131 # present in Python 3 world.
132 elif fn in ('iteritems', 'itervalues'): 132 elif opts['dictiter'] and fn in ('iteritems', 'itervalues'):
133 yield t._replace(string=fn[4:]) 133 yield t._replace(string=fn[4:])
134 continue 134 continue
135 135
136 # Emit unmodified token. 136 # Emit unmodified token.
137 yield t 137 yield t
138 138
139 def process(fin, fout): 139 def process(fin, fout, opts):
140 tokens = tokenize.tokenize(fin.readline) 140 tokens = tokenize.tokenize(fin.readline)
141 tokens = replacetokens(list(tokens)) 141 tokens = replacetokens(list(tokens), opts)
142 fout.write(tokenize.untokenize(tokens)) 142 fout.write(tokenize.untokenize(tokens))
143 143
144 def tryunlink(fname): 144 def tryunlink(fname):
145 try: 145 try:
146 os.unlink(fname) 146 os.unlink(fname)
166 166
167 def main(): 167 def main():
168 ap = argparse.ArgumentParser() 168 ap = argparse.ArgumentParser()
169 ap.add_argument('-i', '--inplace', action='store_true', default=False, 169 ap.add_argument('-i', '--inplace', action='store_true', default=False,
170 help='edit files in place') 170 help='edit files in place')
171 ap.add_argument('--dictiter', action='store_true', default=False,
172 help='rewrite iteritems() and itervalues()'),
171 ap.add_argument('files', metavar='FILE', nargs='+', help='source file') 173 ap.add_argument('files', metavar='FILE', nargs='+', help='source file')
172 args = ap.parse_args() 174 args = ap.parse_args()
175 opts = {
176 'dictiter': args.dictiter,
177 }
173 for fname in args.files: 178 for fname in args.files:
174 if args.inplace: 179 if args.inplace:
175 with editinplace(fname) as fout: 180 with editinplace(fname) as fout:
176 with open(fname, 'rb') as fin: 181 with open(fname, 'rb') as fin:
177 process(fin, fout) 182 process(fin, fout, opts)
178 else: 183 else:
179 with open(fname, 'rb') as fin: 184 with open(fname, 'rb') as fin:
180 fout = sys.stdout.buffer 185 fout = sys.stdout.buffer
181 process(fin, fout) 186 process(fin, fout, opts)
182 187
183 if __name__ == '__main__': 188 if __name__ == '__main__':
184 main() 189 main()