Mercurial > public > mercurial-scm > hg-stable

annotate contrib/byteify-strings.py @ 38396:47dd23e6b116

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
byteify-strings: try to preserve column alignment
author Yuya Nishihara <yuya@tcha.org>
date Fri, 01 Jun 2018 00:13:55 +0900
parents 1d68fd5f614a
children f77bbd34a1df
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
38391
a2976c27dac4 byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
1 #!/usr/bin/env python3
a2976c27dac4 byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
2 #
38390
1d9c97db465f byteify-strings: fork py3 code transformer to make it a standalone command
Yuya Nishihara <yuya@tcha.org>
parents: 36646
diff changeset
3 # byteify-strings.py - transform string literals to be Python 3 safe
27220
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
4 #
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
5 # Copyright 2015 Gregory Szorc <gregory.szorc@gmail.com>
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
6 #
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
7 # This software may be used and distributed according to the terms of the
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
8 # GNU General Public License version 2 or any later version.
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
9
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
10 from __future__ import absolute_import
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
11
38391
a2976c27dac4 byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
12 import argparse
38392
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38391
diff changeset
13 import contextlib
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38391
diff changeset
14 import errno
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38391
diff changeset
15 import os
38391
a2976c27dac4 byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
16 import sys
38392
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38391
diff changeset
17 import tempfile
38390
1d9c97db465f byteify-strings: fork py3 code transformer to make it a standalone command
Yuya Nishihara <yuya@tcha.org>
parents: 36646
diff changeset
18 import token
1d9c97db465f byteify-strings: fork py3 code transformer to make it a standalone command
Yuya Nishihara <yuya@tcha.org>
parents: 36646
diff changeset
19 import tokenize
29550
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
20
38396
47dd23e6b116 byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents: 38395
diff changeset
21 def adjusttokenpos(t, ofs):
47dd23e6b116 byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents: 38395
diff changeset
22 """Adjust start/end column of the given token"""
47dd23e6b116 byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents: 38395
diff changeset
23 return t._replace(start=(t.start[0], t.start[1] + ofs),
47dd23e6b116 byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents: 38395
diff changeset
24 end=(t.end[0], t.end[1] + ofs))
47dd23e6b116 byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents: 38395
diff changeset
25
38390
1d9c97db465f byteify-strings: fork py3 code transformer to make it a standalone command
Yuya Nishihara <yuya@tcha.org>
parents: 36646
diff changeset
26 if True:
38394
f701bc936e7f byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents: 38393
diff changeset
27 def replacetokens(tokens, opts):
29550
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
28 """Transform a stream of tokens from raw to Python 3.
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
29
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
30 Returns a generator of possibly rewritten tokens.
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
31
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
32 The input token list may be mutated as part of processing. However,
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
33 its changes do not necessarily match the output token stream.
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
34 """
38395
1d68fd5f614a byteify-strings: do not rewrite system string literals to u''
Yuya Nishihara <yuya@tcha.org>
parents: 38394
diff changeset
35 sysstrtokens = set()
1d68fd5f614a byteify-strings: do not rewrite system string literals to u''
Yuya Nishihara <yuya@tcha.org>
parents: 38394
diff changeset
36
30165
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
37 # The following utility functions access the tokens list and i index of
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
38 # the for i, t enumerate(tokens) loop below
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
39 def _isop(j, *o):
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
40 """Assert that tokens[j] is an OP with one of the given values"""
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
41 try:
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
42 return tokens[j].type == token.OP and tokens[j].string in o
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
43 except IndexError:
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
44 return False
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
45
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
46 def _findargnofcall(n):
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
47 """Find arg n of a call expression (start at 0)
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
48
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
49 Returns index of the first token of that argument, or None if
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
50 there is not that many arguments.
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
51
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
52 Assumes that token[i + 1] is '('.
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
53
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
54 """
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
55 nested = 0
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
56 for j in range(i + 2, len(tokens)):
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
57 if _isop(j, ')', ']', '}'):
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
58 # end of call, tuple, subscription or dict / set
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
59 nested -= 1
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
60 if nested < 0:
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
61 return None
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
62 elif n == 0:
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
63 # this is the starting position of arg
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
64 return j
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
65 elif _isop(j, '(', '[', '{'):
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
66 nested += 1
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
67 elif _isop(j, ',') and nested == 0:
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
68 n -= 1
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
69
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
70 return None
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
71
38395
1d68fd5f614a byteify-strings: do not rewrite system string literals to u''
Yuya Nishihara <yuya@tcha.org>
parents: 38394
diff changeset
72 def _ensuresysstr(j):
1d68fd5f614a byteify-strings: do not rewrite system string literals to u''
Yuya Nishihara <yuya@tcha.org>
parents: 38394
diff changeset
73 """Make sure the token at j is a system string
30165
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
74
38395
1d68fd5f614a byteify-strings: do not rewrite system string literals to u''
Yuya Nishihara <yuya@tcha.org>
parents: 38394
diff changeset
75 Remember the given token so the string transformer won't add
1d68fd5f614a byteify-strings: do not rewrite system string literals to u''
Yuya Nishihara <yuya@tcha.org>
parents: 38394
diff changeset
76 the byte prefix.
30165
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
77
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
78 Ignores tokens that are not strings. Assumes bounds checking has
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
79 already been done.
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
80
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
81 """
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
82 st = tokens[j]
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
83 if st.type == token.STRING and st.string.startswith(("'", '"')):
38395
1d68fd5f614a byteify-strings: do not rewrite system string literals to u''
Yuya Nishihara <yuya@tcha.org>
parents: 38394
diff changeset
84 sysstrtokens.add(st)
30165
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
85
38396
47dd23e6b116 byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents: 38395
diff changeset
86 coldelta = 0 # column increment for new opening parens
47dd23e6b116 byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents: 38395
diff changeset
87 coloffset = -1 # column offset for the current line (-1: TBD)
47dd23e6b116 byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents: 38395
diff changeset
88 parens = [(0, 0, 0)] # stack of (line, end-column, column-offset)
29550
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
89 for i, t in enumerate(tokens):
38396
47dd23e6b116 byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents: 38395
diff changeset
90 # Compute the column offset for the current line, such that
47dd23e6b116 byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents: 38395
diff changeset
91 # the current line will be aligned to the last opening paren
47dd23e6b116 byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents: 38395
diff changeset
92 # as before.
47dd23e6b116 byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents: 38395
diff changeset
93 if coloffset < 0:
47dd23e6b116 byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents: 38395
diff changeset
94 if t.start[1] == parens[-1][1]:
47dd23e6b116 byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents: 38395
diff changeset
95 coloffset = parens[-1][2]
47dd23e6b116 byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents: 38395
diff changeset
96 elif t.start[1] + 1 == parens[-1][1]:
47dd23e6b116 byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents: 38395
diff changeset
97 # fix misaligned indent of s/util.Abort/error.Abort/
47dd23e6b116 byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents: 38395
diff changeset
98 coloffset = parens[-1][2] + (parens[-1][1] - t.start[1])
47dd23e6b116 byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents: 38395
diff changeset
99 else:
47dd23e6b116 byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents: 38395
diff changeset
100 coloffset = 0
47dd23e6b116 byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents: 38395
diff changeset
101
47dd23e6b116 byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents: 38395
diff changeset
102 # Reset per-line attributes at EOL.
47dd23e6b116 byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents: 38395
diff changeset
103 if t.type in (token.NEWLINE, tokenize.NL):
47dd23e6b116 byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents: 38395
diff changeset
104 yield adjusttokenpos(t, coloffset)
47dd23e6b116 byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents: 38395
diff changeset
105 coldelta = 0
47dd23e6b116 byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents: 38395
diff changeset
106 coloffset = -1
47dd23e6b116 byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents: 38395
diff changeset
107 continue
47dd23e6b116 byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents: 38395
diff changeset
108
47dd23e6b116 byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents: 38395
diff changeset
109 # Remember the last paren position.
47dd23e6b116 byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents: 38395
diff changeset
110 if _isop(i, '(', '[', '{'):
47dd23e6b116 byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents: 38395
diff changeset
111 parens.append(t.end + (coloffset + coldelta,))
47dd23e6b116 byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents: 38395
diff changeset
112 elif _isop(i, ')', ']', '}'):
47dd23e6b116 byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents: 38395
diff changeset
113 parens.pop()
47dd23e6b116 byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents: 38395
diff changeset
114
29550
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
115 # Convert most string literals to byte literals. String literals
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
116 # in Python 2 are bytes. String literals in Python 3 are unicode.
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
117 # Most strings in Mercurial are bytes and unicode strings are rare.
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
118 # Rather than rewrite all string literals to use ``b''`` to indicate
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
119 # byte strings, we apply this token transformer to insert the ``b``
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
120 # prefix nearly everywhere.
38395
1d68fd5f614a byteify-strings: do not rewrite system string literals to u''
Yuya Nishihara <yuya@tcha.org>
parents: 38394
diff changeset
121 if t.type == token.STRING and t not in sysstrtokens:
29550
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
122 s = t.string
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
123
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
124 # Preserve docstrings as string literals. This is inconsistent
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
125 # with regular unprefixed strings. However, the
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
126 # "from __future__" parsing (which allows a module docstring to
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
127 # exist before it) doesn't properly handle the docstring if it
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
128 # is b''' prefixed, leading to a SyntaxError. We leave all
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
129 # docstrings as unprefixed to avoid this. This means Mercurial
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
130 # components touching docstrings need to handle unicode,
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
131 # unfortunately.
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
132 if s[0:3] in ("'''", '"""'):
38396
47dd23e6b116 byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents: 38395
diff changeset
133 yield adjusttokenpos(t, coloffset)
29550
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
134 continue
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
135
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
136 # If the first character isn't a quote, it is likely a string
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
137 # prefixing character (such as 'b', 'u', or 'r'. Ignore.
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
138 if s[0] not in ("'", '"'):
38396
47dd23e6b116 byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents: 38395
diff changeset
139 yield adjusttokenpos(t, coloffset)
29550
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
140 continue
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
141
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
142 # String literal. Prefix to make a b'' string.
38396
47dd23e6b116 byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents: 38395
diff changeset
143 yield adjusttokenpos(t._replace(string='b%s' % t.string),
47dd23e6b116 byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents: 38395
diff changeset
144 coloffset)
47dd23e6b116 byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents: 38395
diff changeset
145 coldelta += 1
29550
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
146 continue
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
147
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
148 # This looks like a function call.
30165
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
149 if t.type == token.NAME and _isop(i + 1, '('):
29550
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
150 fn = t.string
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
151
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
152 # *attr() builtins don't accept byte strings to 2nd argument.
30165
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
153 if (fn in ('getattr', 'setattr', 'hasattr', 'safehasattr') and
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
154 not _isop(i - 1, '.')):
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
155 arg1idx = _findargnofcall(1)
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
156 if arg1idx is not None:
38395
1d68fd5f614a byteify-strings: do not rewrite system string literals to u''
Yuya Nishihara <yuya@tcha.org>
parents: 38394
diff changeset
157 _ensuresysstr(arg1idx)
29550
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
158
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
159 # .encode() and .decode() on str/bytes/unicode don't accept
30165
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
160 # byte strings on Python 3.
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
161 elif fn in ('encode', 'decode') and _isop(i - 1, '.'):
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
162 for argn in range(2):
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
163 argidx = _findargnofcall(argn)
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
164 if argidx is not None:
38395
1d68fd5f614a byteify-strings: do not rewrite system string literals to u''
Yuya Nishihara <yuya@tcha.org>
parents: 38394
diff changeset
165 _ensuresysstr(argidx)
29550
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
166
31454
83e080144faf py3: rewrite itervalues() as values() by importer
Yuya Nishihara <yuya@tcha.org>
parents: 31370
diff changeset
167 # It changes iteritems/values to items/values as they are not
30052
eaaedad68011 py3: switch to .items() using transformer
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30051
diff changeset
168 # present in Python 3 world.
38394
f701bc936e7f byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents: 38393
diff changeset
169 elif opts['dictiter'] and fn in ('iteritems', 'itervalues'):
38396
47dd23e6b116 byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents: 38395
diff changeset
170 yield adjusttokenpos(t._replace(string=fn[4:]), coloffset)
30052
eaaedad68011 py3: switch to .items() using transformer
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30051
diff changeset
171 continue
eaaedad68011 py3: switch to .items() using transformer
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30051
diff changeset
172
29550
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
173 # Emit unmodified token.
38396
47dd23e6b116 byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents: 38395
diff changeset
174 yield adjusttokenpos(t, coloffset)
38391
a2976c27dac4 byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
175
38394
f701bc936e7f byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents: 38393
diff changeset
176 def process(fin, fout, opts):
38391
a2976c27dac4 byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
177 tokens = tokenize.tokenize(fin.readline)
38394
f701bc936e7f byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents: 38393
diff changeset
178 tokens = replacetokens(list(tokens), opts)
38391
a2976c27dac4 byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
179 fout.write(tokenize.untokenize(tokens))
a2976c27dac4 byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
180
38392
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38391
diff changeset
181 def tryunlink(fname):
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38391
diff changeset
182 try:
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38391
diff changeset
183 os.unlink(fname)
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38391
diff changeset
184 except OSError as err:
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38391
diff changeset
185 if err.errno != errno.ENOENT:
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38391
diff changeset
186 raise
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38391
diff changeset
187
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38391
diff changeset
188 @contextlib.contextmanager
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38391
diff changeset
189 def editinplace(fname):
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38391
diff changeset
190 n = os.path.basename(fname)
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38391
diff changeset
191 d = os.path.dirname(fname)
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38391
diff changeset
192 fp = tempfile.NamedTemporaryFile(prefix='.%s-' % n, suffix='~', dir=d,
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38391
diff changeset
193 delete=False)
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38391
diff changeset
194 try:
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38391
diff changeset
195 yield fp
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38391
diff changeset
196 fp.close()
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38391
diff changeset
197 if os.name == 'nt':
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38391
diff changeset
198 tryunlink(fname)
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38391
diff changeset
199 os.rename(fp.name, fname)
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38391
diff changeset
200 finally:
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38391
diff changeset
201 fp.close()
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38391
diff changeset
202 tryunlink(fp.name)
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38391
diff changeset
203
38391
a2976c27dac4 byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
204 def main():
a2976c27dac4 byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
205 ap = argparse.ArgumentParser()
38392
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38391
diff changeset
206 ap.add_argument('-i', '--inplace', action='store_true', default=False,
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38391
diff changeset
207 help='edit files in place')
38394
f701bc936e7f byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents: 38393
diff changeset
208 ap.add_argument('--dictiter', action='store_true', default=False,
f701bc936e7f byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents: 38393
diff changeset
209 help='rewrite iteritems() and itervalues()'),
38391
a2976c27dac4 byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
210 ap.add_argument('files', metavar='FILE', nargs='+', help='source file')
a2976c27dac4 byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
211 args = ap.parse_args()
38394
f701bc936e7f byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents: 38393
diff changeset
212 opts = {
f701bc936e7f byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents: 38393
diff changeset
213 'dictiter': args.dictiter,
f701bc936e7f byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents: 38393
diff changeset
214 }
38391
a2976c27dac4 byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
215 for fname in args.files:
38392
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38391
diff changeset
216 if args.inplace:
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38391
diff changeset
217 with editinplace(fname) as fout:
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38391
diff changeset
218 with open(fname, 'rb') as fin:
38394
f701bc936e7f byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents: 38393
diff changeset
219 process(fin, fout, opts)
38392
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38391
diff changeset
220 else:
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38391
diff changeset
221 with open(fname, 'rb') as fin:
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38391
diff changeset
222 fout = sys.stdout.buffer
38394
f701bc936e7f byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents: 38393
diff changeset
223 process(fin, fout, opts)
38391
a2976c27dac4 byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
224
a2976c27dac4 byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
225 if __name__ == '__main__':
a2976c27dac4 byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
226 main()