/Tools/scripts/cleanfuture.py

  1#! /usr/bin/env python
  2
  3"""cleanfuture [-d][-r][-v] path ...
  4
  5-d  Dry run.  Analyze, but don't make any changes to, files.
  6-r  Recurse.  Search for all .py files in subdirectories too.
  7-v  Verbose.  Print informative msgs.
  8
  9Search Python (.py) files for future statements, and remove the features
 10from such statements that are already mandatory in the version of Python
 11you're using.
 12
 13Pass one or more file and/or directory paths.  When a directory path, all
 14.py files within the directory will be examined, and, if the -r option is
 15given, likewise recursively for subdirectories.
 16
 17Overwrites files in place, renaming the originals with a .bak extension. If
 18cleanfuture finds nothing to change, the file is left alone.  If cleanfuture
 19does change a file, the changed file is a fixed-point (i.e., running
 20cleanfuture on the resulting .py file won't change it again, at least not
 21until you try it again with a later Python release).
 22
 23Limitations:  You can do these things, but this tool won't help you then:
 24
 25+ A future statement cannot be mixed with any other statement on the same
 26  physical line (separated by semicolon).
 27
 28+ A future statement cannot contain an "as" clause.
 29
 30Example:  Assuming you're using Python 2.2, if a file containing
 31
 32from __future__ import nested_scopes, generators
 33
 34is analyzed by cleanfuture, the line is rewritten to
 35
 36from __future__ import generators
 37
 38because nested_scopes is no longer optional in 2.2 but generators is.
 39"""
 40
 41import __future__
 42import tokenize
 43import os
 44import sys
 45
 46dryrun  = 0
 47recurse = 0
 48verbose = 0
 49
 50def errprint(*args):
 51    strings = map(str, args)
 52    msg = ' '.join(strings)
 53    if msg[-1:] != '\n':
 54        msg += '\n'
 55    sys.stderr.write(msg)
 56
 57def main():
 58    import getopt
 59    global verbose, recurse, dryrun
 60    try:
 61        opts, args = getopt.getopt(sys.argv[1:], "drv")
 62    except getopt.error, msg:
 63        errprint(msg)
 64        return
 65    for o, a in opts:
 66        if o == '-d':
 67            dryrun += 1
 68        elif o == '-r':
 69            recurse += 1
 70        elif o == '-v':
 71            verbose += 1
 72    if not args:
 73        errprint("Usage:", __doc__)
 74        return
 75    for arg in args:
 76        check(arg)
 77
 78def check(file):
 79    if os.path.isdir(file) and not os.path.islink(file):
 80        if verbose:
 81            print "listing directory", file
 82        names = os.listdir(file)
 83        for name in names:
 84            fullname = os.path.join(file, name)
 85            if ((recurse and os.path.isdir(fullname) and
 86                 not os.path.islink(fullname))
 87                or name.lower().endswith(".py")):
 88                check(fullname)
 89        return
 90
 91    if verbose:
 92        print "checking", file, "...",
 93    try:
 94        f = open(file)
 95    except IOError, msg:
 96        errprint("%r: I/O Error: %s" % (file, str(msg)))
 97        return
 98
 99    ff = FutureFinder(f, file)
100    changed = ff.run()
101    if changed:
102        ff.gettherest()
103    f.close()
104    if changed:
105        if verbose:
106            print "changed."
107            if dryrun:
108                print "But this is a dry run, so leaving it alone."
109        for s, e, line in changed:
110            print "%r lines %d-%d" % (file, s+1, e+1)
111            for i in range(s, e+1):
112                print ff.lines[i],
113            if line is None:
114                print "-- deleted"
115            else:
116                print "-- change to:"
117                print line,
118        if not dryrun:
119            bak = file + ".bak"
120            if os.path.exists(bak):
121                os.remove(bak)
122            os.rename(file, bak)
123            if verbose:
124                print "renamed", file, "to", bak
125            g = open(file, "w")
126            ff.write(g)
127            g.close()
128            if verbose:
129                print "wrote new", file
130    else:
131        if verbose:
132            print "unchanged."
133
134class FutureFinder:
135
136    def __init__(self, f, fname):
137        self.f = f
138        self.fname = fname
139        self.ateof = 0
140        self.lines = [] # raw file lines
141
142        # List of (start_index, end_index, new_line) triples.
143        self.changed = []
144
145    # Line-getter for tokenize.
146    def getline(self):
147        if self.ateof:
148            return ""
149        line = self.f.readline()
150        if line == "":
151            self.ateof = 1
152        else:
153            self.lines.append(line)
154        return line
155
156    def run(self):
157        STRING = tokenize.STRING
158        NL = tokenize.NL
159        NEWLINE = tokenize.NEWLINE
160        COMMENT = tokenize.COMMENT
161        NAME = tokenize.NAME
162        OP = tokenize.OP
163
164        changed = self.changed
165        get = tokenize.generate_tokens(self.getline).next
166        type, token, (srow, scol), (erow, ecol), line = get()
167
168        # Chew up initial comments and blank lines (if any).
169        while type in (COMMENT, NL, NEWLINE):
170            type, token, (srow, scol), (erow, ecol), line = get()
171
172        # Chew up docstring (if any -- and it may be implicitly catenated!).
173        while type is STRING:
174            type, token, (srow, scol), (erow, ecol), line = get()
175
176        # Analyze the future stmts.
177        while 1:
178            # Chew up comments and blank lines (if any).
179            while type in (COMMENT, NL, NEWLINE):
180                type, token, (srow, scol), (erow, ecol), line = get()
181
182            if not (type is NAME and token == "from"):
183                break
184            startline = srow - 1    # tokenize is one-based
185            type, token, (srow, scol), (erow, ecol), line = get()
186
187            if not (type is NAME and token == "__future__"):
188                break
189            type, token, (srow, scol), (erow, ecol), line = get()
190
191            if not (type is NAME and token == "import"):
192                break
193            type, token, (srow, scol), (erow, ecol), line = get()
194
195            # Get the list of features.
196            features = []
197            while type is NAME:
198                features.append(token)
199                type, token, (srow, scol), (erow, ecol), line = get()
200
201                if not (type is OP and token == ','):
202                    break
203                type, token, (srow, scol), (erow, ecol), line = get()
204
205            # A trailing comment?
206            comment = None
207            if type is COMMENT:
208                comment = token
209                type, token, (srow, scol), (erow, ecol), line = get()
210
211            if type is not NEWLINE:
212                errprint("Skipping file %r; can't parse line %d:\n%s" %
213                         (self.fname, srow, line))
214                return []
215
216            endline = srow - 1
217
218            # Check for obsolete features.
219            okfeatures = []
220            for f in features:
221                object = getattr(__future__, f, None)
222                if object is None:
223                    # A feature we don't know about yet -- leave it in.
224                    # They'll get a compile-time error when they compile
225                    # this program, but that's not our job to sort out.
226                    okfeatures.append(f)
227                else:
228                    released = object.getMandatoryRelease()
229                    if released is None or released <= sys.version_info:
230                        # Withdrawn or obsolete.
231                        pass
232                    else:
233                        okfeatures.append(f)
234
235            # Rewrite the line if at least one future-feature is obsolete.
236            if len(okfeatures) < len(features):
237                if len(okfeatures) == 0:
238                    line = None
239                else:
240                    line = "from __future__ import "
241                    line += ', '.join(okfeatures)
242                    if comment is not None:
243                        line += ' ' + comment
244                    line += '\n'
245                changed.append((startline, endline, line))
246
247            # Loop back for more future statements.
248
249        return changed
250
251    def gettherest(self):
252        if self.ateof:
253            self.therest = ''
254        else:
255            self.therest = self.f.read()
256
257    def write(self, f):
258        changed = self.changed
259        assert changed
260        # Prevent calling this again.
261        self.changed = []
262        # Apply changes in reverse order.
263        changed.reverse()
264        for s, e, line in changed:
265            if line is None:
266                # pure deletion
267                del self.lines[s:e+1]
268            else:
269                self.lines[s:e+1] = [line]
270        f.writelines(self.lines)
271        # Copy over the remainder of the file.
272        if self.therest:
273            f.write(self.therest)
274
275if __name__ == '__main__':
276    main()