python-gadfly 1.0.0-16 / usr / share / doc / python-gadfly / examples / pylint.py

#!/usr/bin/python
"""python lint using kwParsing

The goal of this module/filter is to help find
programming errors in python source files.

As a filter use thusly:

% python kjpylint.py source_file.py

As an internal tool use like this:

  import kjpylint
  (pyg, context) = kjpylint.setup()
  kjpylint.lint(data, pyg, context)

where data is the text of a python program.
You can build your own context structure by
subclassing GlobalContext, and redefining
GlobalContext.complain(string) for example.
You could do a lot more than that too...

Also, to lint all *.py files recursively contained
in a directory hierarchy use

  kjpylint.lintdir("/usr/local/lib/python") # for example

FEATURES:

Lint expects
  1) a newline or two at the end of the data;
  2) consistent indenting (and inconsistency may be invisible)
     [eg " \t" and "\t" are not the same indent
     to Lint, but Python sees them the same.]

If (1) or (2) are not satisfied Lint will raise
an exception.

Buglets: lambdas and for loops on one line generate
  extraneous warnings.

Notes:
======
The lint process works, in outline, like this.
Scan over a python program

x = 1

def f(a):
    a = x
    d.x, y = b

z = w

and build annotations like

[ set("x", 1),
  [
    get("x", 4)
    set("a", 4)
    get("b", 5)
    get("d", 5)
    set("y", 5)
    pop_local()
  ]
  get("w", 7)
  set("z", 7) ]

from this stream conclude
  warning on line 5: b used before set
  warning on line 5: d used before set
  warning on line 5: y set, never used
etc. using simple one pass approximate flow
analysis.
"""

pyg = context = None

#import pygram
from pygram import newlineresult

# reduction rules:
#  only need to consider
#    expressions, assignments, def, class, global, import, from, for
#
# expressions return a list of unqualified names, not known set
# qualified names are automatically put in context as refs
#
# assignments set left names, ref right names
#
# def sets new name for function and args,
#  refs other names
#
# class adds new name for class
#  refs other names
#
# global forces global interpretation for name
#
# import adds FIRST names
# from sets names
# for sets names
#
# related rules
# ASSIGNMENT REQUIRES SPECIAL TREATMENT
#@R assn1 :: assn >> testlist = testlist

def assn1(list, context):
    [t1, e, t2] = list
    return assn(t1, t2)

#@R assnn :: assn >> testlist = assn

def assnn(list, context):
    [t1, e, a1] = list
    return assn(t1, a1)

# @R assn1c :: assn >> testlist , = testlist
def assn1c(list, context):
    [t1, c, e, t2] = list
    return assn(t1, t2)

# @R assn1c2 :: assn >> testlist , = testlist ,
def assn1c2(list, context):
    del list[-1]
    return assn1c(list, context)

# @R assnnc :: assn >> testlist , = assn
def assnnc(list, context):
    return assn1c(list, context)

def assn(left, right):
    result = right
    for x in left:
        (ln, ri, op, name) = x
        if op == "ref":
            result.append( (ln, ri, "set", name) )
        else:
            result.append(x)
    return result

#@R except2 :: except_clause >> except test , test
def except2(list, context):
    [e, t1, c, t2] = list
    result = t1
    for (ln, ri, op, name) in t2:
        result.append( (ln, ri, "set", name) )
    return result

#@R smassn :: small_stmt >> assn
#  ignored

#@R rfrom :: import_stmt >> from dotted_name import name_list
#@R rfromc :: import_stmt >> from dotted_name import name_list ,

def rfrom(list, context):
    #print rfrom, list
    [f, d, i, n] = list
    # ignore d
    return n

def rfromc(list, context):
    return rfrom(list[:-1])

def mark(kind, thing, context):
    L = context.LexD
    lineno = L.lineno
    # are we reducing on a newline?
    if L.lastresult==newlineresult:
        lineno = lineno-1
    return (lineno, -L.realindex, kind, thing)

#@R dn1 :: dotted_name >> NAME

def dn1(list, context):
    #print "dn1", list
    #L = context.LexD
    return [ mark("set", list[0], context) ]
    #return [ (L.lineno, -L.realindex, "set", list[0]) ]



#  handles import case, make name set local
#@R nlistn :: name_list >> name_list  , NAME

def nlistn(list, context):
    #print "nlistn", list
    [nl, c, n] = list
    #L = context.LexD
    #nl.append( (L.lineno, -L.realindex, "set", n) )
    nl.append( mark("set", n, context) )
    return nl

#@R nlist1 :: name_list >> NAME

def nlist1(list, context):
    #print "nlist1", list
    #L = context.LexD
    #return [ (L.lineno, -L.realindex, "set", list[0]) ]
    return [ mark("set", list[0], context) ]

# ignore lhs in calls with keywords.
#@R namearg :: argument >> test = test
def namearg(list, context):
    [t1, e, t2] = list
    return t2

#  handles from case, make names set local
#@R global1 :: global_stmt >> global NAME

def global1(list, context):
    #print "global1", list
    #L = context.LexD
    #return [ (L.lineno, -L.realindex, "global", list[1]) ]
    return [ mark("global", list[1], context) ]

#@R globaln :: global_stmt >> global_stmt , NAME
#  handles global, make names global (not set or reffed)

def globaln(list, context):
    #print "globaln", list
    [g, c, n] = list
    #L = context.LexD
    #g.append( (L.lineno, -L.realindex, "global", n) )
    g.append( mark("global", n, context) )
    return g

#@R for1 :: for_stmt >>
#for exprlist in testlist  :
#     suite

def for1(list, context):
    #print "for1", list
    [f, e, i, t, c, s] = list
    refs = t + s
    return assn(e, refs)

#@R for2 :: for_stmt >>
#for exprlist in testlist  :
#     suite
#else :
#     suite

def for2(list,context):
    #print "for2", list
    [f, e, i, t, c1, s1, el, c2, s2] = list
    refs = t + s1 + s2
    return assn(e, refs)

###
#@R class1 :: classdef  >> class NAME : suite
def class1(list, context):
    [c, n, cl, s] = list
    return Class(n, [], s, context)

#@R class2 :: classdef  >> class NAME ( testlist ) : suite
def class2(list, context):
    [c, n, opn, t, cls, cl, s] = list
    return Class(n, t, s, context)

def Class(name, testlist, suite, context):
    globals = analyse_scope(name, suite, context, unused_ok=1)
    context.defer_globals(globals)
    result = testlist
    L = context.LexD
    # try to correct lineno
    lineno = L.lineno
    realindex = L.realindex
    for (ln, ri, op, n) in testlist+suite:
        lineno = min(lineno, ln)
    result.append((lineno, -realindex, "set", name))
    #result.append( mark("set", name, context) )
    # supress complaints about unreffed classes
    result.append((lineno+1, -realindex, "qref", name))
    #result.append( mark("qref", name, context) )
    return result

# vararsglist requires special treatment.
#  return (innerscope, outerscope) pair of lists
# @R params1 :: parameters >> ( varargslist )
def params1(l, c):
    return l[1]

params1c = params1

#@R params2 :: varargslist >>
def params2(l, c):
    return ([], [])

#@R params3 :: varargslist >> arg
def params3(l, c):
    return l[0]

#@R params4 :: varargslist >> varargslist , arg
def params4(l, c):
    #print "params4", l
    [v, c, a] = l
    v[0][0:0] = a[0]
    v[1][0:0] = a[1]
    return v

#@R argd :: arg >> NAME = test
def argd(l, c):
    [n, e, t] = l
    #L = c.LexD
    #return ([(L.lineno, -L.realindex, "set", n)], t)
    return ([ mark("set", n, c) ], t)

#@R arg2 :: arg >> fpdef
def arg2(l, c):
    return l[0]

#@R arg3 :: arg >> * NAME
def arg3(l, c):
    del l[0]
    return fpdef1(l, c)

#@R arg4 :: arg >> ** NAME
def arg4(l, c):
    #del l[0]
    return arg3(l, c)

#@R fpdef1 :: fpdef  >> NAME
def fpdef1(l, c):
    [n] = l
    #LexD = c.LexD
    return ([ mark("set", n, c) ], [])

#@R fpdef2 :: fpdef  >>  ( fplist )
def fpdef2(l, c):
    return l[1]

## @R fpdef2c :: fpdef  >>  ( fplist , )
#fpdef2c = fpdef2

##31
#@R fplist1 :: fplist >> fpdef
def fplist1(l, c):
    #print l
    return l[0]

#@R fplistn :: fplist >> fplist , fpdef
fplistn = params4

#@R rdef :: funcdef >> def NAME parameters : suite
def rdef(list, context):
    #print "rdef", list
    [ddef, name, parameters, c, suite] = list
    (l, g) = parameters
    globals = analyse_scope(name, l + suite, context)
    # for embedded function defs global internal refs must be deferred.
    context.defer_globals(globals)
    result = g
    L = context.LexD
    # try to steal a lineno from other declarations:
    lineno = L.lineno
    index = L.realindex
    for (ln, ri, op, n) in l+g+suite:
        lineno = min(lineno, ln)
    if name is not None:
        result.append((lineno, -index, "set", name))
        # Note: this is to prevent complaints about unreffed functions
        result.append((lineno+1, -index, "qref", name))
    return result

#@R testlambda1 :: test >> lambda varargslist : test
def testlambda1(list, context):
    [l, v, c, t] = list
    return rdef(["def", None, v, ":", t], context)

def analyse_scope(sname, var_accesses, context, unused_ok=0):
    var_accesses.sort()
    result = []
    globals = {}
    locals = {}
    # scan for globals
    for x in var_accesses:
        (ln, ri, op, name) = x
        if op == "global":
            globals[name] = ln
        #result.append(x) (ignore global sets in local context)
    # scan for locals
    for (ln, ri, op, name) in var_accesses:
        if op == "set" and not locals.has_key(name):
            if globals.has_key(name):
                context.complain(
       "Warning: set of global %s in local context %s" % (`name`, `sname`))
                result.append( (ln, ri, op, name) )
                pass # ignore global set in local context
            else:
                locals[name] = [ln, 0] # line assigned, #refs
    # scan for use before assign, etc.
    for x in var_accesses:
        (ln, ri, op, name) = x
        if locals.has_key(name):
            if op in ["ref", "qref"]:
                set = locals[name]
                set[1] = set[1] + 1
                assnln = set[0]
                if (ln <= assnln):
                    context.complain(
             "(%s) local %s ref at %s before assign at %s" % (
              sname, `name`, ln, `assnln`))
        elif op not in ("global", "set"):
            # ignore global sets in local context.
            result.append(x)
    # scan for no use
    if not unused_ok:
        for (name, set) in locals.items():
            [where, count] = set
            if count<1:
                context.complain(
                   "(%s) %s defined before %s not used" % (sname, `name`, where))
    return result

### note, need to make special case for qualified names
#@R powera :: power >> atom trailerlist

def powera(list, context):
    #print "powera", list
    [a, (t, full)] = list
    if a and full:
        # atom is a qualified name
        (ln, ri, op, n) = a[0]
        result = [ (ln, ri, "qref", n) ]
    else:
        result = a
    result = result + t
    #print "returning", result
    return result

#@R trailerlist0 :: trailerlist >>
def trailerlist0(list, context):
    return ([], 0) # empty trailerlist

#@R trailerlistn :: trailerlist >> trailer trailerlist
def trailerlistn(list, context):
    #print "trailerlistn", list
    result = list[0] + list[1][0]
    for i in xrange(len(result)):
        (a, b, op, d) = result[i]
        result[i] = (a, b, "qref", d)
    return (result, 1)

# make name+parameters set local reduce suite...

def default_reduction(list, context):
    # append all lists
    from types import ListType
    #print "defred", list
    #return
    result = []
    for x in list:
        if type(x)==ListType:
            if result == []:
                if len(x)>0 and type(x[0])==ListType:
                    raise ValueError, "oops: %s" % repr(x)
                result = x
            else:
                for y in x:
                    result.append(y)
    return result

def aname(list, context):
    #print "aname", list, context
    L = context.LexD
    # note -L.realindex makes rhs of assignment seem before lhs in sort.
    return [ (L.lineno, -L.realindex, "ref", list[0]) ]


# the highest level reduction!
# all1 :: all >> file_input DEDENT
def all1(list, context):
    stuff = list[0]
    context.when_done(stuff)

# first test
def BindRules(pyg):
    for name in pyg.RuleNameToIndex.keys():
        pyg.Bind(name, default_reduction)
    pyg.Bind("all1", all1)
    pyg.Bind("testlambda1", testlambda1)
    pyg.Bind("except2", except2)
    pyg.Bind("namearg", namearg)
    pyg.Bind("rfrom", rfrom)
    pyg.Bind("rfromc", rfromc)
    pyg.Bind("class1", class1)
    pyg.Bind("class2", class2)
    pyg.Bind("aname", aname)
    pyg.Bind("assn1", assn1)
    pyg.Bind("assnn", assnn)
    pyg.Bind("assn1c", assn1c)
    pyg.Bind("assn1c2", assn1c2)
    pyg.Bind("assnnc", assnnc)
    pyg.Bind("dn1", dn1)
    pyg.Bind("nlistn", nlistn)
    pyg.Bind("nlist1", nlist1)
    pyg.Bind("global1", global1)
    pyg.Bind("globaln", globaln)
    pyg.Bind("for1", for1)
    pyg.Bind("for2", for2)
    pyg.Bind("powera", powera)
    pyg.Bind("trailerlist0", trailerlist0)
    pyg.Bind("trailerlistn", trailerlistn)
    pyg.Bind("params1", params1)
    pyg.Bind("params1c", params1c)
    pyg.Bind("params2", params2)
    pyg.Bind("params3", params3)
    pyg.Bind("params4", params4)
    pyg.Bind("argd", argd)
    pyg.Bind("arg2", arg2)
    pyg.Bind("arg3", arg3)
    pyg.Bind("arg4", arg4)
    pyg.Bind("fpdef1", fpdef1)
    pyg.Bind("fpdef2", fpdef2)
#    pyg.Bind("fpdef2c", fpdef2c)
    pyg.Bind("fplist1" , fplist1 )
    pyg.Bind("fplistn" , fplistn)
    pyg.Bind("rdef" , rdef)
#    pyg.Bind( , )

class globalContext:
    def __init__(self, lexd):
        self.deferred = []
        self.LexD = lexd
    def complain(self, str):
        print str
    def defer_globals(self, globals):
        self.deferred[0:0] = globals
    def when_done(self, list):
        stuff = list + self.deferred + self.patch_globals()
        globals = analyse_scope("<module global>", stuff, self)
        seen = {}
        for (ln, ri, op, name) in globals:
            if not seen.has_key(name) and op!="set":
                seen[name] = name
                self.complain(
       "%s: (%s) %s not defined in module?" % (ln, op, `name`))
        self.deferred = [] # reset state.
    def patch_globals(self):
        # patch in global names
        import __builtin__
        names = dir(__builtin__)
        list = names[:]
        list2 = names[:]
        for i in xrange(len(list)):
            list[i] = (-2, -900, "set", names[i])
            list2[i] = (-1, -900, "qref", names[i])
        return list + list2

teststring = """
class x(y,z):
  '''
     a doc string
     blah
  '''
  def test(this, that):
    w = that+this+x, n
    x = 1
    return w
"""

def go():
    import sys
    try:
        file = sys.argv[1]
    except IndexError:
        print "required input file missing, defaulting to test string"
        data = teststring
    else:
        data = open(file).read()
    print "setup"
    (pyg, context) = setup()
    print "now parsing"
    lint(data, pyg, context)

def setup():
    global pyg, context
    import pygram
    pyg = pygram.unMarshalpygram()
    BindRules(pyg)
    context = globalContext(pyg.LexD)
    return (pyg, context)

def lint(data, pygin=None, contextin=None):
    if pygin is None: pygin = pyg
    if contextin is None: contextin = context
    pygin.DoParse1(data, contextin)

def lintdir(directory_name):
    """lint all files recursively in directory"""
    from find import find
    print "\n\nrecursively linting %s\n\n" % directory_name
    (pyg, context) = setup()
    python_files = find("*.py", directory_name)
    for x in python_files:
        print "\n\n [ %s ]\n\n" % x
        lint( open(x).read(), pyg, context )
        print "\014"

if __name__=="__main__": go()