[java2python] r184 committed - Inching towards release.
java2...@googlecode.com
Author: troy.melhase
Date: Mon Oct 24 11:13:39 2011
Log: Inching towards release.
http://code.google.com/p/java2python/source/detail?r=184
Added:
/trunk/doc/glossary.rst
Modified:
/trunk/bin/j2py
/trunk/doc/conf.py
/trunk/doc/index.rst
/trunk/doc/intro.rst
/trunk/java2python/compiler/template.py
/trunk/java2python/config/__init__.py
/trunk/java2python/config/default.py
/trunk/java2python/lang/selector.py
/trunk/java2python/mod/basic.py
/trunk/java2python/mod/transform.py
/trunk/test/selector/test_all.py
=======================================
--- /dev/null
+++ /trunk/doc/glossary.rst Mon Oct 24 11:13:39 2011
@@ -0,0 +1,26 @@
+.. _glossary:
+
+********
+Glossary
+********
+
+
+.. glossary::
+
+ ANTLR
+ `ANother Tool for Language Recognition`_, the language tool used by
+ this package to parse Java source code.
+
+ AST
+ Abstract Syntax Tree
+
+ Block
+ A class that combines an AST visitor and template generator.
+
+ Template
+ A class that can serialize a block to Python source.
+
+ Visitor
+ A class that accepts AST nodes and generates code via templating.
+
+.. _ANother Tool for Language Recognition: http://www.antlr.org/
=======================================
--- /trunk/bin/j2py Wed Oct 19 20:00:15 2011
+++ /trunk/bin/j2py Mon Oct 24 11:13:39 2011
@@ -1,6 +1,11 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
-
+""" j2py -> Java to Python compiler script.
+
+This is all very ordinary. We import the package bits, open and read
+a file, translate it, and write it out.
+
+"""
import sys
from collections import defaultdict
from logging import _levelNames as logLevels, exception, warning, info,
basicConfig
@@ -17,15 +22,18 @@
def isWindows():
+ """ True if running on Windows. """
return sys.platform.startswith('win')
def badLogLevel(name, value):
+ """ Raise an error indicating a bad log level. """
msg = 'option %s: invalid loglevel: %r'
raise OptionValueError(msg % (name, value))
def checkLogLevel(option, opt, value):
+ """ Option type checker (see LocalOption class) to verify a log
level. """
try:
lvl = int(value)
except (ValueError, ):
@@ -40,12 +48,14 @@
class LocalOption(Option):
+ """ Supplements the Option class with our log level checker. """
TYPES = Option.TYPES + ('loglevel', )
TYPE_CHECKER = Option.TYPE_CHECKER.copy()
TYPE_CHECKER['loglevel'] = checkLogLevel
-def mainProfile(options):
+def profileMain(options):
+ """ Runs our main function with profiling if indicated by options. """
if options.profile:
import cProfile, pstats
prof = cProfile.Profile()
@@ -59,11 +69,13 @@
def configFromDir(inname, dirname):
+ """ Returns a file name from the given config directory. """
name = path.join(dirname, path.basename(path.splitext(inname)[0]))
return '%s.py' % path.abspath(name)
def main(options):
+ """ Compile the indicated java source with the given options. """
timed = defaultdict(time)
timed['overall']
@@ -155,6 +167,7 @@
def config(argv):
+ """ Return an options object from the given argument sequence. """
parser = OptionParser(option_class=LocalOption,
version='%prog '+version)
addopt = parser.add_option
addopt('-i', '--input', dest='inputfile',
@@ -215,4 +228,4 @@
if __name__ == '__main__':
- sys.exit(mainProfile(config(sys.argv)))
+ sys.exit(profileMain(config(sys.argv)))
=======================================
--- /trunk/doc/conf.py Mon Jul 26 18:14:27 2010
+++ /trunk/doc/conf.py Mon Oct 24 11:13:39 2011
@@ -16,7 +16,7 @@
# If extensions (or modules to document with autodoc) are in another
directory,
# add these directories to sys.path here. If the directory is relative to
the
# documentation root, use os.path.abspath to make it absolute, like shown
here.
-#sys.path.insert(0, os.path.abspath('.'))
+sys.path.insert(0, os.path.abspath('../'))
# -- General configuration
-----------------------------------------------------
@@ -25,7 +25,7 @@
# Add any Sphinx extension module names here, as strings. They can be
extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = ['sphinx.ext.todo', 'sphinx.ext.viewcode']
+extensions =
['sphinx.ext.todo', 'sphinx.ext.viewcode', 'sphinx.ext.autodoc', ]
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
@@ -41,7 +41,7 @@
# General information about the project.
project = u'java2python'
-copyright = u'2010, Troy Melhase'
+copyright = u'2011, Troy Melhase'
# The version info for the project you're documenting, acts as replacement
for
# |version| and |release|, also used in various other places throughout the
=======================================
--- /trunk/doc/index.rst Wed Jul 28 15:44:19 2010
+++ /trunk/doc/index.rst Mon Oct 24 11:13:39 2011
@@ -25,6 +25,7 @@
usage.rst
features.rst
customization.rst
+ glossary.rst
Indices and tables
@@ -33,4 +34,5 @@
* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`
-
+* :ref:`glossary`
+
=======================================
--- /trunk/doc/intro.rst Wed Jul 28 15:44:19 2010
+++ /trunk/doc/intro.rst Mon Oct 24 11:13:39 2011
@@ -10,7 +10,7 @@
|j2py| reads the Java source files you give it and produces
somewhat-roughly-equivalent Python source code. It tries to make the
same decisions you would if you were porting the code manually. It
-can perform the translation faster and more accuratly than you could,
+can perform the translation faster and more accurately than you could,
because it's a dumb machine that does what its told and you're a smart
person with lots of books you haven't read and a love of chocolate so
sometimes you're easily distracted and make mistakes. Like me and
@@ -41,8 +41,8 @@
|j2py| first converts the source code you give it into an abstract
syntax tree. (That's a lie, really. |j2py| doesn't do this step,
-Antlr does this step, and Antlr is a whole lot bigger and cooler than
-|j2py| could ever be. Obviously, really smart people worked on Antlr
+ANTLR does this step, and ANTLR is a whole lot bigger and cooler than
+|j2py| could ever be. Obviously, really smart people worked on ANTLR
and only one fairly dim one worked on |j2py|).
After the syntax tree is constructed, it's walked and its nodes are
@@ -63,16 +63,16 @@
Python equivalents.
To get around these trouble spots, |j2py| takes one of two approaches
-(and sometimes both if she's feeling especially fiesty or if you
+(and sometimes both if she's feeling especially feisty or if you
haven't paid her much attention lately). The first approach is to try
and make the problem go away. For example, in Java the `if` statement
-can contain an assigment expression::
+can contain an assignment expression::
if (++x == 0) { ... }
-There isn't a single statement equivalent in Python because assigments
+There isn't a single statement equivalent in Python because assignments
are statements there, not expressions. So |j2py| does what it can,
-presuably what you would do::
+presumably what you would do::
x += 1
if x == 0:
@@ -82,7 +82,7 @@
over a cliff with that `++x` expression. If the increment had been
done on the other side of the variable, the meaning of the statement
would have changed and the Python code would have been wrong.
-Fortuatly, I've driven by lots of cliffs and have been scared by all
+Fortunately, I've driven by lots of cliffs and have been scared by all
of them so I thought of this ahead of time and decided to do something
about it::
@@ -96,7 +96,7 @@
...
See what |j2py| did there? It tried to do what you would do. For
-further explaination and enumeration see the :ref:`features` chapter.
+further explanation and enumeration see the :ref:`features` chapter.
Why Bother?
=======================================
--- /trunk/java2python/compiler/template.py Tue Oct 18 21:33:24 2011
+++ /trunk/java2python/compiler/template.py Mon Oct 24 11:13:39 2011
@@ -426,6 +426,8 @@
yield blank
yield item
prev = item
+ for handler in self.configHandlers('PostWalk'):
+ handler(self)
head = any(self.iterHead())
body = list(super(Class, self).iterBody())
tail = () if (body or head) else [self.factory.expr(left='pass')]
=======================================
--- /trunk/java2python/config/__init__.py Mon Jul 26 18:14:27 2010
+++ /trunk/java2python/config/__init__.py Mon Oct 24 11:13:39 2011
@@ -1,21 +1,24 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
+""" java2python.config -> subpackage for run-time configuration. """
+
from functools import reduce
from imp import load_source
from os import path
class Config(object):
- """ Config -> wraps multiple configuration modules
-
- """
+ """ Config -> wraps multiple configuration modules """
+
def __init__(self, names):
self.configs = [self.load(name) for name in names]
def every(self, key, default=None):
+ """ Returns the value at the given key from each config module. """
return [getattr(config, key, default) for config in self.configs]
def last(self, key, default=None):
+ """ Returns the value at the given key from the last config module
to define it. """
for config in reversed(self.configs):
if hasattr(config, key):
return getattr(config, key)
@@ -23,7 +26,7 @@
@staticmethod
def load(name):
- """ import and return a module from dotted form or filename. """
+ """ Imports and returns a module from dotted form or filename. """
if path.exists(name):
mod = load_source(str(hash(name)), name)
else:
=======================================
--- /trunk/java2python/config/default.py Tue Oct 18 21:33:24 2011
+++ /trunk/java2python/config/default.py Mon Oct 24 11:13:39 2011
@@ -52,7 +52,7 @@
# These generators are called after a class has been completely
# generated; this specific one sorts method bodies by name.
# NB: the code generator doesn't actually use this.
-classPostWalkMutators = [
+classPostWalkHandlers = [
basic.classContentSort,
]
=======================================
--- /trunk/java2python/lang/selector.py Wed Oct 19 20:00:15 2011
+++ /trunk/java2python/lang/selector.py Mon Oct 24 11:13:39 2011
@@ -1,25 +1,42 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
-""" java2python.lang.selector -> """
+""" :mod:`java2python.lang.selector` -- Declarative AST node selection
+=======================================================================
+
+.. module:: java2python.lang.selector
+ :synopsis: Declarative AST node selection.
+.. moduleauthor:: Troy Melhase <tr...@gci.net>
+
+This module provides classes for simple AST node selection that can be
+easily combined to form complex, declarative rules for retrieving AST
+nodes.
+
+The classes are similar to CSS selectors, with a nod to Python parsing
+libraries like LEPL and PyParsing. At the moment, only a few very
+basic selector types are implemented, but those that are here already
+provide all of the functionality necessary for use within the package.
+
+Projects using java2python should regard this subpackage as
+experimental. While the interfaces are not expected to change, the
+semantics may. Use with caution.
+"""
from java2python.lang import tokens
class Selector(object):
- """ Selector -> base for selectors; provides operator methods. """
+ """ Base class for concrete selectors; provides operator methods. """
def __add__(self, other):
- """ C + C => S
-
- Produces a Sibling selector from this one and the right hand
- side of the expression.
+ """ E + F
+
+ Like CSS "E + F": an F element immediately preceded by an E element
"""
- return Sibling(self, other)
+ return AdjacentSibling(self, other)
def __and__(self, other):
- """ C & C => S
-
- Produces a Descendant selector from this one and the right
- hand side of the expression.
+ """ E & F
+
+ Like CSS "E F": an F element descendant of an E element
"""
return Descendant(self, other)
@@ -28,22 +45,26 @@
raise NotImplementedError('Selector class cannot be called.')
def __getitem__(self, key):
- """ C[n] => S
-
- Produces an Nth child selector from this one at the given key.
-
- This is where we could support attribute selectors like
C[type='void'].
+ """ E[n]
+
+ Like CSS "E:nth-child(n)": an E element, the n-th child of its
parent
"""
return Nth(self, key)
def __gt__(self, other):
- """ C > C => S
-
- Produces a Child selector from this one and the right hand
- side of the expression.
+ """ E > F
+
+ Like CSS: "E > F": an F element child of an E element
"""
return Child(self, other)
+ def __div__(self, other):
+ """ E / F
+
+ Produces a AnySibling.
+ """
+ return AnySibling(self, other)
+
def walk(self, tree):
""" Select items from the tree and from the tree children. """
for item in self(tree):
@@ -56,11 +77,24 @@
class Token(Selector):
""" Token(...) -> select tokens by matching attributes.
+ Token is the most generic and flexible Selector; using it,
+ arbitrary nodes of any type, line number, position, and/or text
+ can be selected.
+
+ Calling Token() without any keywords is equivalent to:
+
+ Token(channel=None, index=None, input=None, line=None,
+ start=None, stop=None, text=None, type=None)
+
+ Note that the value of each keyword can be a constant or a
+ callable. When callables are specified, they are passed a the
+ token and should return a bool.
"""
- # channel=None, index=None, input=None, line=None, start=None,
stop=None, text=None, type=None
def __init__(self, **attrs):
self.attrs = attrs
+ ## we support strings so that the client can refer to the
+ ## token name that way instead of via lookup or worse, integer.
if isinstance(attrs.get('type'), (basestring, )):
self.attrs['type'] = getattr(tokens, attrs.get('type'))
@@ -172,7 +206,7 @@
return 'Descendant({0} & {1})'.format(self.e, self.f)
-class Sibling(Selector):
+class AdjacentSibling(Selector):
""" E + F select any F immediately preceded by a sibling E
"""
@@ -180,16 +214,37 @@
self.e, self.f = e, f
def __call__(self, node):
- parent = node.parent
- if not parent:
+ if not node.parent:
return
for ftree in self.f(node):
- index = node.parent.children.index(ftree)
- if not index:
- return
- previous = node.parent.children[index-1]
- for child in self.e(previous):
- yield ftree
+ index = node.parent.children.index(ftree)
+ if not index:
+ return
+ previous = node.parent.children[index-1]
+ for child in self.e(previous):
+ yield ftree
def __str__(self):
- return 'Sibling({0} + {1})'.format(self.e, self.f)
+ return 'AdjacentSibling({} + {})'.format(self.e, self.f)
+
+
+class AnySibling(Selector):
+ """ E / F select any F preceded by a sibling E
+
+ """
+ def __init__(self, e, f):
+ self.e, self.f = e, f
+
+ def __call__(self, node):
+ if not node.parent:
+ return
+ for ftree in self.f(node):
+ index = node.parent.children.index(ftree)
+ if not index:
+ return
+ for prev in node.parent.children[:index]:
+ for child in self.e(prev):
+ yield ftree
+
+ def __str__(self):
+ return 'AnySibling({} / {})'.format(self.e, self.f)
=======================================
--- /trunk/java2python/mod/basic.py Tue Oct 18 21:33:24 2011
+++ /trunk/java2python/mod/basic.py Mon Oct 24 11:13:39 2011
@@ -144,6 +144,29 @@
def classContentSort(obj):
+ isMethod = lambda x:x and x.isMethod
+
+ def iterBody(body):
+ group = []
+ for value in body:
+ if isMethod(value):
+ group.append(value)
+ yield group
+ group = []
+ else:
+ group.append(value)
+ yield group
+
+ def sortBody(group):
+ methods = [item for item in group if isMethod(item)]
+ return methods[0].name if methods else -1
+
+ gx = list(iterBody(obj.children))
+ sgx = sorted(gx, key=sortBody)
+ sgx = [item for sublist in sgx for item in sublist]
+ obj.children = sgx
+
+ return
obj.children.sort(lambda x, y:-1 if x.isClass else 1)
=======================================
--- /trunk/java2python/mod/transform.py Tue Oct 18 21:33:24 2011
+++ /trunk/java2python/mod/transform.py Mon Oct 24 11:13:39 2011
@@ -1,30 +1,39 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
""" java2python.mod.transform -> input AST transformer functions and
constants. """
+
+##
+#
+# This module provides several transformation functions which are
+# simple callables that modify AST nodes. These functions are not
+# responsible for selecting nodes, only changing the node content.
+# Thus, we have AST generation decoupled from AST traversal and
+# modification.
+#
+# See the `java2python.config.default` and
+# `java2python.lang.selector` modules to understand how and when
+# selectors are associated with these callables.
+
+import keyword
import types
-from keyword import kwlist
-def renameIdents():
+def invalidPythonNames():
""" Creates a list of valid Java identifiers that are invalid in
Python. """
ts = [getattr(types, n) for n in dir(types) if not n.startswith('_')]
ns = [t.__name__ for t in ts if isinstance(t, type)]
- return ['None', 'True', 'False', ] + ns + kwlist
+ return ['None', 'True', 'False', ] + ns + keyword.kwlist
-# one and only list of identifiers that must be renamed
-renameIdents = renameIdents()
-
-
-def keywordSafeIdent(node, config):
+def keywordSafeIdent(node, config, invalid=invalidPythonNames()):
""" Validates and possibly renames a Java identifier. """
ident = node.token.text
- if ident in renameIdents:
+ if ident in invalid:
node.token.text = '%s_' % ident
def makeConst(v):
- """ Returns a closure that indiscriminately changes a node to a
value. """
+ """ Returns a closure that indiscriminately changes node text to a
value. """
def xform(node, config):
node.token.text = v
return xform
=======================================
--- /trunk/test/selector/test_all.py Wed Oct 19 20:00:15 2011
+++ /trunk/test/selector/test_all.py Mon Oct 24 11:13:39 2011
@@ -93,11 +93,17 @@
class TestSimpleSiblings(SelectorTest):
- description = 'select three IDENT nodes that are siblings of a
MODIFIER_LIST'
+ description = 'select three IDENT nodes that are adjacent siblings of
a MODIFIER_LIST'
selector = Type('MODIFIER_LIST') + Type('IDENT')
test = SelectorTest.make(3)
+class TestSimpleAnySibling(SelectorTest):
+ description = 'select three FORMAL_PARAM_LIST nodes that are
non-adjacent siblings of a MODIFIER_LIST'
+ selector = Type('MODIFIER_LIST') / Type('FORMAL_PARAM_LIST')
+ test = SelectorTest.make(2)
+
+
class TestClassIdent(SelectorTest):
description = 'select one IDENT node that is a child of a CLASS node'
selector = Type('CLASS') > Type('IDENT')