[java2python] r182 committed - Getting there...
java2...@googlecode.com
Author: troy.melhase
Date: Tue Oct 18 21:33:24 2011
Log: Getting there...
http://code.google.com/p/java2python/source/detail?r=182
Added:
/trunk/test/selector
/trunk/test/selector/Selector1.java
/trunk/test/selector/test_all.py
Modified:
/trunk/bin/j2py
/trunk/java2python/compiler/block.py
/trunk/java2python/compiler/template.py
/trunk/java2python/compiler/tool.py
/trunk/java2python/config/default.py
/trunk/java2python/lang/__init__.py
/trunk/java2python/lang/base.py
/trunk/java2python/lang/selector.py
/trunk/java2python/mod/__init__.py
/trunk/java2python/mod/basic.py
/trunk/java2python/mod/transform.py
=======================================
--- /dev/null
+++ /trunk/test/selector/Selector1.java Tue Oct 18 21:33:24 2011
@@ -0,0 +1,7 @@
+class Selector1 {
+ void bar() {
+ int foo = 3;
+ }
+
+ void foo() {}
+}
=======================================
--- /dev/null
+++ /trunk/test/selector/test_all.py Tue Oct 18 21:33:24 2011
@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from java2python.config import Config
+from java2python.compiler import buildAST
+from java2python.lang import tokens
+from java2python.lang.selector import Token, Type
+
+
+if __name__ == '__main__':
+ import sys, os
+ fn = os.path.join(os.path.dirname(__file__), 'Selector1.java')
+ source = open(fn).read()
+ tree = buildAST(source, Config(()))
+ tree.dump(sys.stdout)
+
+ selectors = [
+# Type('EXPR'),
+# Type('QUALIFIED_TYPE_IDENT') > Type('IDENT'),
+# Type('CLASS')[2],
+# Type('METHOD_CALL') & Type('IDENT'),
+# Type('IDENT') + Type('IDENT'),
+ Type('CLASS') > Type('IDENT'),
+ Type('IDENT', 'foo'),
+ Token(text='foo'),
+ Token(type='BLOCK_SCOPE', line=2) & Token(type='IDENT', text='foo')
+ ]
+
+ for index, selector in enumerate(selectors):
+ print 'Selector Test {0}:\n ==== {1}'.format(index, selector)
+ for node in selector.walk(tree):
+ name = str(node)
+ ntype = tokens.map[node.type]
+ args = (name, '') if name == ntype else (ntype, name)
+ print '{0}---- match: {1} {2}'.format(*(' '*8, )+args)
+ print '{0}---- token: {1}'.format(' '*8, node.dumps() )
+ print
=======================================
--- /trunk/bin/j2py Wed Jul 28 15:44:19 2010
+++ /trunk/bin/j2py Tue Oct 18 21:33:24 2011
@@ -137,7 +137,7 @@
try:
compile(source, '<string>', 'exec')
except (SyntaxError, ), ex:
- warning('Generated source has invalid syntax.')
+ warning('Generated source has invalid syntax. %s', ex)
else:
info('Generated source has valid syntax.')
=======================================
--- /trunk/java2python/compiler/block.py Fri Jul 30 23:35:02 2010
+++ /trunk/java2python/compiler/block.py Tue Oct 18 21:33:24 2011
@@ -12,22 +12,30 @@
# very reusable. The module split does allow for grouping of related
# methods and does hide some of the cluttered code.
+from sys import modules
from java2python.compiler import template, visitor
-def makeType(name, ftn=None):
- bases = (getattr(template, name), getattr(visitor, name))
- namespace = dict(factoryTypeName=ftn if ftn else name.lower())
- return type(name, bases, namespace)
+def newType(className, factoryTypeName):
+ """ Creates a class derived from template.className and
visitor.className """
+ bases = (getattr(template, className), getattr(visitor, className))
+ return type(className, bases, dict(factoryTypeName=factoryTypeName))
-Annotation = makeType('Annotation', 'at')
-Class = makeType('Class', 'klass')
-Comment = makeType('Comment')
-Enum = makeType('Enum')
-Expression = makeType('Expression', 'expr')
-Interface = makeType('Interface')
-Method = makeType('Method')
-MethodContent = makeType('MethodContent', 'methodContent')
-Module = makeType('Module')
-Statement = makeType('Statement')
+def addTypeToModule((cn, ftn)):
+ """ Adds a new type to this module. """
+ setattr(modules[__name__], cn, newType(cn, ftn))
+
+
+map(addTypeToModule, (
+ ('Annotation', 'at'),
+ ('Class', 'klass'),
+ ('Comment', 'comment'),
+ ('Enum', 'enum'),
+ ('Expression', 'expr'),
+ ('Interface', 'interface'),
+ ('Method', 'method'),
+ ('MethodContent', 'methodContent'),
+ ('Module', 'module'),
+ ('Statement', 'statement'),
+))
=======================================
--- /trunk/java2python/compiler/template.py Sun Oct 16 18:08:14 2011
+++ /trunk/java2python/compiler/template.py Tue Oct 18 21:33:24 2011
@@ -2,10 +2,11 @@
# -*- coding: utf-8 -*-
""" java2python.compiler.template -> Base classes for writing Python
source. """
##
-# This module defines templates, blocks of Python source code, that
-# can be easily manipulated and written. Each base provides string
-# methods (__str__, dump, dumps) for serializing instances as source
-# code. The base types also provide many utility methods.
+# This module defines templates -- blocks of Python source code --
+# that can be easily manipulated and written. Each base provides
+# string methods (__str__, dump, dumps) for serializing instances as a
+# source code string. The base types also provide many utility
+# methods.
#
# The Factory class is used to to provide runtime lookup of concrete
# classes; this was necessary to accommodate splitting the behavior of
@@ -13,9 +14,9 @@
# are usually a sign of a bad design and/or language limitations, and
# this case is no exception.
+from cStringIO import StringIO
from functools import partial
from itertools import chain, ifilter, imap
-from StringIO import StringIO
from java2python.lang import tokens
from java2python.lib import FS
@@ -23,7 +24,24 @@
class Factory(object):
- """ Factory -> creates pre-configured callables for new block
instances. """
+ """ Factory -> creates pre-configured callables for new block
instances.
+
+ The templates use an instance of this class as a quick and simple
+ interface to create new templates like this:
+
+ stat = self.factory.statement()
+
+ The `__getattr__` method does the work of looking up and returning
+ the appropriate template class. The lookup depends on the types
+ registry, which is populated by the FactoryTypeDetector metaclass
+ below.
+
+ The important thing to realize regarding this factory is this:
+ when an attribute is requested (`self.factory.expr` for example),
+ the factory locates the type and returns a constructor for it with
+ the config object pre-applied.
+
+ """
types = {}
def __init__(self, config):
@@ -39,6 +57,16 @@
class FactoryTypeDetector(type):
""" FactoryTypeDetector -> detects factory-creatable types as they are
defined.
+ As subclasses are created they are checked for an attribute called
+ `factoryTypeName`. If present, that key is used to populate the
+ factory type registry above.
+
+ Note that the actual subclasses are not created here (none of
+ these specify a `factoryTypeName`). Actual factory types are
+ created in `java2python.compiler.block`. This is because we're
+ after not templates, but visitors combined with templates, aka
+ blocks. Refer to the `blocks` module for the specific factory
+ type names.
"""
def __init__(cls, name, bases, namespace):
try:
@@ -50,6 +78,38 @@
class Base(object):
""" Base -> base class for formatting Python output.
+ This class defines a large set of attributes and methods for the
+ other concrete templates defined below. The items defined here
+ can be grouped as follows:
+
+ * References
+
+ This class defines `bases`, `children`, `decorators`, etc. for
+ tracking the relationship between this instance and other blocks.
+
+ * Type Information
+
+ This class defines many is-A properties, such as isClass,
+ isModule, isVoid, etc. Subclasses typically override one or more
+ of these with an attribute or property.
+
+ * Configuration
+
+ This class provides utility methods for retrieving values from the
+ run-time configuration. See the definition of `configHandler` and
+ `configHandlers` for details.
+
+ * Serialization
+
+ This class provides a default implementation for subclasses to
+ serialize their instances as Python source code strings. Notably,
+ the `__str__` method is provided, which in turn defers most of its
+ work to the `dumps` method. Subclasses provide different
+ implementations of these methods where needed.
+
+ Also, the `__repr__` method is defined by this class for printing
+ a the template as tree for debugging.
+
"""
__metaclass__ = FactoryTypeDetector
isAnnotation = isClass = isComment = isEnum = isExpression = \
@@ -92,9 +152,7 @@
def altIdent(self, name):
""" Returns an alternate identifier for the one given. """
- #print '## looking for name:', name, 'parent count:', [(type(x),
type(x.parent)) for x in self.parents()]
for klass in self.parents(lambda v:v.isClass):
- #print '#### looking inside', klass.name, klass.variables
if name in klass.variables:
try:
method = self.parents(lambda v:v.isMethod).next()
@@ -105,7 +163,6 @@
if name in method.variables:
return name
return ('cls' if method.isStatic else 'self') + '.' + name
- #print
return name
def configHandler(self, part, suffix='Handler', default=None):
@@ -282,12 +339,6 @@
"""
isComment = True
- def __init__(self, config, left='', right='', fs=FS.lr, parent=None,
tail=''):
- super(Comment, self).__init__(config, left, right, fs, parent, tail)
- if False:# not fs.strip().startswith('#'): # wha?
- prefix = self.config.last('commentPrefix', '# ')
- self.fs = prefix + self.fs
-
def __repr__(self):
""" Returns the debug string representation of this comment. """
parts = [white(self.typeName+':'),
=======================================
--- /trunk/java2python/compiler/tool.py Sat Oct 15 15:17:26 2011
+++ /trunk/java2python/compiler/tool.py Tue Oct 18 21:33:24 2011
@@ -6,9 +6,7 @@
#
# $ python -m java2python.compiler.tool ./SomeClass.java
-from java2python.lang import (
- Lexer, Parser, StringStream, TokenStream, TreeAdaptor,
walkTreeSelector,
- )
+from java2python.lang import Lexer, Parser, StringStream, TokenStream,
TreeAdaptor
def buildAST(source, config=None):
@@ -22,7 +20,7 @@
def transformAST(tree, config):
for selector, call in config.last('astTransforms', ()):
- for node in walkTreeSelector(tree, selector):
+ for node in selector.walk(tree):
call(node, config)
=======================================
--- /trunk/java2python/config/default.py Sun Oct 16 18:08:14 2011
+++ /trunk/java2python/config/default.py Tue Oct 18 21:33:24 2011
@@ -1,8 +1,9 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
+##
# This is the default configuration file for java2python. Unless
-# explicity disabled with the '-n' or '--nodefaults' options, the j2py
+# explicity disabled with the '-n' or '--nodefaults' option, the j2py
# script will import this module for runtime configuration.
from java2python.mod import basic, transform
@@ -11,7 +12,7 @@
# Leading indent character or characters. Four spaces are used
# because that is the recommendation of PEP 8.
-indentPrefix = ' '*4
+indentPrefix = ' ' * 4
# Prefix character or characters for comments. The hash+space is
@@ -19,34 +20,38 @@
commentPrefix = '# '
-# These values are strings or generator-functions that yield strings
+# These values are strings or generators that yield strings
# for a module prologue.
modulePrologueHandlers = [
- basic.simpleShebang,
+ basic.shebangLine,
basic.simpleDocString,
basic.maybeBsr,
]
-# These generator-functions yield lines for a module epilogue.
+# These generators yield lines for a module epilogue.
moduleEpilogueHandlers = [
basic.scriptMainStanza,
]
-# These generator-functions yield (possibly modified) source strings
-# for a module. The 'outputSubs' handler references the list of
+# These generators yield (possibly modified) source strings for a
+# module. The `basic.outputSubs` handler references the list of
# regular expression substitutions near the end of this module.
moduleOutputHandlers = [
basic.outputSubs,
]
+# These generators yield doc strings for a class.
classHeadHandlers = [
basic.simpleDocString,
]
+# These generators are called after a class has been completely
+# generated; this specific one sorts method bodies by name.
+# NB: the code generator doesn't actually use this.
classPostWalkMutators = [
basic.classContentSort,
]
@@ -81,7 +86,7 @@
# This handler creates enum values on enum classes after they've been
-# defined. The handler tries to matches Java semantics by using
+# defined. The handler tries to match Java semantics by using
# strings. Refer to the documentation for details.
enumValueHandler = basic.enumConstStrings
@@ -106,9 +111,13 @@
# modulePackageDeclarationHandler = basic.namespacePackages
-moduleImportDeclarationHandler = basic.commentedImports
+# This handler is turns java imports into python imports. No mapping
+# of packages is performed:
moduleImportDeclarationHandler = basic.simpleImports
+# This import decl. handler can be used instead to produce comments
+# instead of import statements:
+# moduleImportDeclarationHandler = basic.commentedImports
# The AST transformation function uses these declarations to modify an
# AST before compiling it to python source. Having these declarations
@@ -172,6 +181,3 @@
'double' : 'float',
'java.lang.String' : 'str',
}
-
-
-
=======================================
--- /trunk/java2python/lang/__init__.py Fri Jul 30 23:35:02 2010
+++ /trunk/java2python/lang/__init__.py Tue Oct 18 21:33:24 2011
@@ -4,4 +4,4 @@
from java2python.lang.JavaLexer import JavaLexer as Lexer
from java2python.lang.JavaParser import JavaParser as Parser
from java2python.lang.base import StringStream, TokenStream, TreeAdaptor,
tokens
-from java2python.lang.selector import walkTreeSelector
+
=======================================
--- /trunk/java2python/lang/base.py Thu Aug 5 14:17:51 2010
+++ /trunk/java2python/lang/base.py Tue Oct 18 21:33:24 2011
@@ -1,5 +1,26 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
+""" java2python.lang.base -> lexer and parser support classes. """
+##
+# This module provides the following:
+#
+# * `Tokens`
+#
+# This class is used to create the single `token` instance in this
+# module. It is used to map between parser tokens and their ids and
+# vice-versa.
+#
+# * `TreeAdaptor`
+#
+# This class is used by `java2python.compiler.tool`, where the
+# `buildAST` function associates an instance of it to a parser. The
+# `TreeAdaptor` class creates `LocalTree` instances.
+#
+# * `LocalTree`
+#
+# This class provides a slew of extra utility methods that are useful
+# when inspecting and printing tree nodes.
+#
##
# ANTLR notes:
@@ -25,9 +46,12 @@
# Tree objects. Our adaptor, TreeAdaptor, creates the LocalTree
# instances.
#
-##
+
+from cStringIO import StringIO
+
from antlr3 import ANTLRStringStream as StringStream, CommonTokenStream as
TokenStream
from antlr3.tree import CommonTreeAdaptor, CommonTree
+
from java2python.lib import colortools
@@ -143,10 +167,15 @@
token, indent = root.token, ' ' * offset
start, stop = root.tokenStartIndex, root.tokenStopIndex
idxes, ttyp = '', tokens.map.get(token.type, '?')
+ line = token.line
if start and stop and start == stop:
- idxes = ' [{0}]'.format(start)
+ idxes = 'start={}'.format(start)
elif start and stop:
- idxes = ' [{0}:{1}]'.format(start, stop)
+ idxes = 'start={}, stop={}'.format(start, stop)
+ if line:
+ idxes = 'line={}{}{}'.format(line, ', ' if idxes else '',
idxes)
+ idxes = ' [{}]'.format(idxes) if idxes else ''
+ idxes = colortools.black(idxes)
args = [indent, self.colorType(ttyp), '', idxes, '']
if extras(token.text, ttyp):
args[2] = ' ' + self.colorText(ttyp, token.text)
@@ -161,6 +190,12 @@
print >> fd, '{0}{1}'.format(indent, line)
innerDump(self, level)
+ def dumps(self, level=0):
+ """ Dump this token to a string. """
+ fd = StringIO()
+ self.dump(fd, level)
+ return fd.getvalue()
+
def dupNode(self):
""" Called by the parser to create a duplicate of this tree. """
get = lambda v:getattr(self, v, None)
=======================================
--- /trunk/java2python/lang/selector.py Mon Jul 19 15:02:15 2010
+++ /trunk/java2python/lang/selector.py Tue Oct 18 21:33:24 2011
@@ -1,5 +1,6 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
+""" java2python.lang.selector -> """
from java2python.lang import tokens
@@ -8,21 +9,52 @@
"""
def __add__(self, other):
+ """ C + C => S
+
+ Produces a Sibling selector from this one and the right hand
+ side of the expression.
+ """
return Sibling(self, other)
def __and__(self, other):
+ """ C & C => S
+
+ Produces a Descendant selector from this one and the right
+ hand side of the expression.
+ """
return Descendant(self, other)
def __getitem__(self, key):
+ """ C[n] => S
+
+ Produces an Nth child selector from this one at the given key.
+
+ This is where we could support attribute selectors like
C[type='void'].
+ """
return Nth(self, key)
def __gt__(self, other):
+ """ C > C => S
+
+ Produces a Child selector from this one and the right hand
+ side of the expression.
+ """
return Child(self, other)
+ def walk(self, tree):
+ for item in self(tree):
+ yield item
+ for child in tree.children:
+ for item in self.walk(child):
+ yield item
+
+
class Nth(Selector):
- """ E[n] match any slice n of E
-
+ """ E[n] -> match any slice n of E
+
+ Similar to the :nth-child pseudo selector in CSS, but without the
+ support for keywords like 'odd', 'even', etc.
"""
def __init__(self, e, key):
self.e, self.key = e, key
@@ -61,6 +93,7 @@
class Type(Selector):
""" Type(T) select any token of type T
+ Similar to the type selector in CSS.
"""
def __init__(self, key, value=None):
self.key = key if isinstance(key, int) else getattr(tokens, key)
@@ -72,13 +105,37 @@
yield tree
def __str__(self):
- ## TODO: add value
- return 'Type({0}:{1})'.format(tokens.map[self.key], self.key)
+ val = '' if self.value is None else '={0}'.format(self.value)
+ return 'Type({0}{1}:{2})'.format(tokens.map[self.key], val, self.key)
+
+
+class Token(Selector):
+ """ Token(T) select any token by matching attributes.
+
+ Similar to the type selector in CSS.
+ """
+ # channel=None, index=None, input=None, line=None, start=None,
stop=None, text=None, type=None
+
+ def __init__(self, **attrs):
+ self.attrs = attrs
+ if isinstance(attrs.get('type'), (basestring, )):
+ self.attrs['type'] = getattr(tokens, attrs.get('type'))
+
+ def __call__(self, tree):
+ items = self.attrs.items()
+ if all(getattr(tree.token, k)==v for k, v in items if v is not
None):
+ yield tree
+
+ def __str__(self):
+ items = self.attrs.items()
+ keys = ('{}={}'.format(k, v) for k, v in items if v is not None)
+ return 'Token({})'.format(', '.join(keys))
class Star(Selector):
""" * select any
+ Similar to the * selector in CSS.
"""
def __call__(self, tree):
yield tree
@@ -127,39 +184,3 @@
def __str__(self):
return 'Sibling({0} + {1})'.format(self.e, self.f)
-
-
-def walkTreeSelector(tree, selector):
- for item in selector(tree):
- yield item
- for child in tree.children:
- for item in walkTreeSelector(child, selector):
- yield item
-
-
-if __name__ == '__main__':
- import sys
- from java2python.config import Config
- from java2python.compiler import buildAST
-
- source = open(sys.argv[1]).read()
- tree = buildAST(source, Config(()))
- tree.dump(sys.stdout)
-
-
- selectors = [
- Type('EXPR'),
- Type('QUALIFIED_TYPE_IDENT') > Type('IDENT'),
- Type('CLASS')[2],
- Type('METHOD_CALL') & Type('IDENT'),
- Type('IDENT') + Type('IDENT'),
- ]
-
- for index, selector in enumerate(selectors):
- print '{0}: {1}\n ==== {2}'.format(index, selector.__doc__.strip(),
selector)
- for node in walkTreeSelector(tree, selector):
- name = str(node)
- ntype = tokens.map[node.type]
- args = (name, '') if name == ntype else (ntype, name)
- print '{0}---- match: {1} {2}'.format(*(' '*8, )+args)
- print
=======================================
--- /trunk/java2python/mod/__init__.py Sat Jul 24 01:19:21 2010
+++ /trunk/java2python/mod/__init__.py Tue Oct 18 21:33:24 2011
@@ -1,2 +1,19 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
+
+##
+# java2python.mod -> provides simple modification routines for the
+# library and projects using it.
+#
+# the `java2python.mod.basic` module contains various functions for
+# sprinkling generated source with docstrings, comments, decorators,
+# etc.
+#
+# the `java2python.mod.inclues` module contains functions that the
+# library will include directly -- as source code -- in the generated
+# output.
+#
+# the `java2python.mod.transform` module contains values and functions
+# for transforming input AST nodes.
+
+
=======================================
--- /trunk/java2python/mod/basic.py Thu Aug 12 11:20:01 2010
+++ /trunk/java2python/mod/basic.py Tue Oct 18 21:33:24 2011
@@ -1,16 +1,34 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
+""" java2python.mod.basic -> functions to revise generated source
strings. """
from itertools import count
from logging import info, warn
from os import path
from re import sub as rxsub
-def simpleShebang(module):
+def shebangLine(module):
+ """ yields the canonical python shebang line. """
yield '#!/usr/bin/env python'
+def encodingLine(encoding='utf-8'):
+ """ returns a function to yield the specified encoding line.
+
+ Note that this function isn't wired up because the encoding is
+ specified for the source directly, and adding this line produces a
+ syntax error when the compile function is used.
+ """
+ def line(module):
+ yield '# -*- coding: {0} -*-'.format(encoding)
+ return line
+
+
def simpleDocString(obj):
+ """ yields multiple lines for a default docstring.
+
+ This generator works for modules, classes, and functions.
+ """
yield '""" generated source for {0} {1}'.format(obj.typeName, obj.name)
yield ''
yield '"""'
=======================================
--- /trunk/java2python/mod/transform.py Sat Jul 24 01:19:21 2010
+++ /trunk/java2python/mod/transform.py Tue Oct 18 21:33:24 2011
@@ -1,37 +1,44 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
-
+""" java2python.mod.transform -> input AST transformer functions and
constants. """
import types
from keyword import kwlist
def renameIdents():
- typs = [getattr(types, n) for n in dir(types) if not n.startswith('_')]
- names = [t.__name__ for t in typs if isinstance(t, type)]
- return ['None', 'True', 'False', ] + names + kwlist
+ """ Creates a list of valid Java identifiers that are invalid in
Python. """
+ ts = [getattr(types, n) for n in dir(types) if not n.startswith('_')]
+ ns = [t.__name__ for t in ts if isinstance(t, type)]
+ return ['None', 'True', 'False', ] + ns + kwlist
+# one and only list of identifiers that must be renamed
renameIdents = renameIdents()
def keywordSafeIdent(node, config):
+ """ Validates and possibly renames a Java identifier. """
ident = node.token.text
if ident in renameIdents:
node.token.text = '%s_' % ident
-def constXform(v):
+def makeConst(v):
+ """ Returns a closure that indiscriminately changes a node to a
value. """
def xform(node, config):
node.token.text = v
return xform
-null2None = constXform('None')
-false2False = constXform('False')
-true2True = constXform('True')
+# create transformers for mapping well-known Java idents into their
+# Python counterparts.
+null2None = makeConst('None')
+false2False = makeConst('False')
+true2True = makeConst('True')
def syntaxSafeFloatLiteral(node, config):
+ """ Ensures a Java float literal is a valid Python float literal. """
value = node.token.text
if value.startswith('.'):
value = '0' + value
@@ -43,6 +50,11 @@
def typeSub(node, config):
+ """ Maps specific, well-known Java types to their Python counterparts.
+
+ See the `java2python.config.default` module for the default type
+ mapping and further discussion.
+ """
ident = node.token.text
subs = config.last('typeSubs')
if ident in subs: