[java2python] r174 committed - work in progress.
codesite...@google.com
Author: troy.melhase
Date: Fri Jul 30 23:35:02 2010
Log: work in progress.
http://code.google.com/p/java2python/source/detail?r=174
Added:
/branches/0.5/test/configs/Interface1.py
Modified:
/branches/0.5/java2python/compiler/__init__.py
/branches/0.5/java2python/compiler/block.py
/branches/0.5/java2python/compiler/template.py
/branches/0.5/java2python/compiler/visitor.py
/branches/0.5/java2python/config/default.py
/branches/0.5/java2python/lang/Java.g
/branches/0.5/java2python/lang/__init__.py
/branches/0.5/java2python/lang/base.py
/branches/0.5/java2python/mod/basic.py
/branches/0.5/test/Makefile
/branches/0.5/test/runj2py
=======================================
--- /dev/null
+++ /branches/0.5/test/configs/Interface1.py Fri Jul 30 23:35:02 2010
@@ -0,0 +1,7 @@
+
+
+from java2python.config.default import modulePrologueHandlers
+
+modulePrologueHandlers += [
+ "from abc import ABCMeta, abstractmethod",
+ ]
=======================================
--- /branches/0.5/java2python/compiler/__init__.py Mon Jul 26 18:14:27 2010
+++ /branches/0.5/java2python/compiler/__init__.py Fri Jul 30 23:35:02 2010
@@ -3,24 +3,14 @@
from java2python.compiler.block import Module
from java2python.lang import (
- Lexer, Parser,
- LocalSourceStream, LocalTokenStream, LocalTreeAdaptor,
- walkTreeSelector,
+ Lexer, Parser, StringStream, TokenStream, TreeAdaptor,
walkTreeSelector,
)
-
-def makeRecognizersCallback(lexer, parser):
- def setRecognizers(node):
- node.lexer = lexer
- node.parser = parser
- return setRecognizers
-
-
def buildAST(source, config=None):
- lexer = Lexer(LocalSourceStream(source))
- parser = Parser(LocalTokenStream(lexer))
- adapter = LocalTreeAdaptor(makeRecognizersCallback(lexer, parser))
+ lexer = Lexer(StringStream(source))
+ parser = Parser(TokenStream(lexer))
+ adapter = TreeAdaptor(lexer, parser)
parser.setTreeAdaptor(adapter)
scope = parser.javaSource()
return scope.tree
@@ -35,8 +25,8 @@
def buildJavaDocAST(source):
from java2python.lang.JavaDocLexer import JavaDocLexer
from java2python.lang.JavaDocParser import JavaDocParser
- lexer = JavaDocLexer(LocalSourceStream(source))
- parser = JavaDocParser(LocalTokenStream(lexer))
+ lexer = JavaDocLexer(StringStream(source))
+ parser = JavaDocParser(TokenStream(lexer))
scope = parser.commentBody()
return scope.tree
=======================================
--- /branches/0.5/java2python/compiler/block.py Sat Jul 24 12:54:43 2010
+++ /branches/0.5/java2python/compiler/block.py Fri Jul 30 23:35:02 2010
@@ -5,13 +5,13 @@
# This module defines classes which combine AST walking with source
# generation. We've put these two behaviors into separate modules,
# java2python.compiler.template for creating source code, and
-# java2python.compiler.visitor for walking Antlr trees.
+# java2python.compiler.visitor for walking ANTLR trees.
#
# Each of the classes depends on the behavior of its counterpart.
# This means they're very tightly coupled and that the classes are not
# very reusable. The module split does allow for grouping of related
# methods and does hide some of the cluttered code.
-#
+
from java2python.compiler import template, visitor
=======================================
--- /branches/0.5/java2python/compiler/template.py Wed Jul 28 15:44:19 2010
+++ /branches/0.5/java2python/compiler/template.py Fri Jul 30 23:35:02 2010
@@ -2,19 +2,19 @@
# -*- coding: utf-8 -*-
""" java2python.compiler.template -> Base classes for writing Python
source. """
##
-# This module defines base template types. Each base class provides
-# string representation methods (__str__, __repr__, dump, dumps) for
-# serializing instances as source code. The base types also provide
-# many utility methods.
-#
-# The Factory class is used by the BaseTemplate class to provide
-# runtime lookup of concrete classes; this was necessary to
-# accommodate splitting the behavior of the blocks package into
-# multiple modules. So-called patterns are usually a sign of a bad
-# design and/or language limitations, and this case is no exception.
-#
+# This module defines templates, blocks of Python source code, that
+# can be easily manipulated and written. Each base provides string
+# methods (__str__, dump, dumps) serializing instances as source code.
+# The base types also provide many utility methods.
+#
+# The Factory class is used to to provide runtime lookup of concrete
+# classes; this was necessary to accommodate splitting the behavior of
+# the compiler subpackage into multiple modules. So-called patterns
+# are usually a sign of a bad design and/or language limitations, and
+# this case is no exception.
+
from functools import partial
-from itertools import chain, ifilter
+from itertools import chain, ifilter, imap
from StringIO import StringIO
from java2python.lang import tokens
@@ -52,7 +52,8 @@
"""
__metaclass__ = FactoryTypeDetector
- isClass = isComment = isExpression = isMethod = isModule = False
+ isAnnotation = isClass = isComment = isEnum = isExpression = \
+ isInterface = isMethod = isModule = isStatement = False
def __init__(self, config, name=None, type=None, parent=None):
self.bases = []
@@ -68,12 +69,6 @@
self.variables = []
if parent:
parent.children.append(self)
- if self.isMethod:
- self.parameters.append(self.makeParam('self', 'object'))
-
- def insertChild(self, child, index=-1):
- self.children.insert(index, child)
- child.parent = self
def __repr__(self):
""" Returns the debug string representation of this template. """
@@ -90,6 +85,11 @@
handlers = self.configHandlers('Output')
return reduce(lambda v, func:func(self, v), handlers, self.dumps(-1))
+ def adopt(self, child, index=-1):
+ """ Adds child to this objecs children and sets the childs parent. """
+ self.children.insert(index, child)
+ child.parent = self
+
def altIdent(self, name):
""" Returns an alternate identifier for the one given. """
for klass in self.parents(lambda v:v.isClass):
@@ -108,29 +108,21 @@
def configHandlers(self, part, suffix='Handlers'):
""" Returns config handlers for this type of template """
name = '{0}{1}{2}'.format(self.typeName, part, suffix)
- for handler in self.config.last(name, ()):
- if isinstance(handler, (basestring, )):
- def wrapper(*args, **kwds):
- yield handler
- yield wrapper
- else:
- yield handler
-
- def configTransformers(self, visitor):
- name = visitor.factoryTypeName
- name = '{0}Transformers'.format(name[0].lower() + name[1:])
- return name, self.config.handlers(name, ())
+ return imap(self.toIter, self.config.last(name, ()))
def dump(self, fd, level=0):
""" Writes the Python source code for this template to the given file. """
indent, isNotNone = level * self.indent, lambda x:x is not None
+ lineFormat = '{0}{1}\n'.format
for line in ifilter(isNotNone, self.iterPrologue()):
- line = '{0}{1}\n'.format(indent, line)
+ line = lineFormat(indent, line)
fd.write(line if line.strip() else '\n')
+ for item in ifilter(isNotNone, self.iterHead()):
+ item.dump(fd, level+1)
for item in self.iterBody():
item.dump(fd, level+1)
for line in ifilter(isNotNone, self.iterEpilogue()):
- line = '{0}{1}\n'.format(indent, line)
+ line = lineFormat(indent, line)
fd.write(line if line.strip() else '\n')
def dumps(self, level=0):
@@ -168,7 +160,12 @@
def iterPrologue(self):
""" Yields the items in the prologue of this template. """
- yield None
+ return chain(*(h(self) for h in self.configHandlers('Prologue')))
+
+ def iterHead(self):
+ """ Yields the items in the head of this template. """
+ items = chain(*(h(self) for h in self.configHandlers('Head')))
+ return imap(self.toExpr, items)
def iterBody(self):
""" Yields the items in the body of this template. """
@@ -176,7 +173,7 @@
def iterEpilogue(self):
""" Yields the items in the epilogue of this template. """
- yield None
+ return chain(*(h(self) for h in self.configHandlers('Epilogue')))
def makeParam(self, name, type):
""" Creates a parameter as a mapping. """
@@ -189,20 +186,43 @@
yield self
self = self.parent
+ @property
+ def className(self):
+ """ Returns the name of the class of this item. """
+ return self.__class__.__name__
+
@property
def typeName(self):
""" Returns the name of this template type. """
- return self.__class__.__name__.lower()
-
-
-class BaseExpression(Base):
- """ BaseExpression -> base class for formatting Python expressions.
+ return self.className.lower()
+
+ def toExpr(self, value):
+ """ Returns an expression for the given value if it is a string. """
+ try:
+ return self.factory.expr(left=value+'')
+ except (TypeError, ):
+ return value
+
+ def toIter(self, value):
+ """ Returns an iterator for the given value if it is a string. """
+ try:
+ value + ''
+ except (TypeError, ):
+ return value
+ else:
+ def wrapper(*a, **b):
+ yield value
+ return wrapper
+
+
+class Expression(Base):
+ """ Expression -> formatting for Python expressions.
"""
isExpression = True
def __init__(self, config, left='', right='', fs=FS.lr, parent=None,
tail=''):
- super(BaseExpression, self).__init__(config, parent=parent)
+ super(Expression, self).__init__(config, parent=parent)
self.left, self.right, self.fs, self.tail = left, right, fs, tail
def __repr__(self):
@@ -217,7 +237,7 @@
if showfs:
parts.append(white('format:') + yellow(self.fs))
if self.tail:
- parts.append(white('tail:') + yellow(self.tail))
+ parts.append(white('tail:') + black(self.tail))
return ' '.join(parts)
def __str__(self):
@@ -240,17 +260,20 @@
def isComment(self):
""" True if this expression is a comment. """
try:
- return self.left.startswith('#')
+ return self.left.strip().startswith('#')
except (AttributeError, ):
return False
-class BaseComment(BaseExpression):
- """ BaseComment -> base class for formatting Python comments. """
+class Comment(Expression):
+ """ Comment -> formatting for Python comments.
+
+ """
+ isComment = True
def __init__(self, config, left='', right='', fs=FS.lr, parent=None,
tail=''):
- super(BaseComment, self).__init__(config, left, right, fs, parent, tail)
- if not fs.strip().startswith('#'):
+ super(Comment, self).__init__(config, left, right, fs, parent, tail)
+ if False:# not fs.strip().startswith('#'): # wha?
prefix = self.config.last('commentPrefix', '# ')
self.fs = prefix + self.fs
@@ -262,11 +285,14 @@
-class BaseStatement(Base):
- """ BaseStatement -> base class for formatting Python statements. """
+class Statement(Base):
+ """ Statement -> formatting for Python statements.
+
+ """
+ isStatement = True
def __init__(self, config, keyword, fs=FS.lr, parent=None):
- super(BaseStatement, self).__init__(config, parent=parent)
+ super(Statement, self).__init__(config, parent=parent)
self.keyword = keyword
self.expr = self.factory.expr(left=keyword, fs=fs)
self.expr.parent = self
@@ -276,100 +302,87 @@
parts = [green(self.typeName), white('keyword:')+cyan(self.keyword)]
return ' '.join(parts)
+ def iterPrologue(self):
+ """ Yields the keyword (and clause, if any) for this statement . """
+ yield self.expr
+
class Module(Base):
- """ Module -> formatting for Python modules. """
+ """ Module -> formatting for Python modules.
+
+ """
isModule = True
- def iterPrologue(self):
- """ Yields the items in the prologue of this template. """
- return chain(*(h(self) for h in self.configHandlers('Prologue')))
-
def iterBody(self):
""" Yields the items in the body of this template. """
- get = lambda o, n:getattr(o, n, None)
- children, prev = self.children, None
- blank = self.factory.expr(left='\n')
- def pred(c, p):
- return p and get(c, 'isClass') and not get(c, 'isComment') \
- and not get(p, 'isClass')
- for child in children:
- if pred(child, prev):
+ blank, prev = self.factory.expr(), None
+ def pred(p, c):
+ return (p and not p.isClass) and (c.isClass and not c.isComment)
+ for child in super(Module, self).iterBody():
+ if pred(prev, child):
yield blank
yield child
prev = child
- def iterEpilogue(self):
- """ Yields the items in the epilogue of this template. """
- return chain(*(h(self) for h in self.configHandlers('Epilogue')))
-
-
-##
-# class and class-like types
+
+class ClassMethodSharedMixin(object):
+ """ ClassMethodSharedMixin -> shared methods for Class and Method
types. """
+
+ def iterPrologue(self):
+ """ Yields the items in the prologue of this template. """
+ prologue = super(ClassMethodSharedMixin, self).iterPrologue()
+ return chain(prologue, self.decorators, self.iterDecl())
-class Class(Base):
- """ Class -> formatting for Python classes. """
+class Class(ClassMethodSharedMixin, Base):
+ """ Class -> formatting for Python classes.
+
+ """
isClass = True
- def iterDecorators(self):
- """ Yields decorators for this method. """
- def iterLines():
- return chain(*(h(self) for h in self.configHandlers('Prologue')))
- def iterOtherDecos():
- return chain(*(h(self) for h in
self.configHandlers('ExtraDecorator')))
- return chain(self.decorators, iterOtherDecos(), iterLines())
-
- def iterPrologue(self):
- """ Yields the items in the prologue of this template. """
- def iterBases():
- return chain(*(h(self) for h in self.configHandlers('Base')))
- bases = ', '.join(ifilter(None, iterBases()))
+ def iterBases(self):
+ """ Yields the base classes for this type. """
+ return iter(self.bases or ['object'])
+
+ def iterDecl(self):
+ """ Yields the declaration for this type. """
+ bases = ', '.join(self.iterBases())
bases = '({0})'.format(bases) if bases else ''
- for deco in self.iterDecorators():
- yield '@{0}'.format(deco)
yield 'class {0}{1}:'.format(self.name, bases)
def iterBody(self):
""" Yields the items in the body of this template. """
- def iterDocString():
- for handler in self.configHandlers('DocString'):
- for line in handler(self):
- yield self.factory.expr(left=line)
- def intersperseLines(lines):
- get = lambda o, n:getattr(o, n, None)
+ def sprinkleBlanks(body):
blank, prev = self.factory.expr(), None
- for item in lines:
- if type(item) != type(prev) and prev and not get(prev, 'isComment'):
- yield blank
- elif get(item, 'isMethod') and get(prev, 'isMethod'):
- yield blank
- elif get(prev, 'isClass'):
- yield blank
+ for item in body:
+ if prev:
+ if type(prev) != type(item) and not prev.isComment:
+ yield blank
+ elif item.isMethod and prev.isMethod:
+ yield blank
+ elif prev.isClass:
+ yield blank
yield item
prev = item
+ head = any(self.iterHead())
body = list(super(Class, self).iterBody())
- if self.config.last('reorderClassDefs'):
- body.sort(lambda x, y:-1 if x.isClass else 1)
- docs = list(iterDocString())
- more = [self.factory.expr(left='pass')] if not (body or docs) else []
- return chain(docs, iter(body) if more else intersperseLines(body), more)
-
- def iterEpilogue(self):
- """ Yields the items in the epilogue of this template. """
- return chain(*(h(self) for h in self.configHandlers('Epilogue')))
+ tail = () if (body or head) else [self.factory.expr(left='pass')]
+ body = () if tail else sprinkleBlanks(body)
+ return chain(body, tail)
class Annotation(Class):
- """ Annotation -> formatting for annotations converted to Python
classes. """
+ """ Annotation -> formatting for annotations converted to Python
classes.
+
+ """
+ isAnnotation = True
+
def __init__(self, config, name=None, type=None, parent=None):
super(Annotation, self).__init__(config, name, type, parent)
-
m = self.factory.method(parent=self, name='__init__')
m.parameters.append(self.makeParam('*args', 'list'))
m.parameters.append(self.makeParam('**kwds', 'dict'))
## set attributes from kwds?
-
m = self.factory.method(parent=self, name='__call__')
m.parameters.append(self.makeParam('klass', 'type'))
self.factory.expr(parent=m, fs='setattr(klass, self.__class__.__name__,
self)')
@@ -377,74 +390,41 @@
class Enum(Class):
- """ Enum -> formatting for enums converted to Python classes. """
-
-class Interface(Class):
- """ Interface -> formatting for interfaces converted to Python
classes. """
+ """ Enum -> formatting for enums converted to Python classes.
+
+ """
+ isEnum = True
-##
-# method and method content types
+class Interface(Class):
+ """ Interface -> formatting for interfaces converted to Python classes.
+
+ """
+ isInterface = True
class MethodContent(Base):
- """ """
-
-class Method(Base):
- """ MethodTemplte -> formatting for Python methods. """
- isMethod = True
-
- def iterDecorators(self):
- """ Yields decorators for this method. """
- def iterLines():
- return chain(*(h(self) for h in self.configHandlers('Prologue')))
- def iterOtherDecos():
- return chain(*(h(self) for h in
self.configHandlers('ExtraDecorator')))
- return chain(self.decorators, iterOtherDecos(), iterLines())
-
- def iterBody(self):
- """ Yields the items in the body of this template. """
- def iterDocString():
- for handler in self.configHandlers('DocString'):
- for line in handler(self):
- yield self.factory.expr(left=line)
- body = list(super(Method, self).iterBody())
- docs = list(iterDocString())
- more = [self.factory.expr(left='pass')] if not (body or docs) else []
- return chain(docs, body, more)
-
- def iterPrologue(self):
- """ Yields items in this method declaration, maybe with decorators. """
- for deco in self.iterDecorators():
- yield '@{0}'.format(deco)
- params = ', '.join(p['name'] for p in self.parameters)
+ """ MethodContent -> formatting for content within Python methods. """
+
+
+class Method(ClassMethodSharedMixin, Base):
+ """ Method -> formatting for Python methods.
+
+ """
+ isMethod = True
+
+ def __init__(self, config, name=None, type=None, parent=None):
+ super(Method, self).__init__(config, name, type, parent)
+ self.parameters.append(self.makeParam('self', 'object'))
+
+ def iterDecl(self):
+ """ Yields the declaration for this method template. """
+ params = ', '.join(param['name'] for param in self.parameters)
yield 'def {0}({1}):'.format(self.name, params)
-
-
-##
-# expression and comment types
-
-
-class Expression(BaseExpression):
- """ Expression -> formatting for Python expressions. """
-
-
-class Comment(BaseComment):
- """ """
-
-
-
-
-##
-# statement types
-
-
-class Statement(BaseStatement):
- """ Statement -> formatting for Python statements. """
-
- def iterPrologue(self):
- """ Yields the keyword (and clause, if any) for this statement . """
- yield self.expr
-
-
+ def iterBody(self):
+ """ Yields the items in the body of this method template. """
+ head = any(self.iterHead())
+ body = list(super(Method, self).iterBody())
+ tail = () if (body or head) else [self.factory.expr(left='pass')]
+ return chain(body, tail)
=======================================
--- /branches/0.5/java2python/compiler/visitor.py Wed Jul 28 15:44:19 2010
+++ /branches/0.5/java2python/compiler/visitor.py Fri Jul 30 23:35:02 2010
@@ -2,11 +2,8 @@
# -*- coding: utf-8 -*-
""" java2python.compiler.visitor -> Base classes for walking ASTs. """
##
-#
-# This module defines the base visitor class, Base. It
-# implements the accept() and walk() methods. The walk() method
-# handles finding and inserting blocks for hidden comment lexer nodes.
-#
+# Needs some introductory comments.
+
from functools import reduce
from itertools import ifilter, izip, tee
from re import compile as rxcompile, sub as rxsub
@@ -17,15 +14,16 @@
class Memo(object):
""" Memo -> AST walking luggage. """
+
def __init__(self):
self.comments, self.last = set(), 0
class Base(object):
- """ Base -> Base class for AST visitors. """
- commentSubs = (
- rxcompile('^\s*/(\*)+'), rxcompile('(\*)+/\s*$'), rxcompile('^\s*//'),
- )
+ """ Base -> Base class for AST visitors.
+
+ """
+ commentSubs = map(rxcompile, ('^\s*/(\*)+', '(\*)+/\s*$', '^\s*//'))
def accept(self, node, memo):
""" Accept a node, possibly creating a child visitor. """
@@ -38,17 +36,18 @@
if not tree:
return
memo = Memo() if memo is None else memo
- self.insertComments(self, tree, tree.tokenStartIndex, memo)
+ insertc = self.insertComments
+ insertc(self, tree, tree.tokenStartIndex, memo)
visitor = self.accept(tree, memo)
if visitor:
for child in tree.children:
visitor.walk(child, memo)
- self.insertComments(visitor, child, child.tokenStopIndex, memo)
- self.insertComments(self, tree, tree.tokenStopIndex, memo)
- ## we've got to cheat and check to see if this is a
- ## compilation unit node. if so, scan any remaining tokens.
- if tree.token.type == tokens.JAVA_SOURCE:
- self.insertComments(self, tree, len(tree.parser.input.tokens), memo)
+ insertc(visitor, child, child.tokenStopIndex, memo)
+ insertc(self, tree, tree.tokenStopIndex, memo)
+ if tree.isJavaSource: # cheat but cheap
+ insertc(self, tree, len(tree.parser.input.tokens), memo)
+ for handler in self.configHandlers('PostWalk', suffix='Mutators'):
+ handler(self)
def zipWalk(self, nodes, visitors, memo):
""" Walk the given nodes zipped with the given visitors. """
@@ -59,18 +58,11 @@
""" Add comments to the block from tokens in the tree. """
cache, parser = memo.comments, tree.parser
prefix = self.config.last('commentPrefix', '# ')
-
- ## the next for loop should read like this, but it's pretty
- ## slow that way:
- #
- #pred = lambda k:k.type in tokens.commentTypes and k.index not in cache
- #for token in ifilter(pred, parser.input.tokens[0:index]):
for token in parser.input.tokens[memo.last:index]:
if token.type not in tokens.commentTypes or token.index in cache:
- continue
+ continue # lambda in the for statement was slow.
cache.add(token.index)
- isexp = getattr(block, 'isExpression', False)
- if isexp and (token.line == parser.input.tokens[index].line):
+ if block.isExpression and
(token.line==parser.input.tokens[index].line):
block.tail += '' if block.tail.startswith(prefix) else prefix
block.tail += ''.join(self.stripComment(token.text))
else:
@@ -101,7 +93,6 @@
acceptInterface = makeAcceptType('interface')
-
class Module(ModuleClassSharedMixin, Base):
""" Module -> accepts AST branches for module-level objects. """
@@ -118,36 +109,33 @@
acceptPackage = makeAcceptHandledDecl('PackageDeclaration')
-
class ClassMethodSharedMixin(object):
def acceptModifierList(self, node, memo):
""" Accept and process class and method modifiers. """
- ## this is pretty bad. wrong.
+ # this is pretty bad. wrong.
for mod in node.children:
if mod.type == tokens.AT:
- ## needs args, too
- deco = '{0}()'.format(mod.firstChildOfType(tokens.IDENT).text)
+ # needs args, too
+ name = mod.firstChildOfType(tokens.IDENT).text
+ deco = self.factory.expr(left=name, fs='@{left}()')
self.decorators.append(deco)
else:
if node.parentType in tokens.methodTypes:
- self.modifiers.extend(n.text for n in node.children) ## wrong
+ self.modifiers.extend(n.text for n in node.children) # wrong
if self.isStatic:
self.parameters[0]['name'] = 'cls'
self.modifiers.append(mod.text)
return self
-
-
-
class Class(ModuleClassSharedMixin, ClassMethodSharedMixin, Base):
""" Class -> accepts AST branches for class-level objects. """
def acceptAt(self, node, memo):
""" Accept and ignore an annotation declaration. """
- ## this overrides the ModuleClassSharedMixin implementation and
- ## ignores AT tokens; they're sent within the class modifier
- ## list and we have no use for them here.
+ # this overrides the ModuleClassSharedMixin implementation and
+ # ignores AT tokens; they're sent within the class modifier
+ # list and we have no use for them here.
def acceptConstructorDecl(self, node, memo):
@@ -155,7 +143,7 @@
method = self.factory.method(name='__init__', type=self.name, parent=self)
superCalls = node.findChildrenOfType(tokens.SUPER_CONSTRUCTOR_CALL)
if not any(superCalls) and any(self.bases):
- ## from the java tutorial:
+ # from the java tutorial:
# Note: If a constructor does not explicitly invoke a
# superclass constructor, the Java compiler automatically
# inserts a call to the no-argument constructor of the
@@ -169,19 +157,18 @@
names = (n.text for n in node.findChildrenOfType(tokens.IDENT))
self.bases.extend(names)
+ acceptImplementsClause = acceptExtendsClause
+
def acceptFunctionMethodDecl(self, node, memo):
""" Accept and process a typed method declaration. """
ident = node.firstChildOfType(tokens.IDENT)
type = node.firstChildOfType(tokens.TYPE).children[0].text
return self.factory.method(name=ident.text, type=type, parent=self)
- acceptImplementsClause = acceptExtendsClause
-
-
def acceptVarDeclaration(self, node, memo):
""" Creates a new expression for a variable declaration. """
- ## this is strikingly similar to
- ## MethodContent.acceptVarDeclaration; merge and fix.
+ # this is strikingly similar to
+ # MethodContent.acceptVarDeclaration; merge and fix.
decl = node.firstChildOfType(tokens.VAR_DECLARATOR_LIST)
for vard in decl.childrenOfType(tokens.VAR_DECLARATOR):
@@ -213,8 +200,10 @@
ident = node.firstChildOfType(tokens.IDENT)
return self.factory.method(name=ident.text, type='void', parent=self)
+
class Annotation(Class):
""" Annotation -> accepts AST branches for Java annotations. """
+
def acceptAnnotationMethodDecl(self, node, memo):
""" Accept and process an annotation method declaration. """
ident = node.firstChildOfType(tokens.IDENT)
@@ -247,14 +236,10 @@
return self
-
-
class Interface(Class):
""" Interface -> accepts AST branches for Java interfaces. """
-
-
class MethodContent(Base):
""" MethodContent -> accepts trees for blocks within methods. """
@@ -293,10 +278,10 @@
parNode = node.firstChildOfType(tokens.PARENTESIZED_EXPR)
lblNode = node.firstChildOfType(tokens.SWITCH_BLOCK_LABEL_LIST)
caseNodes = lblNode.children
- ## empty switch statement
+ # empty switch statement
if not len(caseNodes):
return
- ## we have at least one node...
+ # we have at least one node...
parExpr = self.factory.expr()
parExpr.walk(parNode)
eqFs = FS.l + '==' + FS.r
@@ -347,10 +332,10 @@
def acceptIf(self, node, memo):
""" Accept and process an if statement. """
- ## the parser will feed us one of three forms:
- ## bare if: PARENTESIZED_EXPR - BLOCK_SCOPE
- ## if with else: PARENTESIZED_EXPR - BLOCK_SCOPE - BLOCK_SCOPE
- ## if with else if: PARENTESIZED_EXPR - BLOCK_SCOPE - IF
+ # the parser will feed us one of three forms:
+ # bare if: PARENTESIZED_EXPR - BLOCK_SCOPE
+ # if with else: PARENTESIZED_EXPR - BLOCK_SCOPE - BLOCK_SCOPE
+ # if with else if: PARENTESIZED_EXPR - BLOCK_SCOPE - IF
children = node.children
ifStat = self.factory.statement('if', fs=FS.lsrc, parent=self)
ifStat.expr.walk(children[0], memo)
@@ -453,8 +438,6 @@
inExpr = identExpr.right = self.factory.expr()
inExpr.walk(node.firstChildOfType(tokens.EXPR), memo)
forBlock = self.factory.methodContent(parent=self)
- #print '############', [tokens.map[n.type] for n in node.children]
- #forBlock.walk(node.firstChildOfType(tokens.BLOCK_SCOPE), memo)
forBlock.walk(node.children[4], memo)
def acceptFor(self, node, memo):
@@ -474,8 +457,7 @@
def acceptFormalParamStdDecl(self, node, memo):
""" Accept and process a single parameter declaration. """
ident = node.firstChildOfType(tokens.IDENT)
-
- # wrong, wrong, wrong.
+ # wrong
ptype = list(node.findChildrenOfType(tokens.TYPE))[0]
try:
ptype = list(ptype.findChildrenOfType(tokens.IDENT))[0].text
@@ -493,7 +475,7 @@
class Expression(Base):
- """ Class -> accepts trees for expression objects. """
+ """ Expression -> accepts trees for expression objects. """
def textExpression(self, node, memo):
""" Assigns node text to the left side of this expression. """
@@ -590,7 +572,7 @@
def acceptMethodCall(self, node, memo):
""" Accept and process a method call. """
- ## note: this creates one too many expression levels.
+ # NB: this creates one too many expression levels.
expr = self.factory.expr
self.fs = FS.l + '(' + FS.r + ')'
self.left = expr(parent=self)
@@ -629,8 +611,7 @@
acceptUnaryPlus = makeAcceptPreFormatted('+' + FS.l)
acceptUnaryMinus = makeAcceptPreFormatted('-' + FS.l)
acceptNot = makeAcceptPreFormatted('~' + FS.l)
-
- ## wrong.
+ # wrong.
acceptCastExpr = makeAcceptPreFormatted(FS.l + '(' + FS.r + ')' )
def makeAcceptPrePost(suffix, pre=False):
@@ -645,9 +626,9 @@
left = ident
else:
left = name
- block.insertChild(expr(fs=FS.l+' = '+FS.r, left=name, right=ident))
+ block.adopt(expr(fs=FS.l+' = '+FS.r, left=name, right=ident))
self.left = expr(parent=self, fs=FS.l, left=left)
- block.insertChild(expr(fs=FS.l + suffix, left=ident))
+ block.adopt(expr(fs=FS.l + suffix, left=ident))
else:
self.fs = FS.l + suffix
self.left, self.right = vs = expr(parent=self), expr(parent=self)
@@ -691,7 +672,8 @@
class Comment(Expression):
- """ """
+ """ Comment -> really implemented only for the type making in
__init__.py """
+
class Statement(MethodContent):
""" Statement -> accepts AST branches for statement objects. """
=======================================
--- /branches/0.5/java2python/config/default.py Wed Jul 28 15:44:19 2010
+++ /branches/0.5/java2python/config/default.py Fri Jul 30 23:35:02 2010
@@ -9,26 +9,22 @@
from java2python.lang.selector import *
-# Leading indent character or characters. Read PEP 8 before you
-# override this.
+# Leading indent character or characters. Four spaces are used
+# because that is the recommendation of PEP 8.
indentPrefix = ' '*4
-# Prefix character or characters for comments. Again, you should read
-# PEP 8 before you override this.
+# Prefix character or characters for comments. The hash+space is
+# recommended by PEP 8.
commentPrefix = '# '
-# When true, class definitions will be bubbled to the top of their
-# container (outer class or module).
-reorderClassDefs = True
-
-
-# These generator-functions yield lines for a module prologue.
+# These values are strings or generator-functions that yield strings
+# for a module prologue.
modulePrologueHandlers = [
basic.simpleShebang,
basic.simpleDocString,
- basic.insertBsr,
+ basic.maybeBsr,
]
@@ -46,51 +42,40 @@
]
-# These generator-functions yield doc strings for classes.
-classDocStringHandlers = [
+classHeadHandlers = [
basic.simpleDocString,
]
-# These generator-functions yield doc strings for enums.
-enumDocStringHandlers = [
- basic.simpleDocString,
+classPostWalkMutators = [
+ basic.classContentSort,
]
-# These generator-functions yield doc strings for methods.
-methodDocStringHandlers = [
+enumHeadHandlers = [
basic.simpleDocString,
]
-
-# These generator-functions yield extra method decorators.
-methodExtraDecoratorHandlers = [
- basic.maybeClassMethod,
- basic.overloadedClassMethods,
+interfaceHeadHandlers = [
+ basic.simpleDocString,
+ '__metaclass__ = ABCMeta',
]
-# These generator-functions yield base types for classes.
-classBaseHandlers = [
- basic.mapClassType
+methodHeadHandlers = [
+ basic.simpleDocString,
]
-# These generator-functions yield base types for enums.
-enumBaseHandlers = [
- basic.mapClassType
+methodPrologueHandlers = [
+ basic.maybeAbstractMethod,
+ basic.maybeClassMethod,
+ basic.overloadedClassMethods,
]
-# generator-functions that yield base types for interfaces.
-interfaceBaseHandlers = [
- basic.mapClassType
-]
-
-
# This handler creates enum values on enum classes after they've been
-# defined. The handler matches Java semantics fairly closely by using
+# defined. The handler tries to matches Java semantics by using
# strings. Refer to the documentation for details.
enumValueHandler = basic.enumConstStrings
@@ -99,7 +84,9 @@
#enumValueHandler = basic.enumConstInts
-
+# When the compiler needs to make up a variable name (for example, to
+# emulate assignment expressions), it calls this handler to produce a
+# new one.
expressionVariableNamingHandler = basic.globalNameCounter
@@ -107,6 +94,7 @@
# declarations.
modulePackageDeclarationHandler = basic.commentedPackages
+
# This handler can be used instead to create __init__.py files for
# 'namespace packages' via pkgutil.
modulePackageDeclarationHandler = basic.namespacePackages
@@ -141,10 +129,6 @@
# in method declarations. set to 0 to disable.
#minIndentParams = 5
-# these handle shift right and bit shift right assignments.
-#bsrHandler = 'basic.functionBsr'
-#bsrHandlerAssign = 'basic.functionBsrAssign'
-
# Values below are used by the handlers. They're here for
# convenience.
@@ -181,3 +165,6 @@
'double' : 'float',
'java.lang.String' : 'str',
}
+
+
+
=======================================
--- /branches/0.5/java2python/lang/Java.g Sat Jul 24 01:19:21 2010
+++ /branches/0.5/java2python/lang/Java.g Fri Jul 30 23:35:02 2010
@@ -42,7 +42,6 @@
language=Python;
output=AST;
ASTLabelType=CommonTree;
- superClass=LocalParser;
}
@@ -218,17 +217,6 @@
VAR_DECLARATOR_LIST;
VOID_METHOD_DECL;
}
-
-@parser::header {
- from java2python.lang.base import LocalParser
-}
-
-@lexer::header {
- ##// the following line replaces antlr's base lexer class with our own.
- ##// this is required because we can't specify the lexer base
- ##// class directly.
- from java2python.lang.base import LocalLexer as Lexer
-}
javaSource
=======================================
--- /branches/0.5/java2python/lang/__init__.py Mon Jul 19 15:02:15 2010
+++ /branches/0.5/java2python/lang/__init__.py Fri Jul 30 23:35:02 2010
@@ -3,7 +3,5 @@
from java2python.lang.JavaLexer import JavaLexer as Lexer
from java2python.lang.JavaParser import JavaParser as Parser
-from java2python.lang.base import (
- LocalSourceStream, LocalTokenStream, LocalTreeAdaptor, tokens,
- )
+from java2python.lang.base import StringStream, TokenStream, TreeAdaptor,
tokens
from java2python.lang.selector import walkTreeSelector
=======================================
--- /branches/0.5/java2python/lang/base.py Sat Jul 24 01:19:21 2010
+++ /branches/0.5/java2python/lang/base.py Fri Jul 30 23:35:02 2010
@@ -2,6 +2,8 @@
# -*- coding: utf-8 -*-
##
+# ANTLR notes:
+#
# recognizers: lexer, parser, treeparser
# streams: string, file name, file handle
#
@@ -14,17 +16,17 @@
# Parsers build trees if their output is AST.
#
# token types: CommonToken and ClassicToken. Our tree adaptor
-# creates LocalTokens instead.
+# creates LocalTree instances instead.
#
# Tree (CommonTree) wraps Token objects. We provide extra functionality
via
# the LocalTree class.
#
# TreeAdaptor (CommonTreeAdaptor) is used by the parser to create
-# Tree objects. Our adaptor, LocalTreeAdaptor, creates the LocalTree
+# Tree objects. Our adaptor, TreeAdaptor, creates the LocalTree
# instances.
#
##
-from antlr3 import ANTLRStringStream, CommonTokenStream, Lexer, Parser
+from antlr3 import ANTLRStringStream as StringStream, CommonTokenStream as
TokenStream
from antlr3.tree import CommonTreeAdaptor, CommonTree
from java2python.lib import colortools
@@ -36,19 +38,24 @@
self.cache, self.parserModule = {}, None
def __getattr__(self, name):
+ """ tokenname -> tokenvalue """
return getattr(self.module, name)
@property
def commentTypes(self):
+ """ Well-known comment types. """
mod = self.module
return (mod.COMMENT, mod.LINE_COMMENT, mod.JAVADOC_COMMENT, )
@property
def methodTypes(self):
- return (self.module.VOID_METHOD_DECL,
self.module.FUNCTION_METHOD_DECL)
+ """ Well-known method types. """
+ mod = self.module
+ return (mod.VOID_METHOD_DECL, mod.FUNCTION_METHOD_DECL, )
@property
def map(self):
+ """ tokentype -> tokenname mapping as a dictionary """
cache, module = self.cache, self.module
if cache:
return cache
@@ -59,47 +66,40 @@
@property
def module(self):
+ """ Provides lazy import to the parser module. """
module = self.parserModule
- if module is None:
- import java2python.lang.JavaParser as module
- self.parserModule = module
+ if module:
+ return module
+ import java2python.lang.JavaParser as module
+ self.parserModule = module
return module
@staticmethod
def title(name):
+ """ Returns a nice title given a token type name. """
return ''.join(part.title() for part in name.split('_'))
+## sometimes you really do only need one.
tokens = Tokens()
-## TODO: switch on string/file-like
-class LocalSourceStream(ANTLRStringStream):
- pass
-
-
-class LocalTokenStream(CommonTokenStream):
- pass
-
-
-class LocalRecognizer(object):
- pass
-
-
-class LocalParser(Parser, LocalRecognizer):
- def __init__(self, input, state=None):
- super(LocalParser, self).__init__(input, state=state)
-
-
-
-
-
-class LocalLexer(Lexer, LocalRecognizer):
- def __init__(self, input=None, state=None):
- super(LocalLexer, self).__init__(input, state)
+class TreeAdaptor(CommonTreeAdaptor):
+ """ TreeAdaptor -> defered tree node creator (for parsers) """
+
+ def __init__(self, lexer, parser):
+ # CommonTreeAdaptor doesn't need to be __init__'ed
+ self.lexer, self.parser = lexer, parser
+
+ def createWithPayload(self, payload):
+ """ Returns a new tree for the calling parser. """
+ return LocalTree(payload, self.lexer, self.parser)
class LocalTree(CommonTree):
+ """ LocalTree -> like CommonTree, but with much more stuff
+
+ """
colorTypeMap = {
'CLASS' : colortools.green,
'JAVA_SOURCE' : colortools.green,
@@ -111,23 +111,34 @@
'FALSE' : colortools.yellow,
'NULL' : colortools.yellow,
}
+
+ def __init__(self, payload, lexer=None, parser=None):
+ super(LocalTree, self).__init__(payload)
+ self.lexer, self.parser = lexer, parser
+
+ def childrenOfType(self, type):
+ """ Returns a generator yielding children of this tree of the given
type. """
+ return (c for c in self.children if c.type==type)
def colorType(self, tokenType):
+ """ Returns a color suitable for the given token type. """
return self.colorTypeMap.get(tokenType, colortools.white)(tokenType)
def colorText(self, tokenType, tokenText):
+ """ Returns a colorized string from the given token type and text. """
return self.colorTypeMap.get(tokenType, colortools.white)(tokenText)
def colorComments(self, token):
+ """ Formats, colors, and returns the comment text from the given
token. """
ttyp = tokens.map.get(token.type)
text =
token.text.replace('\n', '\\n').replace('\r', '\\r').replace('\t', '\\t')
item = '{0} [{1}:{2}] {3}'.format(ttyp, token.start, token.stop, text)
yield colortools.black(item)
def dump(self, fd, level=0):
+ """ Writes a debug representation of this tree to the given file. """
extras = lambda x, y:x and (x != y)
seen, nform = set(), '{0}{1}{2}{3}'
-
def innerDump(root, offset):
token, indent = root.token, ' ' * offset
start, stop = root.tokenStartIndex, root.tokenStopIndex
@@ -150,76 +161,64 @@
print >> fd, '{0}{1}'.format(indent, line)
innerDump(self, level)
- def selectComments(self, stop, memo):
- pred = lambda k:k.type in tokens.commentTypes and k.index not in memo
- ctoks = [t for t in self.parser.input.tokens[0:stop] if pred(t)]
- memo.update(t.index for t in ctoks)
- return ctoks
-
- def childrenOfType(self, type):
- return (c for c in self.children if c.type==type)
+ def dupNode(self):
+ """ Called by the parser to create a duplicate of this tree. """
+ get = lambda v:getattr(self, v, None)
+ return LocalTree(self, get('lexer'), get('parser'))
+
+ def findChildren(self, pred=lambda c:True):
+ """ Depth-first search that yields nodes meeting the predicate. """
+ for child in self.children:
+ if pred(child):
+ yield child
+ for sub in child.findChildren(pred):
+ yield sub
+
+ def findChildrenOfType(self, type):
+ """ Depth-first search that yields nodes of the given type. """
+ return self.findChildren(lambda c:c.type==type)
def firstChild(self, default=None):
+ """ Returns the first child of this tree or the default. """
try:
return self.children[0]
except (IndexError, ):
return default
def firstChildOfType(self, type, default=None):
+ """ Returns the first child of this tree that matches the given type. """
for child in self.children:
if child.type == type:
return child
return default
- def findChildren(self, pred=lambda c:True):
- """ Depth-first search that yields nodes meeting the predicate. """
- for child in self.children:
- if pred(child):
- yield child
- for sub in child.findChildren(pred):
- yield sub
-
- def findChildrenOfType(self, type):
- """ Depth-first search that yields nodes of the given type. """
- return self.findChildren(lambda c:c.type==type)
-
@property
- def withinExpr(self):
- parent = getattr(self.parent, 'parent', None) # skip first expr
- while parent:
- if parent.type in (tokens.EXPR, ):
- return True
- parent = parent.parent
+ def isJavaSource(self):
+ """ True if this tree is the outer most type. """
+ return self.token.type == tokens.JAVA_SOURCE
@property
def parentType(self):
- """ Returns the type of the parent node. """
+ """ Returns the type of the parent tree. """
return self.parent.type
- def dupNode(self):
- node = LocalTree(self)
- node.parser = getattr(self, 'parser', None)
- node.lexer = getattr(self, 'lexer', None)
- return node
-
@property
def parserTokens(self):
+ """ Returns the sequence of tokens used to create this tree. """
return self.parser.input.tokens[self.tokenStartIndex:self.tokenStopIndex]
- def select(self, selector):
- pass
-
-
-
-class LocalTreeAdaptor(CommonTreeAdaptor):
- treeType = LocalTree
-
- def __init__(self, callback=None):
- CommonTreeAdaptor.__init__(self)
- self.callback = callback
-
- def createWithPayload(self, payload):
- node = self.treeType(payload)
- if node.token and self.callback:
- self.callback(node)
- return node
+ def selectComments(self, stop, memo):
+ """ Returns the comment tokens for this tree up to the given index. """
+ pred = lambda k:k.type in tokens.commentTypes and k.index not in memo
+ ctoks = [t for t in self.parser.input.tokens[0:stop] if pred(t)]
+ memo.update(t.index for t in ctoks)
+ return ctoks
+
+ @property
+ def withinExpr(self):
+ """ True if this tree is contained within an expression. """
+ self = getattr(self.parent, 'parent', None) # skip first expr
+ while self:
+ if self.type in (tokens.EXPR, ):
+ return True
+ self = self.parent
=======================================
--- /branches/0.5/java2python/mod/basic.py Wed Jul 28 15:44:19 2010
+++ /branches/0.5/java2python/mod/basic.py Fri Jul 30 23:35:02 2010
@@ -81,13 +81,6 @@
return text
-
-def mapClassType(obj):
- bases = obj.bases or ('object', )
- for name in bases:
- yield name
-
-
def overloadedClassMethods(method):
"""
NB: this implementation does not handle overloaded static (or
@@ -100,38 +93,46 @@
for i, m in enumerate(methods[1:]):
args = [p['type'] for p in m.parameters]
args = ', '.join(args)
- m.decorators.append('{0}.register({1})'.format(method.name, args))
+ m.decorators.append('@{0}.register({1})'.format(method.name, args))
m.name = '{0}_{1}'.format(method.name, i)
# for this one only:
- yield 'overloaded'
+ yield '@overloaded'
def maybeClassMethod(method):
if method.isStatic and 'classmethod' not in method.decorators:
- yield 'classmethod'
-
-
-def simpleInterfaces(method):
- mkExpr = partial(Expression, module.config,
- format='raise NotImplementedError({left})')
- for iface in module.interfaces:
- for method in iface.methods:
- expr = mkExpr(left='"Method \'%s\' is abstract"' % method.name)
- method.children.insert(0, expr)
+ yield '@classmethod'
+
+def maybeAbstractMethod(method):
+ if method.parent and method.parent.isInterface:
+ yield '@abstractmethod'
def globalNameCounter(original, counter=count()):
return '__{0}_{1}'.format(original, counter.next())
-
def getBsrSrc():
from inspect import getsource
from java2python.mod.includes import bsr
return getsource(bsr)
-def insertBsr(module):
+def maybeBsr(module):
if getattr(module, 'needsBsrFunc', False):
for line in getBsrSrc().split('\n'):
yield line
+
+
+# this function implies that there should be a separate modification
+# stage where client code like this can mutate the templates generated
+# by the compiler. this works, but it not really what the prologue
+# handlers are meant to do. the various interface functions fall into
+# the category of mutators also.
+
+# also, the output handers seem to be a special case of this
+# desired mutator-function.
+
+def classContentSort(obj):
+ obj.children.sort(lambda x, y:-1 if x.isClass else 1)
+
=======================================
--- /branches/0.5/test/Makefile Wed Jul 28 15:44:19 2010
+++ /branches/0.5/test/Makefile Fri Jul 30 23:35:02 2010
@@ -36,6 +36,5 @@
%.py: %.class
@$(j2py) -i $(addsuffix .java, $(basename $@)) -o $@
-c ./configs/__init__.py
-
%: %.py
@bash -c "diff -q <($(python) $(addsuffix .py, $@)) <(java -ea $@)" &&
echo "[PASS] $@"
=======================================
--- /branches/0.5/test/runj2py Wed Jul 28 15:44:19 2010
+++ /branches/0.5/test/runj2py Fri Jul 30 23:35:02 2010
@@ -1,2 +1,2 @@
#!/bin/bash
-../bin/j2py -c configs "$@"
+../bin/j2py -d configs "$@"