[java2python] r167 committed - removes lexer node modifier methods in favor of new support for ast tr...
codesite...@google.com
Author: troy.melhase
Date: Mon Jul 19 15:02:15 2010
Log: removes lexer node modifier methods in favor of new support for ast
transformations.
http://code.google.com/p/java2python/source/detail?r=167
Added:
/branches/0.5/java2python/lang/selector.py
/branches/0.5/java2python/mod/transforms.py
Deleted:
/branches/0.5/java2python/compiler/transformer.py
Modified:
/branches/0.5/bin/j2py
/branches/0.5/java2python/compiler/__init__.py
/branches/0.5/java2python/compiler/visitor.py
/branches/0.5/java2python/config/default.py
/branches/0.5/java2python/lang/__init__.py
/branches/0.5/java2python/lang/base.py
/branches/0.5/test/Literals1.java
=======================================
--- /dev/null
+++ /branches/0.5/java2python/lang/selector.py Mon Jul 19 15:02:15 2010
@@ -0,0 +1,165 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+from java2python.lang import tokens
+
+
+class Selector(object):
+ """ Selector -> base for selectors; provides operator methods.
+
+ """
+ def __add__(self, other):
+ return Sibling(self, other)
+
+ def __and__(self, other):
+ return Descendant(self, other)
+
+ def __getitem__(self, key):
+ return Nth(self, key)
+
+ def __gt__(self, other):
+ return Child(self, other)
+
+
+class Nth(Selector):
+ """ E[n] match any slice n of E
+
+ """
+ def __init__(self, e, key):
+ self.e, self.key = e, key
+
+ def __call__(self, tree):
+ for etree in self.e(tree):
+ try:
+ matches = tree.children[self.key]
+ except (IndexError, ):
+ return
+ if not isinstance(matches, (list, )):
+ matches = [matches]
+ for child in matches:
+ yield child
+
+ def __str__(self):
+ return 'Nth({0})[{1}]'.format(self.e, self.key)
+
+
+class Child(Selector):
+ """ E > F select any F that is a child of E
+
+ """
+ def __init__(self, e, f):
+ self.e, self.f = e, f
+
+ def __call__(self, tree):
+ for ftree in self.f(tree):
+ for etree in self.e(tree.parent):
+ yield ftree
+
+ def __str__(self):
+ return 'Child({0} > {1})'.format(self.e, self.f)
+
+
+class Type(Selector):
+ """ Type(T) select any token of type T
+
+ """
+ def __init__(self, key, value=None):
+ self.key = key if isinstance(key, int) else getattr(tokens, key)
+ self.value = value
+
+ def __call__(self, tree):
+ if tree.token.type == self.key:
+ if self.value is None or self.value == tree.token.text:
+ yield tree
+
+ def __str__(self):
+ ## TODO: add value
+ return 'Type({0}:{1})'.format(tokens.map[self.key], self.key)
+
+
+class Star(Selector):
+ """ * select any
+
+ """
+ def __call__(self, tree):
+ yield tree
+
+ def __str__(self):
+ return 'Star(*)'
+
+
+class Descendant(Selector):
+ """ E & F select any F that is a descendant of E
+
+ """
+ def __init__(self, e, f):
+ self.e, self.f = e, f
+
+ def __call__(self, tree):
+ for ftree in self.f(tree):
+ root, ftree = ftree, ftree.parent
+ while ftree:
+ for etree in self.e(ftree):
+ yield root
+ ftree = ftree.parent
+
+ def __str__(self):
+ return 'Descendant({0} & {1})'.format(self.e, self.f)
+
+
+class Sibling(Selector):
+ """ E + F select any F immediately preceded by a sibling E
+
+ """
+ def __init__(self, e, f):
+ self.e, self.f = e, f
+
+ def __call__(self, node):
+ parent = node.parent
+ if not parent:
+ return
+ for ftree in self.f(node):
+ index = node.parent.children.index(ftree)
+ if not index:
+ return
+ previous = node.parent.children[index-1]
+ for child in self.e(previous):
+ yield ftree
+
+ def __str__(self):
+ return 'Sibling({0} + {1})'.format(self.e, self.f)
+
+
+def walkTreeSelector(tree, selector):
+ for item in selector(tree):
+ yield item
+ for child in tree.children:
+ for item in walkTreeSelector(child, selector):
+ yield item
+
+
+if __name__ == '__main__':
+ import sys
+ from java2python.config import Config
+ from java2python.compiler import buildAST
+
+ source = open(sys.argv[1]).read()
+ tree = buildAST(source, Config(()))
+ tree.dump(sys.stdout)
+
+
+ selectors = [
+ Type('EXPR'),
+ Type('QUALIFIED_TYPE_IDENT') > Type('IDENT'),
+ Type('CLASS')[2],
+ Type('METHOD_CALL') & Type('IDENT'),
+ Type('IDENT') + Type('IDENT'),
+ ]
+
+ for index, selector in enumerate(selectors):
+ print '{0}: {1}\n ==== {2}'.format(index, selector.__doc__.strip(),
selector)
+ for node in walkTreeSelector(tree, selector):
+ name = str(node)
+ ntype = tokens.map[node.type]
+ args = (name, '') if name == ntype else (ntype, name)
+ print '{0}---- match: {1} {2}'.format(*(' '*8, )+args)
+ print
=======================================
--- /dev/null
+++ /branches/0.5/java2python/mod/transforms.py Mon Jul 19 15:02:15 2010
@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import types
+from keyword import kwlist
+
+
+def renameIdents():
+ typs = [getattr(types, n) for n in dir(types) if not n.startswith('_')]
+ names = [t.__name__ for t in typs if isinstance(t, type)]
+ return ['None', 'True', 'False', ] + names + kwlist
+
+
+renameIdents = renameIdents()
+
+
+def keywordSafeIdent(node):
+ ident = node.token.text
+ if ident in renameIdents:
+ node.token.text = '%s_' % ident
+
+
+def constXform(v):
+ def xform(node):
+ node.token.text = v
+ return xform
+
+
+null2None = constXform('None')
+false2False = constXform('False')
+true2True = constXform('True')
+
+
+def syntaxSafeFloatLiteral(node):
+ value = node.token.text
+ if value.startswith('.'):
+ value = '0' + value
+ if value.endswith(('f', 'd')):
+ value = value[:-1]
+ elif value.endswith(('l', 'L')):
+ value = value[:-1] + 'L'
+ node.token.text = value
+
+
=======================================
--- /branches/0.5/java2python/compiler/transformer.py Mon Jul 19 14:14:07
2010
+++ /dev/null
@@ -1,170 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-from java2python.lang import tokens
-
-
-class Transformer(object):
- def __init__(self, configs=()):
- self.configs = configs
-
-
-class Selector(object):
- """ Selector -> base for selectors; provides operator methods.
-
- """
- def __add__(self, other):
- return Sibling(self, other)
-
- def __and__(self, other):
- return Descendant(self, other)
-
- def __getitem__(self, key):
- return Nth(self, key)
-
- def __gt__(self, other):
- return Child(self, other)
-
-
-class Nth(Selector):
- """ E[n] match any slice n of E
-
- """
- def __init__(self, e, key):
- self.e, self.key = e, key
-
- def __call__(self, tree):
- for etree in self.e(tree):
- try:
- matches = tree.children[self.key]
- except (IndexError, ):
- return
- if not isinstance(matches, (list, )):
- matches = [matches]
- for child in matches:
- yield child
-
- def __str__(self):
- return 'Nth({0})[{1}]'.format(self.e, self.key)
-
-
-class Child(Selector):
- """ E > F select any F that is a child of E
-
- """
- def __init__(self, e, f):
- self.e, self.f = e, f
-
- def __call__(self, tree):
- for ftree in self.f(tree):
- for etree in self.e(tree.parent):
- yield ftree
-
- def __str__(self):
- return 'Child({0} > {1})'.format(self.e, self.f)
-
-
-class Type(Selector):
- """ Type(T) select any token of type T
-
- """
- def __init__(self, key, value=None):
- self.key = key if isinstance(key, int) else getattr(tokens, key)
- self.value = value
-
- def __call__(self, tree):
- if tree.token.type == self.key:
- if self.value is None or self.value == tree.token.text:
- yield tree
-
- def __str__(self):
- ## TODO: add value
- return 'Type({0}:{1})'.format(tokens.map[self.key], self.key)
-
-
-class Star(Selector):
- """ * select any
-
- """
- def __call__(self, tree):
- yield tree
-
- def __str__(self):
- return 'Star(*)'
-
-
-class Descendant(Selector):
- """ E & F select any F that is a descendant of E
-
- """
- def __init__(self, e, f):
- self.e, self.f = e, f
-
- def __call__(self, tree):
- for ftree in self.f(tree):
- root, ftree = ftree, ftree.parent
- while ftree:
- for etree in self.e(ftree):
- yield root
- ftree = ftree.parent
-
- def __str__(self):
- return 'Descendant({0} & {1})'.format(self.e, self.f)
-
-
-class Sibling(Selector):
- """ E + F select any F immediately preceded by a sibling E
-
- """
- def __init__(self, e, f):
- self.e, self.f = e, f
-
- def __call__(self, node):
- parent = node.parent
- if not parent:
- return
- for ftree in self.f(node):
- index = node.parent.children.index(ftree)
- if not index:
- return
- previous = node.parent.children[index-1]
- for child in self.e(previous):
- yield ftree
-
- def __str__(self):
- return 'Sibling({0} + {1})'.format(self.e, self.f)
-
-
-def walkSelector(tree, selector):
- for item in selector(tree):
- yield item
- for child in tree.children:
- for item in walkSelector(child, selector):
- yield item
-
-
-if __name__ == '__main__':
- import sys
- from java2python.config import Config
- from java2python.compiler import buildAST
-
- source = open(sys.argv[1]).read()
- tree = buildAST(source, Config(()))
- tree.dump(sys.stdout)
-
-
- selectors = [
- Type('EXPR'),
- Type('QUALIFIED_TYPE_IDENT') > Type('IDENT'),
- Type('CLASS')[2],
- Type('METHOD_CALL') & Type('IDENT'),
- Type('IDENT') + Type('IDENT'),
- ]
-
- for index, selector in enumerate(selectors):
- print '{0}: {1}\n ==== {2}'.format(index, selector.__doc__.strip(),
selector)
- for node in walkSelector(tree, selector):
- name = str(node)
- ntype = tokens.map[node.type]
- args = (name, '') if name == ntype else (ntype, name)
- print '{0}---- match: {1} {2}'.format(*(' '*8, )+args)
- print
=======================================
--- /branches/0.5/bin/j2py Sat Jul 17 04:01:55 2010
+++ /branches/0.5/bin/j2py Mon Jul 19 15:02:15 2010
@@ -133,7 +133,7 @@
info('Parse: %.4f seconds', timed['comp_finish'] - timed['comp'])
info('Visit: %.4f seconds', timed['visit_finish'] - timed['visit'])
- #info('Transform: %.4f seconds', timed['xform_finish'] -
timed['xform'])
+ info('Transform: %.4f seconds', timed['xform_finish'] - timed['xform'])
info('Encode: %.4f seconds', timed['encode_finish'] -
timed['encode'])
info('Total: %.4f seconds', timed['overall_finish'] -
timed['overall'])
return 0
=======================================
--- /branches/0.5/java2python/compiler/__init__.py Sat Jul 17 04:01:55 2010
+++ /branches/0.5/java2python/compiler/__init__.py Mon Jul 19 15:02:15 2010
@@ -2,9 +2,9 @@
# -*- coding: utf-8 -*-
from java2python.compiler.block import Module
-from java2python.compiler.transformer import Transformer
from java2python.lang import (
- Lexer, Parser, LocalSourceStream, LocalTokenStream, LocalTreeAdaptor
+ Lexer, Parser, LocalSourceStream, LocalTokenStream, LocalTreeAdaptor,
+ walkTreeSelector,
)
@@ -17,7 +17,6 @@
def callback(node):
node.parser = sourceParser
node.lexer = sourceLexer
- sourceLexer.transform(node)
treeAdaptor = LocalTreeAdaptor(callback)
sourceParser.setTreeAdaptor(treeAdaptor)
@@ -32,9 +31,9 @@
def transformAST(tree, config):
- return # bah
- transformer = Transformer(config)
- transformer(tree)
+ for selector, call in config.handlers('transforms'):
+ for node in walkTreeSelector(tree, selector):
+ call(node)
if __name__ == '__main__':
=======================================
--- /branches/0.5/java2python/compiler/visitor.py Wed Jul 14 12:03:07 2010
+++ /branches/0.5/java2python/compiler/visitor.py Mon Jul 19 15:02:15 2010
@@ -4,9 +4,8 @@
##
#
# This module defines the base visitor class, BaseVisitor. It
-# implements the accept(...) and walk(...) methods. The walk(...)
-# method handles finding and inserting blocks for hidden comment lexer
-# nodes.
+# implements the accept() and walk() methods. The walk() method
+# handles finding and inserting blocks for hidden comment lexer nodes.
#
from functools import reduce
from itertools import ifilter, izip, tee
@@ -41,9 +40,6 @@
self.insertComments(self, tree, tree.tokenStartIndex, memo)
visitor = self.accept(tree, memo)
if visitor:
- ## look for a transformation function
- #n, hs = self.configTransformers(visitor)
- #print '############', n, list(hs)
for child in tree.children:
visitor.walk(child, memo)
self.insertComments(visitor, child, child.tokenStopIndex, memo)
=======================================
--- /branches/0.5/java2python/config/default.py Wed Jul 14 11:32:04 2010
+++ /branches/0.5/java2python/config/default.py Mon Jul 19 15:02:15 2010
@@ -27,6 +27,8 @@
# In some cases, noted below, the value or values may be a dotted path
# to a python value. In these cases, the value is imported as needed.
+import java2python.mod
+
## leading indent characters (or character). this is a "last" option.
leadingIndent = ' '
@@ -38,39 +40,39 @@
## this is an 'every' handler sequence
modulePrologueHandlers = [
- 'java2python.mod.simpleShebang',
- 'java2python.mod.simpleDocString',
- 'java2python.mod.configImports',
- 'java2python.mod.commentedImports',
- 'java2python.mod.commentedPackageName',
+ java2python.mod.simpleShebang,
+ java2python.mod.simpleDocString,
+ java2python.mod.configImports,
+ java2python.mod.commentedImports,
+ java2python.mod.commentedPackageName,
]
## this is an 'every' handler sequence
moduleEpilogueHandlers = [
- 'java2python.mod.scriptMainStanza',
+ java2python.mod.scriptMainStanza,
]
## ???
moduleOutputHandlers = [
- 'java2python.mod.outputSubs',
+ java2python.mod.outputSubs,
]
## handlers for doc strings.
classDocStringHandlers = [
- 'java2python.mod.simpleDocString',
+ java2python.mod.simpleDocString,
]
enumDocStringHandlers = [
- 'java2python.mod.simpleDocString',
+ java2python.mod.simpleDocString,
]
methodDocStringHandlers = [
- 'java2python.mod.simpleDocString',
+ java2python.mod.simpleDocString,
]
## extra decorator methods
methodExtraDecoratorHandlers = [
- 'java2python.mod.maybeClassMethod',
- 'java2python.mod.overloadedClassMethods',
+ java2python.mod.maybeClassMethod,
+ java2python.mod.overloadedClassMethods,
]
@@ -79,9 +81,9 @@
## these next 4 handler values should get morphed into lists.
-classBaseHandler = 'java2python.mod.mapClassType'
-enumBaseHandler = 'java2python.mod.mapClassType'
-interfaceBaseHandler = 'java2python.mod.mapClassType'
+classBaseHandler = java2python.mod.mapClassType
+enumBaseHandler = java2python.mod.mapClassType
+interfaceBaseHandler = java2python.mod.mapClassType
##
# Note that the following two enum value handlers are only called for
@@ -93,7 +95,7 @@
# This handler creates enum values on enum classes after they've been
# defined. The handler matches Java semantics fairly closely by using
# strings.
-enumValueHandler = 'java2python.mod.enumConstStrings'
+enumValueHandler = java2python.mod.enumConstStrings
# Alternatively, you can use this handler to construct enum values as
# integers.
@@ -159,3 +161,19 @@
'java.lang.String' : 'str',
}
+
+
+
+
+
+
+from java2python.lang.selector import *
+from java2python.mod import transforms
+
+transforms = [
+ (Type('NULL'), transforms.null2None),
+ (Type('FALSE'), transforms.false2False),
+ (Type('TRUE'), transforms.true2True),
+ (Type('IDENT'), transforms.keywordSafeIdent),
+ (Type('FLOATING_POINT_LITERAL'), transforms.syntaxSafeFloatLiteral),
+]
=======================================
--- /branches/0.5/java2python/lang/__init__.py Wed Jul 14 11:32:04 2010
+++ /branches/0.5/java2python/lang/__init__.py Mon Jul 19 15:02:15 2010
@@ -6,3 +6,4 @@
from java2python.lang.base import (
LocalSourceStream, LocalTokenStream, LocalTreeAdaptor, tokens,
)
+from java2python.lang.selector import walkTreeSelector
=======================================
--- /branches/0.5/java2python/lang/base.py Sat Jul 17 04:01:55 2010
+++ /branches/0.5/java2python/lang/base.py Mon Jul 19 15:02:15 2010
@@ -24,8 +24,6 @@
# instances.
#
##
-import types
-from keyword import kwlist
from antlr3 import ANTLRStringStream, CommonTokenStream, Lexer, Parser
from antlr3.tree import CommonTreeAdaptor, CommonTree
from java2python.lib import colortools
@@ -92,48 +90,13 @@
super(LocalParser, self).__init__(input, state=state)
-def typeNames():
- typs = [getattr(types, n) for n in dir(types) if not n.startswith('_')]
- names = [t.__name__ for t in typs if isinstance(t, type)]
- return ['None', 'True', 'False', ] + names
-def transformConstMethod(v):
- def transformMethod(self, node):
- node.token.text = v
- return transformMethod
-
class LocalLexer(Lexer, LocalRecognizer):
- renameIdents = kwlist + typeNames()
-
def __init__(self, input=None, state=None):
super(LocalLexer, self).__init__(input, state)
- def transform(self, node):
- title = tokens.title(tokens.map.get(node.token.type, ''))
- if title:
- call = getattr(self, 'transform{0}'.format(title), lambda n:None)
- call(node)
-
- transformFalse = transformConstMethod('False')
- transformTrue = transformConstMethod('True')
- transformNull = transformConstMethod('None')
-
- def transformIdent(self, node):
- ident = node.token.text
- node.token.text = '%s_' % ident if ident in self.renameIdents else ident
-
- def transformFloatingPointLiteral(self, node):
- value = node.token.text
- if value.startswith('.'):
- value = '0' + value
- if value.endswith(('f', 'd')):
- value = value[:-1]
- elif value.endswith(('l', 'L')):
- value = value[:-1] + 'L'
- node.token.text = value
-
class LocalTree(CommonTree):
colorTypeMap = {
=======================================
--- /branches/0.5/test/Literals1.java Fri Jul 2 23:54:05 2010
+++ /branches/0.5/test/Literals1.java Mon Jul 19 15:02:15 2010
@@ -1,6 +1,6 @@
class Literals1 {
public static void main (String [] args) {
-/*
+
Literals1 lit = new Literals1();
int h = 0xAA;
int o = 034;
@@ -11,9 +11,9 @@
String s = "string theory";
Boolean F = false;
Boolean T = true;
-*/
+
Object n = null;
-/*
+
System.out.println(h);
System.out.println(o);
System.out.println(f);
@@ -23,7 +23,7 @@
System.out.println(s);
System.out.println(T ? 1 : 0);
System.out.println(F ? 1 : 0);
-*/
+
System.out.println(n==null ? 1 : 0);
}
}