[java2python] r183 committed - Adds selector tests.
java2...@googlecode.com
Author: troy.melhase
Date: Wed Oct 19 20:00:15 2011
Log: Adds selector tests.
http://code.google.com/p/java2python/source/detail?r=183
Added:
/trunk/test/selector/Makefile
Deleted:
/trunk/java2python/compiler/tool.py
Modified:
/trunk/bin/j2py
/trunk/java2python/compiler/__init__.py
/trunk/java2python/lang/selector.py
/trunk/test/Makefile
/trunk/test/selector/Selector1.java
/trunk/test/selector/test_all.py
=======================================
--- /dev/null
+++ /trunk/test/selector/Makefile Wed Oct 19 20:00:15 2011
@@ -0,0 +1,12 @@
+.PHONY: all
+
+test_targets := $(sort $(notdir $(basename $(wildcard *.py))))
+
+
+all:
+ $(MAKE) $(test_targets)
+
+%:
+ @python -m unittest -v $@
+
+
=======================================
--- /trunk/java2python/compiler/tool.py Tue Oct 18 21:33:24 2011
+++ /dev/null
@@ -1,46 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-""" java2python.compiler.tool -> """
-##
-# This module can be run as a script, e.g.:
-#
-# $ python -m java2python.compiler.tool ./SomeClass.java
-
-from java2python.lang import Lexer, Parser, StringStream, TokenStream,
TreeAdaptor
-
-
-def buildAST(source, config=None):
- lexer = Lexer(StringStream(source))
- parser = Parser(TokenStream(lexer))
- adapter = TreeAdaptor(lexer, parser)
- parser.setTreeAdaptor(adapter)
- scope = parser.javaSource()
- return scope.tree
-
-
-def transformAST(tree, config):
- for selector, call in config.last('astTransforms', ()):
- for node in selector.walk(tree):
- call(node, config)
-
-
-def buildJavaDocAST(source):
- from java2python.lang.JavaDocLexer import JavaDocLexer
- from java2python.lang.JavaDocParser import JavaDocParser
- lexer = JavaDocLexer(StringStream(source))
- parser = JavaDocParser(TokenStream(lexer))
- scope = parser.commentBody()
- return scope.tree
-
-
-def walkJavaDoc(tree, callback=lambda x:None):
- pass
-
-
-if __name__ == '__main__':
- import sys
- tree = buildAST(open(sys.argv[1]).read())
- for idx, tok in enumerate(tree.parser.input.tokens):
- print '{0} {1}'.format(idx, tok)
- print
- tree.dump(sys.stdout)
=======================================
--- /trunk/bin/j2py Tue Oct 18 21:33:24 2011
+++ /trunk/bin/j2py Wed Oct 19 20:00:15 2011
@@ -93,15 +93,15 @@
print 'IOError: %s.' % (msg, )
return code
- config = Config(configs)
timed['comp']
try:
- tree = buildAST(source, config)
+ tree = buildAST(source)
except (Exception, ), exc:
exception('exception while parsing')
return 1
timed['comp_finish']
+ config = Config(configs)
timed['xform']
transformAST(tree, config)
timed['xform_finish']
@@ -118,6 +118,11 @@
timed['encode_finish']
timed['overall_finish']
+ if options.lexertokens:
+ for idx, tok in enumerate(tree.parser.input.tokens):
+ print >> sys.stderr, '{0} {1}'.format(idx, tok)
+ print >> sys.stderr
+
if options.javaast:
tree.dump(sys.stderr)
print >> sys.stderr
@@ -171,6 +176,9 @@
addopt('-l', '--loglevel', dest='loglevel',
help='Set log level by name or value.',
default='WARN', type='loglevel')
+ addopt('-t', '--lexer-tokens', dest='lexertokens',
+ help='Print lexer tokens to stderr.',
+ default=False, action='store_true')
addopt('-p', '--python-tree', dest='pytree',
help='Print python object tree to stderr.',
default=False, action='store_true')
=======================================
--- /trunk/java2python/compiler/__init__.py Sat Oct 15 15:17:26 2011
+++ /trunk/java2python/compiler/__init__.py Wed Oct 19 20:00:15 2011
@@ -7,4 +7,31 @@
# instead of using directly referencing items within the subpackage.
from java2python.compiler.block import Module
-from java2python.compiler.tool import buildAST, transformAST
+from java2python.lang import Lexer, Parser, StringStream, TokenStream,
TreeAdaptor
+
+
+def buildAST(source):
+ """ Returns an AST for the given source. """
+ lexer = Lexer(StringStream(source))
+ parser = Parser(TokenStream(lexer))
+ adapter = TreeAdaptor(lexer, parser)
+ parser.setTreeAdaptor(adapter)
+ scope = parser.javaSource()
+ return scope.tree
+
+
+def buildJavaDocAST(source):
+ """ Returns an AST for the given javadoc source. """
+ from java2python.lang.JavaDocLexer import JavaDocLexer
+ from java2python.lang.JavaDocParser import JavaDocParser
+ lexer = JavaDocLexer(StringStream(source))
+ parser = JavaDocParser(TokenStream(lexer))
+ scope = parser.commentBody()
+ return scope.tree
+
+
+def transformAST(tree, config):
+ """ Walk the tree and apply the transforms in the config. """
+ for selector, call in config.last('astTransforms', ()):
+ for node in selector.walk(tree):
+ call(node, config)
=======================================
--- /trunk/java2python/lang/selector.py Tue Oct 18 21:33:24 2011
+++ /trunk/java2python/lang/selector.py Wed Oct 19 20:00:15 2011
@@ -5,9 +5,8 @@
class Selector(object):
- """ Selector -> base for selectors; provides operator methods.
-
- """
+ """ Selector -> base for selectors; provides operator methods. """
+
def __add__(self, other):
""" C + C => S
@@ -24,6 +23,10 @@
"""
return Descendant(self, other)
+ def __call__(self, *args, **kwds):
+ """ Subclasses must implement. """
+ raise NotImplementedError('Selector class cannot be called.')
+
def __getitem__(self, key):
""" C[n] => S
@@ -41,8 +44,8 @@
"""
return Child(self, other)
-
def walk(self, tree):
+ """ Select items from the tree and from the tree children. """
for item in self(tree):
yield item
for child in tree.children:
@@ -50,6 +53,35 @@
yield item
+class Token(Selector):
+ """ Token(...) -> select tokens by matching attributes.
+
+ """
+ # channel=None, index=None, input=None, line=None, start=None,
stop=None, text=None, type=None
+
+ def __init__(self, **attrs):
+ self.attrs = attrs
+ if isinstance(attrs.get('type'), (basestring, )):
+ self.attrs['type'] = getattr(tokens, attrs.get('type'))
+
+ def __call__(self, tree):
+ items = self.attrs.items()
+ token = tree.token
+
+ def match_or_call(k, v):
+ if callable(v):
+ return v(token)
+ return getattr(token, k)==v
+
+ if all(match_or_call(k, v) for k, v in items if v is not None):
+ yield tree
+
+ def __str__(self):
+ items = self.attrs.items()
+ keys = ('{}={}'.format(k, v) for k, v in items if v is not None)
+ return 'Token({})'.format(', '.join(keys))
+
+
class Nth(Selector):
""" E[n] -> match any slice n of E
@@ -109,29 +141,6 @@
return 'Type({0}{1}:{2})'.format(tokens.map[self.key], val, self.key)
-class Token(Selector):
- """ Token(T) select any token by matching attributes.
-
- Similar to the type selector in CSS.
- """
- # channel=None, index=None, input=None, line=None, start=None,
stop=None, text=None, type=None
-
- def __init__(self, **attrs):
- self.attrs = attrs
- if isinstance(attrs.get('type'), (basestring, )):
- self.attrs['type'] = getattr(tokens, attrs.get('type'))
-
- def __call__(self, tree):
- items = self.attrs.items()
- if all(getattr(tree.token, k)==v for k, v in items if v is not
None):
- yield tree
-
- def __str__(self):
- items = self.attrs.items()
- keys = ('{}={}'.format(k, v) for k, v in items if v is not None)
- return 'Token({})'.format(', '.join(keys))
-
-
class Star(Selector):
""" * select any
=======================================
--- /trunk/test/Makefile Sun Oct 16 16:26:06 2011
+++ /trunk/test/Makefile Wed Oct 19 20:00:15 2011
@@ -14,6 +14,7 @@
all:
$(MAKE) $(test_targets)
+ @cd selector && make
clean:
=======================================
--- /trunk/test/selector/Selector1.java Tue Oct 18 21:33:24 2011
+++ /trunk/test/selector/Selector1.java Wed Oct 19 20:00:15 2011
@@ -1,7 +1,9 @@
class Selector1 {
- void bar() {
+ void bar(int x, int y) {
int foo = 3;
+ String z = "foo";
}
- void foo() {}
-}
+ void foo() {};
+
+}
=======================================
--- /trunk/test/selector/test_all.py Tue Oct 18 21:33:24 2011
+++ /trunk/test/selector/test_all.py Wed Oct 19 20:00:15 2011
@@ -1,37 +1,104 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
-
-from java2python.config import Config
+import os
+import sys
+import unittest
+
from java2python.compiler import buildAST
from java2python.lang import tokens
from java2python.lang.selector import Token, Type
+from java2python.lib import colortools
-if __name__ == '__main__':
- import sys, os
+def setUpModule():
fn = os.path.join(os.path.dirname(__file__), 'Selector1.java')
- source = open(fn).read()
- tree = buildAST(source, Config(()))
- tree.dump(sys.stdout)
-
- selectors = [
-# Type('EXPR'),
-# Type('QUALIFIED_TYPE_IDENT') > Type('IDENT'),
-# Type('CLASS')[2],
-# Type('METHOD_CALL') & Type('IDENT'),
-# Type('IDENT') + Type('IDENT'),
- Type('CLASS') > Type('IDENT'),
- Type('IDENT', 'foo'),
- Token(text='foo'),
- Token(type='BLOCK_SCOPE', line=2) & Token(type='IDENT', text='foo')
- ]
-
- for index, selector in enumerate(selectors):
- print 'Selector Test {0}:\n ==== {1}'.format(index, selector)
- for node in selector.walk(tree):
- name = str(node)
- ntype = tokens.map[node.type]
- args = (name, '') if name == ntype else (ntype, name)
- print '{0}---- match: {1} {2}'.format(*(' '*8, )+args)
- print '{0}---- token: {1}'.format(' '*8, node.dumps() )
- print
+ SelectorTest.tree = buildAST(open(fn).read())
+ SelectorTest.tree.dump(sys.stdout)
+
+
+class SelectorTest(unittest.TestCase):
+ def walk(self, selector):
+ return list(selector.walk(self.tree))
+
+ def assertNodes(self, nodes, length):
+ self.assertTrue(nodes)
+ self.assertEqual(len(nodes), length)
+
+ def shortDescription(self):
+ fs = 'Description: {}\nSelector: {}\n'
+ args = (colortools.cyan(self.description),
colortools.yellow(self.selector))
+ return fs.format(*args)
+
+ @classmethod
+ def make(cls, count):
+ def t(self):
+ nodes = self.walk(self.selector)
+ self.assertNodes(nodes, count)
+ return t
+
+
+class TestIdentChildOfClass(SelectorTest):
+ description = 'select one IDENT node that is a child of a CLASS node'
+ selector = Type('CLASS') > Type('IDENT')
+ test = SelectorTest.make(1)
+
+
+class TestIdentWithText(SelectorTest):
+ description = 'select two IDENT nodes with text "foo"'
+ selector = Type('IDENT', 'foo')
+ test = SelectorTest.make(2)
+
+
+class TestTokensWithText(SelectorTest):
+ description = 'select two nodes with text "foo"'
+ selector = Token(text='foo')
+ test = SelectorTest.make(2)
+
+
+class TestTokenCallableCombo(SelectorTest):
+ description = 'select BLOCK_SCOPE on line 7'
+ selector = Token(type=lambda t: t.type == tokens.BLOCK_SCOPE, line=7)
+ test = SelectorTest.make(1)
+
+
+class TestTokenMultipleCallables(SelectorTest):
+ description = 'select BLOCK_SCOPE on line 2 or 7'
+ selector = Token(type=lambda t: t.type == tokens.BLOCK_SCOPE,
line=lambda t:t.line in (2, 7))
+ test = SelectorTest.make(2)
+
+
+class TestTokenChildCallable(SelectorTest):
+ description = 'select BLOCK_SCOPE with one child IDENT starting
with "f"'
+ selector = Token(type=lambda t: t.type == tokens.BLOCK_SCOPE) &
Token(type='IDENT', text=lambda tok:tok.text.startswith('f'))
+ test = SelectorTest.make(1)
+
+
+class TestNthChildWithExtraChecks(SelectorTest):
+ description = 'select two children of FORMAL_PARAM_STD_DECL at index
2 '
+ selector = Type('FORMAL_PARAM_STD_DECL')[2]
+
+ def test(self):
+ nodes = self.walk(self.selector)
+ self.assertNodes(nodes, 2)
+ self.assertEquals(nodes[0].type, tokens.IDENT)
+ self.assertEquals(nodes[1].type, tokens.IDENT)
+ self.assertEquals(nodes[0].text, 'x')
+ self.assertEquals(nodes[1].text, 'y')
+
+
+class TestDirectChildren(SelectorTest):
+ description = 'select two TYPE nodes that are children of a
VAR_DECLARATION node'
+ selector = Type('VAR_DECLARATION') > Type('TYPE')
+ test = SelectorTest.make(2)
+
+
+class TestSimpleSiblings(SelectorTest):
+ description = 'select three IDENT nodes that are siblings of a
MODIFIER_LIST'
+ selector = Type('MODIFIER_LIST') + Type('IDENT')
+ test = SelectorTest.make(3)
+
+
+class TestClassIdent(SelectorTest):
+ description = 'select one IDENT node that is a child of a CLASS node'
+ selector = Type('CLASS') > Type('IDENT')
+ test = SelectorTest.make(1)