=== modified file 'basil/common/python/DFAParser.py'
--- basil/common/python/DFAParser.py	2010-01-11 09:03:29 +0000
+++ basil/common/python/DFAParser.py	2010-01-11 11:19:56 +0000
@@ -264,7 +264,7 @@
     # Parse all of it.
     result = E_OK
     while result == E_OK:
-        type, tokStr, lineno = tokenizer()
+        type, tokStr, lineno = tokenizer.next()
         result, parseStack, errMsg = addToken(grammar, parseStack, type,
                                               tokStr, lineno)
     if result == E_DONE:
@@ -289,7 +289,7 @@
         fileObj = sys.stdin
     else:
         fileObj = open(inputFile)
-    tokenizer = StdTokenizer.StdTokenizer(inputFile, fileObj.readline)
+    tokenizer = StdTokenizer.StdTokenizer().tokenize(inputFile)
     # ____________________________________________________________
     # Build parser
     import pprint

=== modified file 'basil/common/python/PgenParser.py'
--- basil/common/python/PgenParser.py	2010-01-11 09:03:29 +0000
+++ basil/common/python/PgenParser.py	2010-01-11 11:27:31 +0000
@@ -49,7 +49,7 @@
     MSTART := ( RULE | NEWLINE )* ENDMARKER
     """
     children = []
-    crntToken = tokenizer()
+    crntToken = tokenizer.next()
     while token.ENDMARKER != crntToken[0]:
         if token.NEWLINE == crntToken[0]:
             children.append((crntToken, []))
@@ -58,7 +58,7 @@
             ruleResult, crntToken = handleRule(tokenizer, crntToken)
             children.append(ruleResult)
         if None == crntToken:
-            crntToken = tokenizer()
+            crntToken = tokenizer.next()
     children.append((crntToken, []))
     return (MSTART, children)
 
@@ -73,13 +73,13 @@
         crntToken = tokenzier()
     expect(token.NAME, crntToken)
     children.append((crntToken, []))
-    crntToken = tokenizer()
+    crntToken = tokenizer.next()
     expect(token.COLON, crntToken)
     children.append((crntToken, []))
     rhsResult, crntToken = handleRhs(tokenizer)
     children.append(rhsResult)
     if None == crntToken:
-        crntToken = tokenizer()
+        crntToken = tokenizer.next()
     expect(token.NEWLINE, crntToken)
     children.append((crntToken, []))
     result = (RULE, children)
@@ -97,13 +97,13 @@
     altResult, crntToken = handleAlt(tokenizer, crntToken)
     children.append(altResult)
     if None == crntToken:
-        crntToken = tokenizer()
+        crntToken = tokenizer.next()
     while crntToken[0] == token.VBAR:
         children.append((crntToken, []))
         altResult, crntToken = handleAlt(tokenizer)
         children.append(altResult)
         if None == crntToken:
-            crntToken = tokenizer()
+            crntToken = tokenizer.next()
     result = (RHS, children)
     if __DEBUG__:
         pprint.pprint(result)
@@ -119,12 +119,12 @@
     itemResult, crntToken = handleItem(tokenizer, crntToken)
     children.append(itemResult)
     if None == crntToken:
-        crntToken = tokenizer()
+        crntToken = tokenizer.next()
     while crntToken[0] in (token.LSQB, token.LPAR, token.NAME, token.STRING):
         itemResult, crntToken = handleItem(tokenizer, crntToken)
         children.append(itemResult)
         if None == crntToken:
-            crntToken = tokenizer()
+            crntToken = tokenizer.next()
     return (ALT, children), crntToken
 
 # ______________________________________________________________________
@@ -191,8 +191,7 @@
 def parseString (inString, tokenizer = None):
     if tokenizer == None:
         import StdTokenizer
-        tokenizer = StdTokenizer.StdTokenizer()
-        tokenizer.tokenizeString(inString)
+        tokenizer = StdTokenizer.StdTokenizer().tokenizeString(inString)
     return handleStart(tokenizer)
 
 # ______________________________________________________________________
@@ -200,8 +199,7 @@
 def parseFile (filename, tokenizer = None):
     if tokenizer == None:
         import StdTokenizer
-        tokenizer = StdTokenizer.StdTokenizer()
-        tokenizer.tokenizeFile(filename)
+        tokenizer = StdTokenizer.StdTokenizer().tokenizeFile(filename)
     return handleStart(tokenizer)
 
 # ______________________________________________________________________

=== modified file 'basil/common/python/PyPgen.py'
--- basil/common/python/PyPgen.py	2010-01-11 09:03:29 +0000
+++ basil/common/python/PyPgen.py	2010-01-11 11:25:46 +0000
@@ -366,7 +366,7 @@
         symbols = []
         result = 0L # XXX Can I get this arb. size stuff to translate to C?
         state = dfa[3][dfa[2]]
-        for arc in state[0]: 
+        for arc in state[0]:
             sym = arc[0]
             if sym not in symbols:
                 symbols.append(sym)
@@ -444,7 +444,7 @@
         else:
             inputFile = "<stdin>"
             fileObj = sys.stdin
-        tokenizer = StdTokenizer.StdTokenizer(inputFile, fileObj.readline)
+        tokenizer = StdTokenizer.StdTokenizer().tokenize(fileObj)
         parseTree = DFAParser.parsetok(tokenizer, grammarObj, 257)
         fileObj.close()
         # __________________________________________________

=== modified file 'basil/common/python/pyparser.py'
--- basil/common/python/pyparser.py	2010-01-11 09:03:29 +0000
+++ basil/common/python/pyparser.py	2010-01-11 11:26:42 +0000
@@ -279,8 +279,7 @@
     CPython PyParser_SimpleParseString() (I think.)
     """
     global pygrammar
-    tokenizer = PyTokenizer.PyTokenizer()
-    tokenizer.tokenizeString(source)
+    tokenizer = PyTokenizer.PyTokenizer().tokenizeString(source)
     return STType(DFAParser.parsetok(tokenizer, pygrammar, start))
 
 # ______________________________________________________________________

=== modified file 'basil/lang/python/DFAParser.py'
--- basil/lang/python/DFAParser.py	2010-01-11 09:03:29 +0000
+++ basil/lang/python/DFAParser.py	2010-01-11 11:24:12 +0000
@@ -247,7 +247,7 @@
     # ____________________________________________________________
     def handleDFA (dfa):
         type, name, initial, states, first = dfa
-        
+
         return (type, name, initial, map(handleState, states))
     # ____________________________________________________________
     dfas, labels, start, accel = g
@@ -275,7 +275,7 @@
     # Parse all of it.
     result = E_OK
     while result == E_OK:
-        type, tokStr, lineno = tokenizer()
+        type, tokStr, lineno = tokenizer.next()
         result, parseStack, errMsg = addToken(grammar, parseStack, type,
                                               tokStr, lineno)
     if result == E_DONE:
@@ -300,7 +300,7 @@
         fileObj = sys.stdin
     else:
         fileObj = open(inputFile)
-    tokenizer = StdTokenizer.StdTokenizer(inputFile, fileObj.readline)
+    tokenizer = StdTokenizer.StdTokenizer().tokenize(fileObj)
     # ____________________________________________________________
     # Build parser
     import pprint

=== modified file 'basil/parsing/PyPgen.py'
--- basil/parsing/PyPgen.py	2010-01-11 09:03:29 +0000
+++ basil/parsing/PyPgen.py	2010-01-11 11:34:34 +0000
@@ -380,7 +380,7 @@
         symbols = []
         result = 0L # XXX Can I get this arb. size stuff to translate to C?
         state = dfa[3][dfa[2]]
-        for arc in state[0]: 
+        for arc in state[0]:
             sym = arc[0]
             if sym not in symbols:
                 symbols.append(sym)
@@ -427,7 +427,7 @@
         """PyPgen.__call__()
         """
         nfaGrammar = self.handleStart(ast)
-        
+
         grammar = self.generateDfaGrammar(nfaGrammar)
         self.translateLabels(grammar)
         self.generateFirstSets(grammar)
@@ -476,14 +476,13 @@
         tree.
         """
         return DFAParser.parsetok(tokenizer, self.grammarObj, self.start)
-    
+
     # ____________________________________________________________
     def parseFile (self, filename):
         """PyPgenParser.parseFile
         Accepts filename, returns parse tree.
         """
-        tokenizer = self.tokenizerFactory.getTokenizer()
-        tokenizer.tokenizeFile(filename)
+        tokenizer = self.tokenizerFactory.getTokenizer().tokenizeFile(filename)
         return self.parseTokens(tokenizer)
 
     # ____________________________________________________________
@@ -491,8 +490,7 @@
         """PyPgenParser.parseString
         Accepts input string, return parse tree.
         """
-        tokenizer = self.tokenizerFactory.getTokenizer()
-        tokenizer.tokenizeString(inString)
+        tokenizer = self.tokenizerFactory.getTokenizer().tokenizeString(inString)
         return self.parseTokens(tokenizer)
 
     # ____________________________________________________________
@@ -591,7 +589,7 @@
     else:
         inputFile = "<stdin>"
         fileObj = sys.stdin
-    tokenizer = StdTokenizer.StdTokenizer(inputFile, fileObj.readline)
+    tokenizer = StdTokenizer.StdTokenizer.tokenize(fileObj)
     parser.setStart(257)
     parseTree = parser.parseTokens(tokenizer)
     fileObj.close()

