=== modified file 'basil/common/python/PyTokenizer.py'
--- basil/common/python/PyTokenizer.py	2010-01-11 09:03:29 +0000
+++ basil/common/python/PyTokenizer.py	2010-01-11 09:38:15 +0000
@@ -14,18 +14,12 @@
 # ______________________________________________________________________
 
 import pytokenize
-from TokenUtils import AbstractTokenizer
+from TokenUtils import Tokenizer
 
 # ______________________________________________________________________
 
-class PyTokenizer (AbstractTokenizer):
-    """Class PyTokenizer
-    """
-    # ____________________________________________________________
-    def __init__ (self, filename = None, linereader = None):
-        """PyTokenizer.__init__
-        """
-        AbstractTokenizer.__init__(self, pytokenize, filename, linereader)
+class PyTokenizer (Tokenizer):
+    pass
 
 # ______________________________________________________________________
 

=== modified file 'basil/common/python/StdTokenizer.py'
--- basil/common/python/StdTokenizer.py	2010-01-11 09:03:29 +0000
+++ basil/common/python/StdTokenizer.py	2010-01-11 09:38:41 +0000
@@ -13,11 +13,11 @@
 # ______________________________________________________________________
 
 import tokenize
-from TokenUtils import AbstractTokenizer
+from TokenUtils import Tokenizer
 
 # ______________________________________________________________________
 
-class StdTokenizer (AbstractTokenizer):
+class StdTokenizer (Tokenizer):
     """Class StdTokenizer
     Wrapper class (along with maintaining state and whatnot) for the
     Python tokenizer in the standard library.
@@ -28,15 +28,6 @@
     Note that some information is lost, as the token generator returns
     more information than the parser needs (XXX should this change?)
     """
-    # ____________________________________________________________
-    def __init__ (self, filename = None, linereader = None):
-        """StdTokenizer.__init__()
-        Args:
-        self - Object instance
-        linereader - Callable readline()-compatible object that can be passed
-        directly to the generate_tokens() function.
-        """
-        AbstractTokenizer.__init__(self, tokenize, filename, linereader)
 
 # ______________________________________________________________________
 

=== modified file 'basil/common/python/TokenUtils.py'
--- basil/common/python/TokenUtils.py	2010-01-11 09:12:42 +0000
+++ basil/common/python/TokenUtils.py	2010-01-11 09:42:36 +0000
@@ -11,7 +11,7 @@
 
 import StringIO
 import string
-import token
+import tokenize
 
 # ______________________________________________________________________
 
@@ -19,124 +19,211 @@
 
 # ______________________________________________________________________
 
-operatorMap = {
-        '(' : token.LPAR,
-        ')' : token.RPAR,
-        '[' : token.LSQB,
-        ']' : token.RSQB,
-        ':' : token.COLON,
-        ',' : token.COMMA,
-        ';' : token.SEMI,
-        '+' : token.PLUS,
-        '+=' : token.PLUSEQUAL,
-        '-' : token.MINUS,
-        '-=' : token.MINEQUAL,
-        '*' : token.STAR,
-        '**' : token.DOUBLESTAR,
-        '**=' : token.DOUBLESTAREQUAL,
-        '*=' : token.STAREQUAL,
-        '/' : token.SLASH,
-        '//' : token.DOUBLESLASH,
-        '//=' : token.DOUBLESLASHEQUAL,
-        '/=' : token.SLASHEQUAL,
-        '|' : token.VBAR,
-        '|=' : token.VBAREQUAL,
-        '&' : token.AMPER,
-        '&=' : token.AMPEREQUAL,
-        '<' : token.LESS,
-        '<>' : token.NOTEQUAL,
-        '<=' : token.LESSEQUAL,
-        '<<' : token.LEFTSHIFT,
-        '<<=' : token.LEFTSHIFTEQUAL,
-        '>' : token.GREATER,
-        '>=' : token.GREATEREQUAL,
-        '>>' : token.RIGHTSHIFT,
-        '>>=' : token.RIGHTSHIFTEQUAL,
-        '=' : token.EQUAL,
-        '==' : token.EQEQUAL,
-        '.' : token.DOT,
-        '%' : token.PERCENT,
-        '%=' : token.PERCENTEQUAL,
-        '`' : token.BACKQUOTE,
-        '{' : token.LBRACE,
-        '}' : token.RBRACE,
-        '^' : token.CIRCUMFLEX,
-        '^=' : token.CIRCUMFLEXEQUAL,
-        '~' : token.TILDE,
-        '!=' : token.NOTEQUAL
-    }
-
-# ______________________________________________________________________
-
 def testTokenizer (TokenizerClass):
     """testTokenizer()
     Run some silly little test on the tokenizer class argument.
     """
     import sys
     if len(sys.argv) == 1:
-        tokenizer = TokenizerClass("<stdin>", sys.stdin.readline)
+        tokenizer = TokenizerClass().tokenize(sys.stdin)
     else:
-        tokenizer = TokenizerClass()
-        tokenizer.tokenizeFile(sys.argv[1])
-    tokenData = (token.NEWLINE, None, 0)
-    while tokenData[0] not in (token.ENDMARKER, token.ERRORTOKEN):
-        tokenData = tokenizer()
+        tokenizer = TokenizerClass().tokenizeFile(sys.argv[1])
+    tokenData = (tokenize.NEWLINE, None, 0)
+    while tokenData[0] not in (tokenize.ENDMARKER, tokenize.ERRORTOKEN):
+        tokenData = tokenizer.next()
         print tokenData
 
 # ______________________________________________________________________
 
-class AbstractTokenizer:
-    """Class AbstractTokenizer
+class Tokenizer (object) :
+    """A simple lexical analyser based on Python's tokenize module.
+
+    The differences with tokenize module are:
+     - new simple tokens may be added (just strings, no regexps)
+     - Python's token may be not included
+     - tokens may be automatically skipped (removed from the output)
+     - tokenize.OP kind is refined (eg, ':' gets kind tokenize.COLON)
     """
+    operatorMap = {
+        '(' : tokenize.LPAR,
+        ')' : tokenize.RPAR,
+        '[' : tokenize.LSQB,
+        ']' : tokenize.RSQB,
+        ':' : tokenize.COLON,
+        ',' : tokenize.COMMA,
+        ';' : tokenize.SEMI,
+        '+' : tokenize.PLUS,
+        '+=' : tokenize.PLUSEQUAL,
+        '-' : tokenize.MINUS,
+        '-=' : tokenize.MINEQUAL,
+        '*' : tokenize.STAR,
+        '**' : tokenize.DOUBLESTAR,
+        '**=' : tokenize.DOUBLESTAREQUAL,
+        '*=' : tokenize.STAREQUAL,
+        '/' : tokenize.SLASH,
+        '//' : tokenize.DOUBLESLASH,
+        '//=' : tokenize.DOUBLESLASHEQUAL,
+        '/=' : tokenize.SLASHEQUAL,
+        '|' : tokenize.VBAR,
+        '|=' : tokenize.VBAREQUAL,
+        '&' : tokenize.AMPER,
+        '&=' : tokenize.AMPEREQUAL,
+        '<' : tokenize.LESS,
+        '<=' : tokenize.LESSEQUAL,
+        '<<' : tokenize.LEFTSHIFT,
+        '<<=' : tokenize.LEFTSHIFTEQUAL,
+        '>' : tokenize.GREATER,
+        '>=' : tokenize.GREATEREQUAL,
+        '>>' : tokenize.RIGHTSHIFT,
+        '>>=' : tokenize.RIGHTSHIFTEQUAL,
+        '=' : tokenize.EQUAL,
+        '==' : tokenize.EQEQUAL,
+        '.' : tokenize.DOT,
+        '%' : tokenize.PERCENT,
+        '%=' : tokenize.PERCENTEQUAL,
+        '`' : tokenize.BACKQUOTE,
+        '{' : tokenize.LBRACE,
+        '}' : tokenize.RBRACE,
+        '^' : tokenize.CIRCUMFLEX,
+        '^=' : tokenize.CIRCUMFLEXEQUAL,
+        '~' : tokenize.TILDE,
+        '!=' : tokenize.NOTEQUAL,
+        '<>' : tokenize.NOTEQUAL,
+        '@' : tokenize.AT
+        }
+    def __init__ (self, python=True, opmap={}, skip=None, **extra) :
+        """Initialize a new instance.
+
+        Expected arguments are:
+         - python: a bool to indicate whether to include or not
+           Python's tokens (default to True)
+         - opmap: a dict to map litteral tokens (given as '...' in the
+           grammar) to token kinds (default to {}). This parameter is
+           useful only to redefine Python's mapping
+         - skip: a collection of tokens that the tokenizer will
+           automatically skip (default to [COMMENT, NL])
+         - additional keywords arguments allow to define new tokens,
+           for instance, providing
+              DOLLAR='$'
+           defines a new token called 'DOLLAR' (its kind will be
+           automatically computed)
+
+        An instance of Tokenizer has the following attributes:
+         - self.operatorMap: a dict mapping operators token literals to the
+           corresponding kind, for instance, ':' is mapped to
+           tokenize.COLON (this can be overridden using argument
+           opmap)
+         - self.tok_name: a replacement of tokenize.tok_name that also
+           include the user-defined tokens
+         - for each token called FOO (including user-defined ones), an
+           attribute self.FOO hols the corresponding kind
+        """
+        self._python = python
+        self.operatorMap = opmap.copy()
+        if python :
+            self.operatorMap = self.__class__.operatorMap.copy()
+            self.operatorMap.update(opmap)
+        else :
+            self.operatorMap = opmap.copy()
+        self.tok_name = {}
+        self._extra = {}
+        if python :
+            for kind, name in tokenize.tok_name.iteritems() :
+                self.tok_name[kind] = name
+                setattr(self, name, kind)
+        last = max(n for n in self.tok_name if n != self.NT_OFFSET)
+        for shift, (name, txt) in enumerate(sorted(extra.iteritems())) :
+            #WARNING: sorted above is required to guaranty that extra
+            # tokens will always get the same number (dict order is
+            # not guaranteed)
+            kind = last + shift
+            if kind >= self.NT_OFFSET :
+                raise TypeError, "too many new tokens"
+            self.tok_name[kind] = name
+            setattr(self, name, kind)
+            self._extra[txt] = kind
+        self.operatorMap.update(self._extra)
+        if skip is None :
+            skip = [self.COMMENT, self.NL]
+        self._skip = set(skip)
+    def __repr__ (self) :
+        """Encodes an instance as Python source code.
+
+        Non-default arguments provided to the constructor are included
+        so that exactly the same Tokenizer instance can be recovered
+        from the returned source code.
+
+        >>> print repr(Tokenizer())
+        Tokenizer()
+        >>> print repr(Tokenizer(DOLLAR='$'))
+        Tokenizer(DOLLAR='$')
+        >>> print repr(Tokenizer(skip=[], DOLLAR='$'))
+        Tokenizer(skip=[], DOLLAR='$')
+        """
+        args = []
+        if not self._python :
+            args.append("python=%s" % self._python)
+        if self._opmap :
+            args.append("opmap=%r" % self._opmap)
+        if self._skip != set([self.COMMENT, self.NL]) :
+            args.append("skip=%r" % list(self._skip))
+        args.extend("%s=%r" % (self.tok_name[kind], txt) for txt, kind
+                    in self._extra.iteritems())
+        return "%s(%s)" % (self.__class__.__name__, ", ".join(args))
+    def tokenize (self, stream) :
+        """Break an input stream into tokens.
+
+        Expected argument is:
+         - stream: a file-like object (with a method readline)
+
+        Return a generator of tokens, SyntaxError is raised whenever an
+        erroneous token is encountered.
+
+        This is basically the same as tokenize.generate_tokens but:
+         - the appropriate tokens are skipped
+         - OP kind is converted according to self.operatorMap
+         - user-defined tokens are handled
+
+        During the iteration, two more attributes can be used:
+         - self.last: last recognized token (ie, last yielded)
+         - self.infile: the input stream passed to method tokenize
+        """
+        self.infile = stream
+        self.last = None
+        err = self.ERRORTOKEN
+        for token in tokenize.generate_tokens(stream.readline) :
+            if token[0] == err :
+                try :
+                    token = (self._extra[token[1]],) + token[1:]
+                except :
+                    raise ParseError(Token(token, self))
+            elif token[0] in self._skip :
+                continue
+            elif token[0] == self.OP :
+                token = (self.operatorMap[token[1]],) + token[1:]
+            self.last = token
+            yield (self.last[0],
+                   self.last[1] or self.tok_name[self.last[0]],
+                   self.last[2][0])
+   # ____________________________________________________________
+    def getOperatorMap (self):
+        """getOperatorMap
+        """
+        return self.operatorMap
     # ____________________________________________________________
-    def __init__ (self, tokenize, filename = None, linereader = None):
-        """AbstractTokenizer.__init__()
+    def tokenizeFile (self, filename):
+        """StdTokenizer.tokenizeFile()
         """
-        self.tokenize = tokenize
         self.filename = filename
-        self.fileObj = None
-        if None == linereader:
-            self.tokenGenerator = None
-        else:
-            self.tokenGenerator = self.tokenize.generate_tokens(linereader)
-    # ____________________________________________________________
-    def tokenizeFile (self, filename):
-       """AbstractTokenizer.tokenizeFile()
-       """
-       self.filename = filename
-       self.fileObj = open(filename)
-       rl = self.fileObj.readline
-       self.tokenGenerator = self.tokenize.generate_tokens(rl)
+        self.fileObj = open(filename)
+        return self.tokenize(self.fileObj)
     # ____________________________________________________________
     def tokenizeString (self, inString):
-        """AbstractTokenizer.tokenizeString()
+        """StdTokenizer.tokenizeString()
         """
         self.filename = "<string>"
-        self.fileObj = None
-        rl = StringIO.StringIO(inString).readline
-        self.tokenGenerator = self.tokenize.generate_tokens(rl)
-    # ____________________________________________________________
-    def __call__ (self):
-        """AbstractTokenizer.__call__()
-        """
-        retVal = None
-        while 1:
-            if None != self.tokenGenerator:
-                (type, name, (lineno, startCol), endPos,
-                 crntLine) = self.tokenGenerator.next()
-                # Drop tokens unique to tokenize
-                if type in (self.tokenize.COMMENT, self.tokenize.NL):
-                    continue
-                elif type == token.OP:
-                    type = operatorMap[name]
-                retVal = (type, name, lineno)
-                break
-            else:
-                # XXX - What kind of error should be raised?
-                raise ValueError, "Uninitialized tokenizer object."
-        if __DEBUG__:
-            print "AbstractTokenizer.__call__():", retVal
-        return retVal
+        self.fileObj = StringIO.StringIO(inString)
+        return self.tokenize(self.fileObj)
 
 # ______________________________________________________________________
 # End of TokenUtils.py

=== modified file 'basil/lang/python/PyTokenizer.py'
--- basil/lang/python/PyTokenizer.py	2010-01-11 09:03:29 +0000
+++ basil/lang/python/PyTokenizer.py	2010-01-11 09:38:13 +0000
@@ -14,15 +14,8 @@
 
 # ______________________________________________________________________
 
-class PyTokenizer (TokenUtils.AbstractTokenizer):
-    """Class PyTokenizer
-    """
-    # ____________________________________________________________
-    def __init__ (self, filename = None, linereader = None):
-        """PyTokenizer.__init__
-        """
-        TokenUtils.AbstractTokenizer.__init__(self, pytokenize, filename,
-                                              linereader)
+class PyTokenizer (TokenUtils.Tokenizer):
+    pass
 
 # ______________________________________________________________________
 

=== modified file 'basil/lang/python/StdTokenizer.py'
--- basil/lang/python/StdTokenizer.py	2010-01-11 09:03:29 +0000
+++ basil/lang/python/StdTokenizer.py	2010-01-11 09:39:02 +0000
@@ -14,7 +14,7 @@
 
 # ______________________________________________________________________
 
-class StdTokenizer (TokenUtils.AbstractTokenizer):
+class StdTokenizer (TokenUtils.Tokenizer):
     """Class StdTokenizer
     Wrapper class (along with maintaining state and whatnot) for the
     Python tokenizer in the standard library.
@@ -25,21 +25,8 @@
     Note that some information is lost, as the token generator returns
     more information than the parser needs (XXX should this change?)
     """
-    # ____________________________________________________________
-    def __init__ (self, filename = None, linereader = None):
-        """StdTokenizer.__init__()
-        Args:
-        self - Object instance
-        linereader - Callable readline()-compatible object that can be passed
-        directly to the generate_tokens() function.
-        """
-        TokenUtils.AbstractTokenizer.__init__(self, tokenize, filename,
-                                              linereader)
-
-# ______________________________________________________________________
-
-stdTokenizerFactory = TokenUtils.TokenizerFactory(StdTokenizer,
-                                                  TokenUtils.operatorMap)
+
+stdTokenizerFactory = TokenUtils.TokenizerFactory(StdTokenizer)
 
 # ______________________________________________________________________
 
@@ -47,6 +34,7 @@
     """main()
     Run a silly little test on the tokenizer class.
     """
+    from TokenUtils import testTokenizer
     testTokenizer(StdTokenizer)
 
 # ______________________________________________________________________

=== modified file 'basil/lang/python/TokenUtils.py'
--- basil/lang/python/TokenUtils.py	2010-01-11 09:12:42 +0000
+++ basil/lang/python/TokenUtils.py	2010-01-11 09:43:19 +0000
@@ -11,57 +11,7 @@
 
 import string
 import StringIO
-import token
-
-# ______________________________________________________________________
-
-operatorMap = {
-        '(' : token.LPAR,
-        ')' : token.RPAR,
-        '[' : token.LSQB,
-        ']' : token.RSQB,
-        ':' : token.COLON,
-        ',' : token.COMMA,
-        ';' : token.SEMI,
-        '+' : token.PLUS,
-        '+=' : token.PLUSEQUAL,
-        '-' : token.MINUS,
-        '-=' : token.MINEQUAL,
-        '*' : token.STAR,
-        '**' : token.DOUBLESTAR,
-        '**=' : token.DOUBLESTAREQUAL,
-        '*=' : token.STAREQUAL,
-        '/' : token.SLASH,
-        '//' : token.DOUBLESLASH,
-        '//=' : token.DOUBLESLASHEQUAL,
-        '/=' : token.SLASHEQUAL,
-        '|' : token.VBAR,
-        '|=' : token.VBAREQUAL,
-        '&' : token.AMPER,
-        '&=' : token.AMPEREQUAL,
-        '<' : token.LESS,
-        '<>' : token.NOTEQUAL,
-        '<=' : token.LESSEQUAL,
-        '<<' : token.LEFTSHIFT,
-        '<<=' : token.LEFTSHIFTEQUAL,
-        '>' : token.GREATER,
-        '>=' : token.GREATEREQUAL,
-        '>>' : token.RIGHTSHIFT,
-        '>>=' : token.RIGHTSHIFTEQUAL,
-        '=' : token.EQUAL,
-        '==' : token.EQEQUAL,
-        '.' : token.DOT,
-        '%' : token.PERCENT,
-        '%=' : token.PERCENTEQUAL,
-        '`' : token.BACKQUOTE,
-        '{' : token.LBRACE,
-        '}' : token.RBRACE,
-        '^' : token.CIRCUMFLEX,
-        '^=' : token.CIRCUMFLEXEQUAL,
-        '~' : token.TILDE,
-        '!=' : token.NOTEQUAL,
-        '@' : token.AT
-    }
+import tokenize
 
 # ______________________________________________________________________
 
@@ -71,36 +21,189 @@
     """
     import sys
     if len(sys.argv) == 1:
-        tokenizer = TokenizerClass("<stdin>", sys.stdin.readline)
+        tokenizer = TokenizerClass().tokenize(sys.stdin)
     else:
-        tokenizer = TokenizerClass()
-        tokenizer.tokenizeFile(sys.argv[1])
-    tokenData = (token.NEWLINE, None, 0)
-    while tokenData[0] not in (token.ENDMARKER, token.ERRORTOKEN):
-        tokenData = tokenizer()
+        tokenizer = TokenizerClass().tokenizeFile(sys.argv[1])
+    tokenData = (tokenize.NEWLINE, None, 0)
+    while tokenData[0] not in (tokenize.ENDMARKER, tokenize.ERRORTOKEN):
+        tokenData = tokenizer.next()
         print tokenData
 
 # ______________________________________________________________________
 
-class AbstractTokenizer:
-    """Class AbstractTokenizer
+class Tokenizer (object) :
+    """A simple lexical analyser based on Python's tokenize module.
+
+    The differences with tokenize module are:
+     - new simple tokens may be added (just strings, no regexps)
+     - Python's token may be not included
+     - tokens may be automatically skipped (removed from the output)
+     - tokenize.OP kind is refined (eg, ':' gets kind tokenize.COLON)
     """
-    # ____________________________________________________________
-    def __init__ (self, tokenize, filename = None, linereader = None):
-        """AbstractTokenizer.__init__()
-        """
-        global operatorMap
-        self.tokenize = tokenize
-        self.filename = filename
-        self.fileObj = None
-        self.operatorMap = operatorMap
-        if None == linereader:
-            self.tokenGenerator = None
-        else:
-            self.tokenGenerator = self.tokenize.generate_tokens(linereader)
-    # ____________________________________________________________
+    operatorMap = {
+        '(' : tokenize.LPAR,
+        ')' : tokenize.RPAR,
+        '[' : tokenize.LSQB,
+        ']' : tokenize.RSQB,
+        ':' : tokenize.COLON,
+        ',' : tokenize.COMMA,
+        ';' : tokenize.SEMI,
+        '+' : tokenize.PLUS,
+        '+=' : tokenize.PLUSEQUAL,
+        '-' : tokenize.MINUS,
+        '-=' : tokenize.MINEQUAL,
+        '*' : tokenize.STAR,
+        '**' : tokenize.DOUBLESTAR,
+        '**=' : tokenize.DOUBLESTAREQUAL,
+        '*=' : tokenize.STAREQUAL,
+        '/' : tokenize.SLASH,
+        '//' : tokenize.DOUBLESLASH,
+        '//=' : tokenize.DOUBLESLASHEQUAL,
+        '/=' : tokenize.SLASHEQUAL,
+        '|' : tokenize.VBAR,
+        '|=' : tokenize.VBAREQUAL,
+        '&' : tokenize.AMPER,
+        '&=' : tokenize.AMPEREQUAL,
+        '<' : tokenize.LESS,
+        '<=' : tokenize.LESSEQUAL,
+        '<<' : tokenize.LEFTSHIFT,
+        '<<=' : tokenize.LEFTSHIFTEQUAL,
+        '>' : tokenize.GREATER,
+        '>=' : tokenize.GREATEREQUAL,
+        '>>' : tokenize.RIGHTSHIFT,
+        '>>=' : tokenize.RIGHTSHIFTEQUAL,
+        '=' : tokenize.EQUAL,
+        '==' : tokenize.EQEQUAL,
+        '.' : tokenize.DOT,
+        '%' : tokenize.PERCENT,
+        '%=' : tokenize.PERCENTEQUAL,
+        '`' : tokenize.BACKQUOTE,
+        '{' : tokenize.LBRACE,
+        '}' : tokenize.RBRACE,
+        '^' : tokenize.CIRCUMFLEX,
+        '^=' : tokenize.CIRCUMFLEXEQUAL,
+        '~' : tokenize.TILDE,
+        '!=' : tokenize.NOTEQUAL,
+        '<>' : tokenize.NOTEQUAL,
+        '@' : tokenize.AT
+        }
+    def __init__ (self, python=True, opmap={}, skip=None, **extra) :
+        """Initialize a new instance.
+
+        Expected arguments are:
+         - python: a bool to indicate whether to include or not
+           Python's tokens (default to True)
+         - opmap: a dict to map litteral tokens (given as '...' in the
+           grammar) to token kinds (default to {}). This parameter is
+           useful only to redefine Python's mapping
+         - skip: a collection of tokens that the tokenizer will
+           automatically skip (default to [COMMENT, NL])
+         - additional keywords arguments allow to define new tokens,
+           for instance, providing
+              DOLLAR='$'
+           defines a new token called 'DOLLAR' (its kind will be
+           automatically computed)
+
+        An instance of Tokenizer has the following attributes:
+         - self.operatorMap: a dict mapping operators token literals to the
+           corresponding kind, for instance, ':' is mapped to
+           tokenize.COLON (this can be overridden using argument
+           opmap)
+         - self.tok_name: a replacement of tokenize.tok_name that also
+           include the user-defined tokens
+         - for each token called FOO (including user-defined ones), an
+           attribute self.FOO hols the corresponding kind
+        """
+        self._python = python
+        self.operatorMap = opmap.copy()
+        if python :
+            self.operatorMap = self.__class__.operatorMap.copy()
+            self.operatorMap.update(opmap)
+        else :
+            self.operatorMap = opmap.copy()
+        self.tok_name = {}
+        self._extra = {}
+        if python :
+            for kind, name in tokenize.tok_name.iteritems() :
+                self.tok_name[kind] = name
+                setattr(self, name, kind)
+        last = max(n for n in self.tok_name if n != self.NT_OFFSET)
+        for shift, (name, txt) in enumerate(sorted(extra.iteritems())) :
+            #WARNING: sorted above is required to guaranty that extra
+            # tokens will always get the same number (dict order is
+            # not guaranteed)
+            kind = last + shift
+            if kind >= self.NT_OFFSET :
+                raise TypeError, "too many new tokens"
+            self.tok_name[kind] = name
+            setattr(self, name, kind)
+            self._extra[txt] = kind
+        self.operatorMap.update(self._extra)
+        if skip is None :
+            skip = [self.COMMENT, self.NL]
+        self._skip = set(skip)
+    def __repr__ (self) :
+        """Encodes an instance as Python source code.
+
+        Non-default arguments provided to the constructor are included
+        so that exactly the same Tokenizer instance can be recovered
+        from the returned source code.
+
+        >>> print repr(Tokenizer())
+        Tokenizer()
+        >>> print repr(Tokenizer(DOLLAR='$'))
+        Tokenizer(DOLLAR='$')
+        >>> print repr(Tokenizer(skip=[], DOLLAR='$'))
+        Tokenizer(skip=[], DOLLAR='$')
+        """
+        args = []
+        if not self._python :
+            args.append("python=%s" % self._python)
+        if self._opmap :
+            args.append("opmap=%r" % self._opmap)
+        if self._skip != set([self.COMMENT, self.NL]) :
+            args.append("skip=%r" % list(self._skip))
+        args.extend("%s=%r" % (self.tok_name[kind], txt) for txt, kind
+                    in self._extra.iteritems())
+        return "%s(%s)" % (self.__class__.__name__, ", ".join(args))
+    def tokenize (self, stream) :
+        """Break an input stream into tokens.
+
+        Expected argument is:
+         - stream: a file-like object (with a method readline)
+
+        Return a generator of tokens, SyntaxError is raised whenever an
+        erroneous token is encountered.
+
+        This is basically the same as tokenize.generate_tokens but:
+         - the appropriate tokens are skipped
+         - OP kind is converted according to self.operatorMap
+         - user-defined tokens are handled
+
+        During the iteration, two more attributes can be used:
+         - self.last: last recognized token (ie, last yielded)
+         - self.infile: the input stream passed to method tokenize
+        """
+        self.infile = stream
+        self.last = None
+        err = self.ERRORTOKEN
+        for token in tokenize.generate_tokens(stream.readline) :
+            if token[0] == err :
+                try :
+                    token = (self._extra[token[1]],) + token[1:]
+                except :
+                    raise ParseError(Token(token, self))
+            elif token[0] in self._skip :
+                continue
+            elif token[0] == self.OP :
+                token = (self.operatorMap[token[1]],) + token[1:]
+            self.last = token
+            yield (self.last[0],
+                   self.last[1] or self.tok_name[self.last[0]],
+                   self.last[2][0])
+   # ____________________________________________________________
     def getOperatorMap (self):
-        """AbstractTokenizer.getOperatorMap
+        """getOperatorMap
         """
         return self.operatorMap
     # ____________________________________________________________
@@ -109,33 +212,14 @@
         """
         self.filename = filename
         self.fileObj = open(filename)
-        rl = self.fileObj.readline
-        self.tokenGenerator = self.tokenize.generate_tokens(rl)
+        return self.tokenize(self.fileObj)
     # ____________________________________________________________
     def tokenizeString (self, inString):
         """StdTokenizer.tokenizeString()
         """
         self.filename = "<string>"
-        self.fileObj = None
-        rl = StringIO.StringIO(inString).readline
-        self.tokenGenerator = self.tokenize.generate_tokens(rl)
-    # ____________________________________________________________
-    def __call__ (self):
-        """StdTokenizer.__call__()
-        """
-        while 1:
-            if None != self.tokenGenerator:
-                (type, name, (lineno, startCol), endPos,
-                 crntLine) = self.tokenGenerator.next()
-                # Drop tokens unique to tokenize
-                if type in (self.tokenize.COMMENT, self.tokenize.NL):
-                    continue
-                elif type == token.OP:
-                    type = self.operatorMap[name]
-                return (type, name, lineno)
-            else:
-                # XXX - What kind of error should be raised?
-                raise ValueError, "Uninitialized tokenizer object."
+        self.fileObj = StringIO.StringIO(inString)
+        return self.tokenize(self.fileObj)
 
 # ______________________________________________________________________
 
@@ -143,8 +227,7 @@
     # ____________________________________________________________
     def __init__ (self, tokenizerClass, opMap = None):
         if None == opMap:
-            global operatorMap
-            opMap = operatorMap
+            opMap = tokenizerClass.operatorMap
         self.tokenizerClass = tokenizerClass
         self.operatorMap = opMap
 

=== modified file 'basil/parsing/PgenParser.py'
--- basil/parsing/PgenParser.py	2010-01-11 09:03:29 +0000
+++ basil/parsing/PgenParser.py	2010-01-11 09:41:45 +0000
@@ -49,7 +49,7 @@
     MSTART := ( RULE | NEWLINE )* ENDMARKER
     """
     children = []
-    crntToken = tokenizer()
+    crntToken = tokenizer.next()
     while token.ENDMARKER != crntToken[0]:
         if token.NEWLINE == crntToken[0]:
             children.append((crntToken, []))
@@ -58,7 +58,7 @@
             ruleResult, crntToken = handleRule(tokenizer, crntToken)
             children.append(ruleResult)
         if None == crntToken:
-            crntToken = tokenizer()
+            crntToken = tokenizer.next()
     children.append((crntToken, []))
     return (MSTART, children)
 
@@ -73,13 +73,13 @@
         crntToken = tokenzier()
     expect(token.NAME, crntToken)
     children.append((crntToken, []))
-    crntToken = tokenizer()
+    crntToken = tokenizer.next()
     expect(token.COLON, crntToken)
     children.append((crntToken, []))
     rhsResult, crntToken = handleRhs(tokenizer)
     children.append(rhsResult)
     if None == crntToken:
-        crntToken = tokenizer()
+        crntToken = tokenizer.next()
     expect(token.NEWLINE, crntToken)
     children.append((crntToken, []))
     result = (RULE, children)
@@ -97,13 +97,13 @@
     altResult, crntToken = handleAlt(tokenizer, crntToken)
     children.append(altResult)
     if None == crntToken:
-        crntToken = tokenizer()
+        crntToken = tokenizer.next()
     while crntToken[0] == token.VBAR:
         children.append((crntToken, []))
         altResult, crntToken = handleAlt(tokenizer)
         children.append(altResult)
         if None == crntToken:
-            crntToken = tokenizer()
+            crntToken = tokenizer.next()
     result = (RHS, children)
     if __DEBUG__:
         pprint.pprint(result)
@@ -119,12 +119,12 @@
     itemResult, crntToken = handleItem(tokenizer, crntToken)
     children.append(itemResult)
     if None == crntToken:
-        crntToken = tokenizer()
+        crntToken = tokenizer.next()
     while crntToken[0] in (token.LSQB, token.LPAR, token.NAME, token.STRING):
         itemResult, crntToken = handleItem(tokenizer, crntToken)
         children.append(itemResult)
         if None == crntToken:
-            crntToken = tokenizer()
+            crntToken = tokenizer.next()
     return (ALT, children), crntToken
 
 # ______________________________________________________________________
@@ -136,13 +136,13 @@
     """
     children = []
     if None == crntToken:
-        crntToken = tokenizer()
+        crntToken = tokenizer.next()
     if crntToken[0] == token.LSQB:
         children.append((crntToken, []))
         rhsResult, crntToken = handleRhs(tokenizer)
         children.append(rhsResult)
         if None == crntToken:
-            crntToken = tokenizer()
+            crntToken = tokenizer.next()
         expect(token.RSQB, crntToken)
         children.append((crntToken, []))
         crntToken = None
@@ -150,7 +150,7 @@
         atomResult, crntToken = handleAtom(tokenizer,crntToken)
         children.append(atomResult)
         if None == crntToken:
-            crntToken = tokenizer()
+            crntToken = tokenizer.next()
         if crntToken[0] in (token.STAR, token.PLUS):
             children.append((crntToken, []))
             crntToken = None
@@ -166,14 +166,14 @@
     """
     children = []
     if None == crntToken:
-        crntToken = tokenizer()
+        crntToken = tokenizer.next()
     tokType = crntToken[0]
     if tokType == token.LPAR:
         children.append((crntToken, []))
         rhsResult, crntToken = handleRhs(tokenizer)
         children.append(rhsResult)
         if None == crntToken:
-            crntToken = tokenizer()
+            crntToken = tokenizer.next()
         expect(token.RPAR, crntToken)
         children.append((crntToken, []))
         #crntToken = None
@@ -191,8 +191,7 @@
 def parseString (inString, tokenizer = None):
     if tokenizer == None:
         from basil.lang.python import StdTokenizer
-        tokenizer = StdTokenizer.StdTokenizer()
-        tokenizer.tokenizeString(inString)
+        tokenizer = StdTokenizer.StdTokenizer().tokenizeString(inString)
     return handleStart(tokenizer)
 
 # ______________________________________________________________________
@@ -200,8 +199,7 @@
 def parseFile (filename, tokenizer = None):
     if tokenizer == None:
         from basil.lang.python import StdTokenizer
-        tokenizer = StdTokenizer.StdTokenizer()
-        tokenizer.tokenizeFile(filename)
+        tokenizer = StdTokenizer.StdTokenizer().tokenizeFile(filename)
     return handleStart(tokenizer)
 
 # ______________________________________________________________________

