175 # default set of valid characters for the initial letter of symbols

   175 # default set of valid characters for the initial letter of symbols

   176 _syminitletters = set(c for c in [chr(i) for i in xrange(256)]

   176 _syminitletters = set(c for c in [chr(i) for i in xrange(256)]

   177                       if c.isalnum() or c in '._@' or ord(c) > 127)

   177                       if c.isalnum() or c in '._@' or ord(c) > 127)

   178 

   178 

   179 # default set of valid characters for non-initial letters of symbols

   179 # default set of valid characters for non-initial letters of symbols

   182 

   181 

   183 def tokenize(program, lookup=None, syminitletters=None, symletters=None):

   182 def tokenize(program, lookup=None, syminitletters=None, symletters=None):

   184     '''

   183     '''

   185     Parse a revset statement into a stream of tokens

   184     Parse a revset statement into a stream of tokens

   186 

   185