Given a string, return a list of math symbol tokens
320849
def tokenize(string, language=None, escape='___'):
    
    # Set all words to lowercase
    string = string.lower()

    # Ignore punctuation
    if len(string) and not string[-1].isalnum():
        character = string[-1]
        string = string[:-1] + ' ' + character

    # Parenthesis must have space around them to be tokenized properly
    string = string.replace('(', ' ( ')
    string = string.replace(')', ' ) ')

    if language:
        words = mathwords.words_for_language(language)

        for phrase in words:
            escaped_phrase = phrase.replace(' ', escape)
            string = string.replace(phrase, escaped_phrase)

    tokens = string.split()

    for index, token in enumerate(tokens):
        tokens[index] = token.replace(escape, ' ')

    return tokens
1090033
def _get_tokens_and_tags(parse_str):
  """"""
  tokens = []
  parse_split = parse_str.split(' ')
  for p in parse_split:
    assert p.startswith('(') or p.endswith(')')
    if p.endswith(')'):
      token = p.replace(')', '')
      tokens.append(token)

  return tokens
509172
def __parse(self, tokenlist):
        """"""
        #empty stack
        #iterate over symbollist
        tokenlist = [x for x in tokenlist]
        if not isinstance(tokenlist, list):
            raise TypeError("Expected list, got %s" % tokenlist.__class__.__name__)
        LOG.debug("get_trees: checking list: " + str(tokenlist))
        stack = [(0, Extended_S)]
        while True:
            state = stack[-1][0]
            if len(tokenlist):#FIXME: tokenlist with one element is reported as false
                token = tokenlist[0]
            else:
                token = EndSymbol()
            newdic = self.__parsertable.insert(state, token)
            action = newdic["action"]
            if action == "Fail":
                return False
            elif action == "Accept":
                return True
            if action == "Reduce":
                reductionrule = newdic["rule"]
                #TODO extract len(right side) of the rule and insert left side
                for rsymbol in reversed(reductionrule.rightside):
                    state, symbol = stack.pop() # TODO: check
                state = stack[-1][0]
                state = self.__parsertable.goto(state,reductionrule.leftside[0])
                stack.append((state, reductionrule.leftside[0]))
            elif action == "Shift":
                stack.append((newdic['dest'], tokenlist.pop(0)))
            else:
                raise ValueError("Unknown action")
        return False
263339
def recover_codes(tokens: Iterator[Tokenizer]) -> str:
    

    tokens = iter(tokens)

    s = []

    try:
        head = next(tokens)

    except StopIteration:
        return s

    append = s.append
    lineno = head.lineno

    start_indent = colno = head.colno

    append(head.value)

    colno += len(s[-1])

    for each in tokens:
        n = each.lineno - lineno

        if n:
            append('\n' * n)
            lineno = each.lineno
            colno = each.colno

            if colno - start_indent > 0:
                append(' ' * colno)

        else:
            c = each.colno - colno
            if c:
                colno = each.colno
                if c > 0:
                    append(' ' * c)

        append(each.value)

        colno += len(s[-1])

    return s
1205305
def split_token_in_parts(token):
    
    result = []
    current = []
    for part in token + (':', ):
        if part == ':':
            if current:
                result.append(tuple(current))
                current = []
        else:
            current.append(part)

    return result
745764
def get_reduced_symbols(symbols):
    

    reduced_symbols = []

    for ss in symbols:
        if not (ss in reduced_symbols):
            reduced_symbols.append(ss)

    return reduced_symbols
131796
def _translate_symbols(string):
    """"""
    res = []
    for s in re.split(r'([,.:\s=]+)', string):
        tex_str = _TEX_GREEK_DICTIONARY.get(s)
        if tex_str:
            res.append(tex_str)
        elif s.lower() in greek_letters_set:
            res.append("\\" + s.lower())
        elif s in other_symbols:
            res.append("\\" + s)
        else:
            if re.match(r'^[a-zA-Z]{4,}$', s):
                res.append(r'\text{' + s + '}')
            else:
                res.append(s)
    return "".join(res)
1310116
def get_symbols(self):
        
        return [s if isinstance(s, Symbol) else s.args[0] for s in self.get_literals()]
296831
def parse_expr(tokens, options):
    """"""
    seq = parse_seq(tokens, options)
    if tokens.current() != '|':
        return seq
    result = [Required(*seq)] if len(seq) > 1 else seq
    while tokens.current() == '|':
        tokens.move()
        seq = parse_seq(tokens, options)
        result += [Required(*seq)] if len(seq) > 1 else seq
    return [Either(*result)] if len(result) > 1 else result
197325
def p_string_list(self, t):
        """"""

        if len(t) == 4:
            t[0] = t[1]
            t[0].append(t[3])
        elif t[1]:
            t[0] = [t[1]]
        else:
            t[0] = []
254863
static List<String> getCommaTokens(String inString) {
        String input = inString.trim();
        List<String> tokens = new ArrayList<String>();
        String token;
        while (0 < input.length()) {
            if (COMMA == input.charAt(0)) {
                input = input.substring(1);
            }
            token = getNextCommaToken(input);

            input = input.substring(token.length());
            tokens.add(token.trim());
        }

        return tokens;

    }
308802
def pformat_tokens(self, tokens):
        
        result = []
        for token_value in self.iter_token_values(tokens):
            char = self.token2ascii(token_value)
            if token_value > 0xff:
                result.append("\t$%04x -> %s" % (token_value, repr(char)))
            else:
                result.append("\t  $%02x -> %s" % (token_value, repr(char)))

        return result
1280705
def tokenize(expr):
    
    tokens = []
    escape = False
    cur_token = ''

    for c in expr:
        if escape == True:
            cur_token += c
            escape = False
        else:
            if c == '\\':
                # Next char will be escaped
                escape = True
                continue
            elif c == '[':
                # Next token is of type index (list)
                if len(cur_token) > 0:
                    tokens.append(cur_token)
                    cur_token = ''
            elif c == ']':
                # End of index token. Next token defaults to a key (dict)
                if len(cur_token) > 0:
                    tokens.append(int(cur_token))
                    cur_token = ''
            elif c == '.':
                # End of key token. Next token defaults to a key (dict)
                if len(cur_token) > 0:
                    tokens.append(cur_token)
                    cur_token = ''
            else:
                # Append char to token name
                cur_token += c
    if len(cur_token) > 0:
        tokens.append(cur_token)

    return tokens
321770
def _parse_literal_list(self, indent):
        """"""
        if self._cur_token['type'] not in self._literals:
            raise Exception(
                "Parser failed, _parse_literal_list was called on non-literal"
                " {} on line {}.".format(
                    repr(self._cur_token['value']), self._cur_token['line']
                )
            )

        # find next token after whitespace without incrementing
        temp_position = self._cur_position
        while (
            temp_position < self.num_tokens-1 and (
                self.tokens[temp_position]['type'] is TT.ws or
                self.tokens[temp_position]['type'] in self._literals
            )
        ):
            temp_position += 1
        next_token = self.tokens[temp_position]

        # end of stream
        if next_token['type'] is TT.ws:
            return self._cur_token['value']
        elif next_token['type'] is TT.comma:
            return self._parse_comma_list()
        elif next_token['type'] is TT.lbreak:
            while (
                temp_position < self.num_tokens-1 and
                self.tokens[temp_position]['type'] in (TT.lbreak, TT.ws)
            ):
                temp_position += 1
            if self.tokens[temp_position]['type'] in self._literals:
                return self._parse_newline_list(indent)
            else:
                rval = self._cur_token['value']
                self._increment()
                return rval
        else:
            rval = self._cur_token['value']
            self._increment()
            return rval
1072066
function getTokens (string) {
  let tokens = null
  try { tokens = esprima.tokenize('`'+ string +'`') }
  catch (e) { tokens = [] }
  return tokens
}
828404
def tokenize string
      return nil if string.empty?
      return string if @string_cache.key?(string)
      return @symbol_cache[string] if @symbol_cache.key?(string)

      case string
      # Check for a String type, being careful not to get caught by hash keys, hex values, and
      # special floats (e.g., -.inf).
      when /^[^\d\.:-]?[A-Za-z_\s!@#\$%\^&\*\(\)\{\}\<\>\|\/\\~;=]+/, /\n/
        if string.length > 5
          @string_cache[string] = true
          return string
        end

        case string
        when /^[^ytonf~]/i
          @string_cache[string] = true
          string
        when '~', /^null$/i
          nil
        when /^(yes|true|on)$/i
          true
        when /^(no|false|off)$/i
          false
        else
          @string_cache[string] = true
          string
        end
      when TIME
        begin
          parse_time string
        rescue ArgumentError
          string
        end
      when /^\d{4}-(?:1[012]|0\d|\d)-(?:[12]\d|3[01]|0\d|\d)$/
        require 'date'
        begin
          class_loader.date.strptime(string, '%Y-%m-%d')
        rescue ArgumentError
          string
        end
      when /^\.inf$/i
        Float::INFINITY
      when /^-\.inf$/i
        -Float::INFINITY
      when /^\.nan$/i
        Float::NAN
      when /^:./
        if string =~ /^:(["'])(.*)\1/
          @symbol_cache[string] = class_loader.symbolize($2.sub(/^:/, ''))
        else
          @symbol_cache[string] = class_loader.symbolize(string.sub(/^:/, ''))
        end
      when /^[-+]?[0-9][0-9_]*(:[0-5]?[0-9]){1,2}$/
        i = 0
        string.split(':').each_with_index do |n,e|
          i += (n.to_i * 60 ** (e - 2).abs)
        end
        i
      when /^[-+]?[0-9][0-9_]*(:[0-5]?[0-9]){1,2}\.[0-9_]*$/
        i = 0
        string.split(':').each_with_index do |n,e|
          i += (n.to_f * 60 ** (e - 2).abs)
        end
        i
      when FLOAT
        if string =~ /\A[-+]?\.\Z/
          @string_cache[string] = true
          string
        else
          Float(string.gsub(/[,_]|\.([Ee]|$)/, '\1'))
        end
      else
        int = parse_int string.gsub(/[,_]/, '')
        return int if int

        @string_cache[string] = true
        string
      end
    end
157699
def _segment_with_tokens(text, tokens):
        """"""
        list_form = []
        text_ptr = 0
        for token in tokens:
            inter_token_string = []
            while not text[text_ptr:].startswith(token):
                inter_token_string.append(text[text_ptr])
                text_ptr += 1
                if text_ptr >= len(text):
                    raise ValueError("Tokenization produced tokens that do not belong in string!")
            text_ptr += len(token)
            if inter_token_string:
                list_form.append(''.join(inter_token_string))
            list_form.append(token)
        if text_ptr < len(text):
            list_form.append(text[text_ptr:])
        return list_form