Same as tokenize, but returns only tokens
    (and at all parantheses levels).
320088
def tokens(istr):
    

    # make a list of all alphanumeric tokens
    toks = re.findall(r'[^\*\\\+\-\^\(\)]+\(?', istr)

    # remove the functions
    return [t for t in toks if not t.endswith('(')]
661874
def tokenize(self, text):
        '''
        
        '''
        start = -1
        tokens = []
        for i, character in enumerate(text):
            if character == ' ' or character == '\t':
                if start >= 0:
                    word = text[start:i]
                    tokens.append({
                        'word': word,
                        'original_text': word,
                        'char_begin': start,
                        'char_end': i})
                    start = -1
            else:
                if start < 0:
                    start = i
        if start >= 0:
            tokens.append({
                'word': text[start:len(text)],
                'original_text': text[start:len(text)],
                'char_begin': start,
                'char_end': len(text)
            })
        return tokens
1280705
def tokenize(expr):
    
    tokens = []
    escape = False
    cur_token = ''

    for c in expr:
        if escape == True:
            cur_token += c
            escape = False
        else:
            if c == '\\':
                # Next char will be escaped
                escape = True
                continue
            elif c == '[':
                # Next token is of type index (list)
                if len(cur_token) > 0:
                    tokens.append(cur_token)
                    cur_token = ''
            elif c == ']':
                # End of index token. Next token defaults to a key (dict)
                if len(cur_token) > 0:
                    tokens.append(int(cur_token))
                    cur_token = ''
            elif c == '.':
                # End of key token. Next token defaults to a key (dict)
                if len(cur_token) > 0:
                    tokens.append(cur_token)
                    cur_token = ''
            else:
                # Append char to token name
                cur_token += c
    if len(cur_token) > 0:
        tokens.append(cur_token)

    return tokens
213562
def tokenize_by_number(s):
    

    r = find_number(s)

    if r == None:
        return [ s ]
    else:
        tokens = []
        if r[0] > 0:
            tokens.append(s[0:r[0]])
        tokens.append( float(s[r[0]:r[1]]) )
        if r[1] < len(s):
            tokens.extend(tokenize_by_number(s[r[1]:]))
        return tokens
    assert False
464289
def process_tokens(
    tokens: Sequence[Token], *, end: str = "\n", sep: str = " "
) -> Tuple[str, str]:
    
    # Flatten the list of tokens in case some of them are of
    # class UnicodeSequence:
    flat_tokens = list()  # type: List[Token]
    for token in tokens:
        if isinstance(token, UnicodeSequence):
            flat_tokens.extend(token.tuple())
        else:
            flat_tokens.append(token)

    with_color = _process_tokens(flat_tokens, end=end, sep=sep, color=True)
    without_color = _process_tokens(flat_tokens, end=end, sep=sep, color=False)
    return (with_color, without_color)
952021
def tokenize(self, source, name=None, filename=None, state=None):
        
        stream = self.tokeniter(source, name, filename, state)
        return TokenStream(self.wrap(stream, name, filename), name, filename)
1205305
def split_token_in_parts(token):
    
    result = []
    current = []
    for part in token + (':', ):
        if part == ':':
            if current:
                result.append(tuple(current))
                current = []
        else:
            current.append(part)

    return result
324856
def get_tokens(tu, extent):
        
        tokens_memory = POINTER(Token)()
        tokens_count = c_uint()

        conf.lib.clang_tokenize(tu, extent, byref(tokens_memory),
                byref(tokens_count))

        count = int(tokens_count.value)

        # If we get no tokens, no memory was allocated. Be sure not to return
        # anything and potentially call a destructor on nothing.
        if count < 1:
            return

        tokens_array = cast(tokens_memory, POINTER(Token * count)).contents

        token_group = TokenGroup(tu, tokens_memory, tokens_count)

        for i in xrange(0, count):
            token = Token()
            token.int_data = tokens_array[i].int_data
            token.ptr_data = tokens_array[i].ptr_data
            token._tu = tu
            token._group = token_group

            yield token
1340724
def tokens(self):
        
        for content in self.contents:
            if isinstance(content, TokenWithPosition):
                for word in content.split():
                    yield word
            else:
                yield content
35232
def _indices(lexer):
    """"""
    first = _expect_token(lexer, {IntegerToken}).value
    rest = _zom_index(lexer)
    return (first, ) + rest
292250
def tokenize(text=""):
    
    if text == '':
        return []
    else:
        # split tokens
        mylist = TOKEN_PATTERN.split(text)
        # don't remove newline \n
        mylist = [TOKEN_REPLACE.sub('', x) for x in mylist if x]
        # remove empty substring
        mylist = [x for x in mylist if x]
        return mylist
475696
def _tokenize(bytes, mimetype, extra_tokens=True):
    

    # Start with only the ASCII bytes. Limit it to 8+ character strings.
    try:
        ascii_bytes = b' '.join(re.compile(b'[\x00\x09\x0A\x0D\x20-\x7E]{8,}').findall(bytes))
        ascii_bytes = ascii_bytes.replace(b'\x00', b'')
    except:
        return []

    tokens = []

    # Find anything sandwiched between ( )
    results = re.compile(b'\(([^\)]+)\)').findall(ascii_bytes)
    #results = re.compile(b'\(([^\(\)]+)\)').findall(ascii_bytes)
    tokens += results

    # Find anything sandwiched between < >
    results = re.compile(b'\<([^\>]+)\>').findall(ascii_bytes)
    #results = re.compile(b'\<([^\<\>]+)\>').findall(ascii_bytes)
    tokens += results

    # Find anything sandwiched between [ ]
    results = re.compile(b'\[([^\]]+)\]').findall(ascii_bytes)
    #results = re.compile(b'\[([^\[\]]+)\]').findall(ascii_bytes)
    tokens += results

    # Find anything sandwiched between { }
    results = re.compile(b'\{([^\}]+)\}').findall(ascii_bytes)
    #results = re.compile(b'\{([^\{\}]+)\}').findall(ascii_bytes)
    tokens += results

    if not mimetype == 'data':

        # Find anything sandwiched between ' '
        results = re.compile(b'\'\s*(.*?)\s*\'').findall(ascii_bytes)
        tokens += results

        # Find anything sandwiched between " "
        results = re.compile(b'\"\s*(.*?)\s*\"').findall(ascii_bytes)
        tokens += results

    # Find anything sandwiched between spaces
    results = re.compile(b'\ ([^\ ]+)\ ').findall(ascii_bytes)
    tokens += results

    # Break the bytes apart if we want to generate even more tokens.
    if extra_tokens:

        # Add each newline as a token.
        lines = ascii_bytes.decode('ascii', errors='ignore').splitlines()
        tokens += [l.encode('ascii') for l in lines]

        # Now replace these characters in the ASCII bytes with spaces to create more tokens.
        replace_these = [b'(', b')', b'<', b'>', b'[', b']', b'{', b'}', b"'", b'"', b'\n', b'\r', b'\t', b'\\', b'`', b';']
        for char in replace_these:
            ascii_bytes = ascii_bytes.replace(char, b' ')

        # Split the bytes on spaces and use the results as tokens.
        tokens += ascii_bytes.split(b' ')

    # Decode the tokens as ASCII.
    tokens = [token.decode('ascii', errors='ignore') for token in tokens]

    # Since we want this to find full URLs and not just potential domain names, we want
    # to ensure that the tokens have at least "http" or "/" in them.
    tokens = list(set([t for t in tokens if ('http' in t or 'ftp' in t or ('/' in t and '.' in t)) and len(t) >= 8]))

    # Remove any tokens that look like they're probably bad based on the regex statements above.
    tokens = [t for t in tokens if not all(x in t for x in ['(', ')'])]
    tokens = [t for t in tokens if not all(x in t for x in ['<', '>'])]
    tokens = [t for t in tokens if not all(x in t for x in ['[', ']'])]
    tokens = [t for t in tokens if not all(x in t for x in ['{', '}'])]
    tokens = [t for t in tokens if not any(x in t for x in ['\t', '\n', '\r'])]
    tokens = [t for t in tokens if t[:1].isalpha() or t[:1].isdigit()]
    tokens = [t for t in tokens if not t.count('\'') > 1]
    tokens = [t for t in tokens if not t.count('\"') > 1]

    # Return a list of the tokens in ASCII format.
    return tokens
876465
def completer_tokenize(cls, value, min_length=3):
        ''''''
        tokens = list(itertools.chain(*[
            [m for m in n.split("'") if len(m) > min_length]
            for n in value.split(' ')
        ]))
        return list(set([value] + tokens + [' '.join(tokens)]))
273190
def _get_tokens(morph_fd):
    '''
    '''
    for line in morph_fd:
        line = line.rstrip()   # remove \r\n
        line = line.split(';', 1)[0]  # strip comments
        squash_token = []  # quoted strings get squashed into one token

        if '<(' in line:  # skip spines, which exist on a single line
            assert ')>' in line, 'Missing end of spine'
            continue

        for token in line.replace('(', ' ( ').replace(')', ' ) ').split():
            if squash_token:
                squash_token.append(token)
                if token.endswith('"'):
                    token = ' '.join(squash_token)
                    squash_token = []
                    yield token
            elif token.startswith('"') and not token.endswith('"'):
                squash_token.append(token)
            else:
                yield token
158809
def _tokenize_wkt(tokens):
    
    negative = False
    for t in tokens:
        if t == '-':
            negative = True
            continue
        else:
            if negative:
                yield '-%s' % t
            else:
                yield t
            negative = False
8439
def find_tokens(self, q):
    """"""
    found_me = []
    if hasattr(self, 'location'):
      if self.location.contains(q):
        found_me = [self]
    elif self._found_by(q):
      found_me = [self]

    cs = [n.find_tokens(q) for n in self._children()]
    return found_me + list(itertools.chain(*cs))
764704
def as_tokens(ctx: List[ParserRuleContext]) -> List[str]:
    
    return [as_token(e) for e in ctx]