Convert a document into a list of tokens.

    This lowercases, tokenizes, de-accents (optional). -- the output are final
    tokens = unicode strings, that won't be processed any further.
331563
def simple_preprocess(doc, deacc=False, min_len=2, max_len=15):
    
    tokens = [
        token for token in tokenize(doc, lower=True, deacc=deacc, errors='ignore')
        if min_len <= len(token) <= max_len and not token.startswith('_')
    ]
    return tokens
49677
def to_unicode_optional_iterator(x):
    
    if isinstance(x, STRING_TYPES):
        return to_unicode(x)

    try:
        l = list(x)
    except TypeError as e:
        assert 'is not iterable' in str(e)
        return x
    else:
        return [ to_unicode(e) for e in l ]
149062
def _convert_token(self, token):
        
        # Beware that ADAL returns a pointer to its own dict, do
        # NOT change it in place
        token = token.copy()

        # If it's from ADAL, expiresOn will be in ISO form.
        # Bring it back to float, using expiresIn
        if "expiresOn" in token and "expiresIn" in token:
            token["expiresOn"] = token['expiresIn'] + time.time()
        return {self._case.sub(r'\1_\2', k).lower(): v
                for k, v in token.items()}
1065304
def tokenize(text, format=None):
    
    text = Text(text)
    text = text.replace("\t", " ")
    tokens = re.findall(patterns, text)
    tokens = [token[0] for token in tokens]
    if format == "text":
        return " ".join(tokens)
    else:
        return tokens
428927
def tokenize(self, tokenizer=None):
        
        t = tokenizer if tokenizer is not None else self.tokenizer
        return WordList(t.tokenize(self.raw))
64490
function( documents ) {
    
    var key, document
    var fillwords = new Array( 0 )
    
    if( this.language && this.language.fillwords ) {
      fillwords = this.language.fillwords
    }
    
    for( key in documents ) {
      
      document = ( documents[ key ] + '' )
      document = document.toLowerCase()
      document = document.split( /\W+/ )
      
      documents[ key ] = document.reduce(
        function ( document, word ) {
          if( !~fillwords.indexOf( word ) && word !== '' ) {
            document[ word ] = ( document[ word ] ? document[ word ] + 1 : 1 )
          }
          return document
        },
        Object.create( null )
      )
      
    }
    
    return documents
    
  }
989352
function(source) {
			// transform tokens
			var tokens = this.lex(source), pos = 0, token;
			for (var i = 0, il = tokens.length; i < il; i++) {
				token = tokens[i];
				token.start = pos;
				token.end = (pos += token.value.length);
			}
			return tokens;
		}
22788
def parse_doc(doc):
    
    if not doc:
        return {}
    out = {}
    for s in doc.split('\n'):
        s = s.strip().split(':', maxsplit=1)
        if len(s) == 2:
            out[s[0]] = s[1]
    return out
608148
function(tokens) {
            	if(typeof tokens === "string") {
            		tokens = [tokens];
            	}
                    var results = [];
            	var rule_count = rules$3.length;
            	var num_tokens = tokens.length;
                    var i, token, r, rule;
                
                    for (i = 0; i < num_tokens; i++) {
                        token = tokens[i];
                        // Check the conversion table
                        if (conversionTable[token.toLowerCase()]) {
                            results = results.concat(conversionTable[token.toLowerCase()].split(/\W+/));
                        }
                        
                        // Apply the rules
                        else {
                            var matched = false;
                            for ( r = 0; r < rule_count; r++) {
                                rule = rules$3[r];
                                if (token.match(rule.regex)) {
                                    results = results.concat(token.replace(rule.regex, rule.output).split(/\W+/));
                                    matched = true;
                                    break;
                                }
                            }
                            if (!matched) {
                                results.push(token);
                            }
                        }
                    }

            	return results;
            }
269682
def get_tokens(self, format_string):
        
        return [t for t in self.tokenize_pattern.split(format_string) if t]
847173
def lexical(token):
    
    lowercase = token.lower()
    first4 = lowercase[:4]
    last4 = lowercase[-4:]
    return OrderedDict([
            ('lowercase', lowercase), 
            ('first4', first4),
            ('last4', last4)
            ])
733002
def tokenize(contents):
    """"""
    tokens = _scan_for_tokens(contents)
    tokens = _compress_tokens(tokens)
    tokens = [token for token in tokens if token.type != TokenType.Whitespace]
    return tokens
1197978
def tokenize(self, docs):
        
        if self.n_jobs == 1:
            return [self._tokenize(doc) for doc in docs]
        else:
            return parallel(self._tokenize, docs, self.n_jobs)
876465
def completer_tokenize(cls, value, min_length=3):
        ''''''
        tokens = list(itertools.chain(*[
            [m for m in n.split("'") if len(m) > min_length]
            for n in value.split(' ')
        ]))
        return list(set([value] + tokens + [' '.join(tokens)]))
263339
def recover_codes(tokens: Iterator[Tokenizer]) -> str:
    

    tokens = iter(tokens)

    s = []

    try:
        head = next(tokens)

    except StopIteration:
        return s

    append = s.append
    lineno = head.lineno

    start_indent = colno = head.colno

    append(head.value)

    colno += len(s[-1])

    for each in tokens:
        n = each.lineno - lineno

        if n:
            append('\n' * n)
            lineno = each.lineno
            colno = each.colno

            if colno - start_indent > 0:
                append(' ' * colno)

        else:
            c = each.colno - colno
            if c:
                colno = each.colno
                if c > 0:
                    append(' ' * c)

        append(each.value)

        colno += len(s[-1])

    return s
666334
def _split_string_to_tokens(text):
  """"""
  if not text:
    return []
  ret = []
  token_start = 0
  # Classify each character in the input string
  is_alnum = [c in _ALPHANUMERIC_CHAR_SET for c in text]
  for pos in xrange(1, len(text)):
    if is_alnum[pos] != is_alnum[pos - 1]:
      token = text[token_start:pos]
      if token != u" " or token_start == 0:
        ret.append(token)
      token_start = pos
  final_token = text[token_start:]
  ret.append(final_token)
  return ret
987135
def token_list_to_text(tokenlist):
    
    ZeroWidthEscape = Token.ZeroWidthEscape
    return ''.join(item[1] for item in tokenlist if item[0] != ZeroWidthEscape)