Split the text on spaces and convert each word into
a Token
420996
protected static List<Token> baseTokenize(String text) {
    String[] words = text.split(" ");
    List<Token> tokens = new LinkedList<Token>();
    for (String word : words) {
      tokens.add(new Token(word));
    }
    return tokens;
  }
478019
def _tokenize(self, text, token_class=None):
        
        token_class = token_class or Token
        tokens = {}

        for i, match in enumerate(self.regex.finditer(text)):
            value = match.group(0)

            try:
                token = tokens[value]
            except KeyError:
                type = match.lastgroup
                token = token_class(value, type=type)
                tokens[value] = token

            yield token
418737
def wordify(text):
    
    stopset = set(nltk.corpus.stopwords.words('english'))
    tokens = nltk.WordPunctTokenizer().tokenize(text)
    return [w for w in tokens if w not in stopset]
1327732
def split(text):
    

    # Cleanup text
    text = text.strip()
    text = re.sub('\s+', ' ', text)  # collpse multiple spaces

    space, quote, parts = ' ', '"', []
    part, quoted = '', False

    for char in text:

        # Encoutered beginning double quote
        if char is quote and quoted is False:
            quoted = True
            continue

        # Encountered the ending double quote
        if char is quote and quoted is True:
            quoted = False
            parts.append(part.strip())
            part = ''
            continue

        # Found space in quoted
        if char is space and quoted is True:
            part += char
            continue

        # Found space but not quoted
        if char is space:
            if part:
                parts.append(part)
                part = ''
            continue

        # Found other character
        if char is not space:
            part += char
            continue

    if part:
        parts.append(part.strip())

    return parts
73099
function zoneParser (span) {
  if (whitespace.test (span[0])) { return ["", span]; }
  var tokens = [];
  var sep = [];     // Separation indices.
  var section = toplevel (span);   // textual zones.
  while (section.zone !== null) {
    var zone = section.zone;
    // Add tokens before the zone.
    tokens = tokens.concat (span.slice (0, zone.from)
                                .split (whitespace)
                                .filter (nonEmpty));
    // Add the zone.
    tokens.push (span.slice (zone.from + 2, zone.to - 2));
    // Prepare for next iteration.
    span = span.slice (zone.to);
    section = toplevel (span);
  }
  tokens = tokens.concat (span.slice (0)
                              .split (whitespace)
                              .filter (nonEmpty));
  return tokens;
}
292250
def tokenize(text=""):
    
    if text == '':
        return []
    else:
        # split tokens
        mylist = TOKEN_PATTERN.split(text)
        # don't remove newline \n
        mylist = [TOKEN_REPLACE.sub('', x) for x in mylist if x]
        # remove empty substring
        mylist = [x for x in mylist if x]
        return mylist
733999
def split(self, sep=TOKENS):
        
        if sep != TOKENS:
            return unicode.split(self, sep)
        if len(self) == 0:
            return []
        return [[[x.replace("&slash;", "/") for x in token.split("/")]
            for token in sentence.split(" ")]
                for sentence in unicode.split(self, "\n")]
1354571
def tokenize(string):
    """"""
    for match in TOKENS_REGEX.finditer(string):
        yield Token(match.lastgroup, match.group().strip(), match.span())
1219493
def tokenize(s):
    

    # Crude tokenization
    s = s.replace('+', ' PLUS ').replace('-', ' MINUS ') \
        .replace('/', ' DIVIDE ').replace('*', ' MULTIPLY ')
    words = [w for w in s.split(' ') if w]
    return words
1373600
def wrap_onspace(self, text):

        '''
        '''

        def _truncate(line, word):
            return '{line}{part}{word}'.format(
                        line=line,
                        part=' \n'[(len(line[line.rfind('\n')+1:]) + len(word.split('\n', 1)[0]) >= self.width)],
                        word=word
                    )

        return reduce(_truncate, text.split(' '))
399299
def tokenize(self, text: str, model: object = None):
        
        sentences = re.split(self.pattern, text)
        return sentences
1266667
def split_into_words(s):
  """"""
  s = re.sub(r"\W+", " ", s)
  s = re.sub(r"[_0-9]+", " ", s)
  return s.split()
1052752
def split_sentences(tokens):
    """"""
    begin = 0
    for i, token in enumerate(tokens):
        if is_end_of_sentence(tokens, i):
            yield tokens[begin:i+1]
            begin = i+1
    if begin <= len(tokens)-1:
        yield tokens[begin:]
672325
def split(self, text):
        """"""
        text = cleanup(text)
        return self.sent_detector.tokenize(text.strip())
1225653
def ansi_split(text, _re=re.compile(u"(\x1b\\[(\\d*;?)*\\S)")):
    """"""

    for part in _re.split(text):
        if part:
            yield (bool(_re.match(part)), part)
1048963
def tokenize_paragraphs(cls, text):
        """"""
        paragraphs = []
        paragraphs_first_pass = text.split('\n')
        for p in paragraphs_first_pass:
            paragraphs_second_pass = re.split('\s{4,}', p)
            paragraphs += paragraphs_second_pass

        # Remove empty strings from list
        paragraphs = [p for p in paragraphs if p]
        return paragraphs
855442
def tokenize_string(cls, string, separator):
        """"""
        results = []
        token = ""
        found_escape = False
        for c in string:
            if found_escape:
                if c == separator:
                    token += separator
                else:
                    token += "\\" + c
                found_escape = False
                continue
            if c == "\\":
                found_escape = True
            elif c == separator:
                results.append(token)
                token = ""
            else:
                token += c

        results.append(token)
        return results