Segments an utterance into space delimited characters.
548128
def segment_into_chars(utterance: str) -> str:
    

    if not isinstance(utterance, str):
        raise TypeError("Input type must be a string. Got {}.".format(type(utterance)))

    utterance.strip()
    utterance = utterance.replace(" ", "")
    return " ".join(utterance)
378072
def slugify(text, delim='-'):
    """"""
    result = []
    for word in _punct_re.split((text or '').lower()):
        result.extend(codecs.encode(word, 'ascii', 'replace').split())
    return delim.join([str(r) for r in result])
1010683
private String[] segment(final String builtText) {

    // these are fine because they do not affect offsets
    String line = builtText.trim();
    line = RuleBasedTokenizer.doubleSpaces.matcher(line).replaceAll(" ");

    if (this.isHardParagraph) {
      // convert every (spurious) paragraph in newlines and keep them
      line = paragraph.matcher(line).replaceAll("\n$1");
    } else {
      // end of sentence markers, paragraph mark and beginning of link
      line = endPunctLinkPara.matcher(line).replaceAll("$1\n$2$3");
      line = conventionalPara.matcher(line).replaceAll("$1\n$2$3");
      line = endInsideQuotesPara.matcher(line).replaceAll("$1\n$3$4");
      line = multiDotsParaStarters.matcher(line).replaceAll("$1\n$2$3");
      // remove spurious paragraphs
      line = alphaNumParaLowerNum.matcher(line).replaceAll("$1 $3");
      line = spuriousParagraph.matcher(line).replaceAll(" $2");
    }
    // non-period end of sentence markers (?!) followed by sentence starters.
    line = noPeriodSpaceEnd.matcher(line).replaceAll("$1\n$2");
    // multi-dots followed by sentence starters
    line = multiDotsSpaceStarters.matcher(line).replaceAll("$1\n$2");
    // end of sentence inside quotes or brackets
    line = endInsideQuotesSpace.matcher(line).replaceAll("$1\n$2");
    // end of sentence marker, sentence starter punctuation and upper case.
    line = punctSpaceUpper.matcher(line).replaceAll("$1\n$2");
    // end of sentence markers, maybe space or paragraph mark and beginning of
    // link
    line = endPunctLinkSpace.matcher(line).replaceAll("$1\n$2");

    // special case of multi-punctuation
    line = punctSpaceMultiPunct.matcher(line).replaceAll("$1\n$2");

    // non breaker segments everything else with some exceptions
    final String[] lines = line.split("\n");
    final String[] sentences = this.nonBreaker.segmenterExceptions(lines);
    return sentences;
  }
733002
def tokenize(contents):
    """"""
    tokens = _scan_for_tokens(contents)
    tokens = _compress_tokens(tokens)
    tokens = [token for token in tokens if token.type != TokenType.Whitespace]
    return tokens
433648
def detailed_tokens(tokenizer, text):
    """"""
    node = tokenizer.parseToNode(text)
    node = node.next  # first node is beginning of sentence and empty, skip it
    words = []
    while node.posid != 0:
        surface = node.surface
        base = surface  # a default value. Updated if available later.
        parts = node.feature.split(",")
        pos = ",".join(parts[0:4])
        if len(parts) > 7:
            # this information is only available for words in the tokenizer
            # dictionary
            base = parts[7]
        words.append(ShortUnitWord(surface, base, pos))
        node = node.next
    return words
777123
def segment(text: str, custom_dict: Trie = None) -> List[str]:
    
    if not text or not isinstance(text, str):
        return []

    if not custom_dict:
        custom_dict = DEFAULT_DICT_TRIE

    return list(_onecut(text, custom_dict))
395010
function tokenize(content, rules = parseOrder) {
    let data = []
    if(content) {
        let lines = content.split(/(\r\n|\n)/)
        for (let idx = 0; idx < lines.length; idx++) {
            for (let i = 0; i < lines[idx].length;) {
                let chunk = lines[idx].slice(i)
                let match = rules.some(item => {
                    if (item.pattern.test(chunk)) {
                        let matchObject = chunk.match(item.pattern)
                        i += matchObject[0].length
                        data.push([ item.label, matchObject[0] ])
                        return true
                    }
                })
                if (!match) {
                    data.push([ UNKNOWN, chunk[i] ])
                    i++
                }
            }
        }
    }
    return data
}
1169192
public String wordSegment(String text){
		String ret = text;
		
		if (vnSegmenter == null) return ret;
		ret = vnSegmenter.segmenting(ret);
		return ret;
	}
1375064
def segment(self, text, lower=True, use_stop_words=True, use_speech_tags_filter=False):
        
        jieba_result = cut_for_search(text)
        jieba_result = [w for w in jieba_result]

        # 去除特殊符号
        word_list = [w.strip() for w in jieba_result]
        word_list = [word for word in word_list if len(word) > 0]

        if use_speech_tags_filter == False:
            jieba_result = [w for w in jieba_result]

        if lower:
            word_list = [word.lower() for word in word_list]

        if use_stop_words:
            word_list = [word.strip() for word in word_list if word.strip() not in self.stop_words]

        return word_list
538327
def tokenize_documents(docs):
    """"""
    sep = '\n\n'
    texts = []
    for doc in docs:
        text = '\n\n'.join(['\n'.join(para[SENTENCES]) for para in doc[PARAGRAPHS]])
        doc[TEXT] = text
        del doc[PARAGRAPHS]
        texts.append(Text(doc))
    return texts
89691
def segment(text: str) -> str:
    

    if not text or not isinstance(text, str):
        return ""

    if re.search(r"[เแ]" + _C + r"[" + "".join(_UV) + r"]" + r"\w", text):
        search = re.findall(r"[เแ]" + _C + r"[" + "".join(_UV) + r"]" + r"\w", text)
        for i in search:
            text = re.sub(i, "/" + i + "/", text)

    if re.search(_C + r"[" + "".join(_UV1) + r"]" + _C + _C + r"ุ" + r"์", text):
        search = re.findall(
            _C + r"[" + "".join(_UV1) + r"]" + _C + _C + r"ุ" + r"์", text
        )
        for i in search:
            text = re.sub(i, "//" + i + "/", text)

    if re.search(_C + _UV2 + _C, text):
        search = re.findall(_C + _UV2 + _C, text)
        for i in search:
            text = re.sub(i, "/" + i + "/", text)
    re.sub("//", "/", text)

    if re.search("เ" + _C + "า" + "ะ", text):
        search = re.findall("เ" + _C + "า" + "ะ", text)
        for i in search:
            text = re.sub(i, "/" + i + "/", text)

    if re.search("เ" + r"\w\w" + "า" + "ะ", text):
        search = re.findall("เ" + r"\w\w" + "า" + "ะ", text)
        for i in search:
            text = re.sub(i, "/" + i + "/", text)
    text = re.sub("//", "/", text)

    if re.search(_C + "[" + "".join(_UV1) + "]" + _C + _C + "์", text):
        search = re.findall(_C + "[" + "".join(_UV1) + "]" + _C + _C + "์", text)
        for i in search:
            text = re.sub(i, "/" + i + "/", text)

    if re.search("/" + _C + "".join(["ุ", "์"]) + "/", text):
        # แก้ไขในกรณี พัน/ธุ์
        search = re.findall("/" + _C + "".join(["ุ", "์"]) + "/", text)
        for i in search:
            ii = re.sub("/", "", i)
            text = re.sub(i, ii + "/", text)

    text = re.sub("//", "/", text)

    return text.split("/")
666334
def _split_string_to_tokens(text):
  """"""
  if not text:
    return []
  ret = []
  token_start = 0
  # Classify each character in the input string
  is_alnum = [c in _ALPHANUMERIC_CHAR_SET for c in text]
  for pos in xrange(1, len(text)):
    if is_alnum[pos] != is_alnum[pos - 1]:
      token = text[token_start:pos]
      if token != u" " or token_start == 0:
        ret.append(token)
      token_start = pos
  final_token = text[token_start:]
  ret.append(final_token)
  return ret
501607
def _tokenize(sentence):
    ''''''

    _tokens = nltk.word_tokenize(sentence)
    tokens = [stemmer.stem(tk) for tk in _tokens]
    return tokens
1266667
def split_into_words(s):
  """"""
  s = re.sub(r"\W+", " ", s)
  s = re.sub(r"[_0-9]+", " ", s)
  return s.split()
1048963
def tokenize_paragraphs(cls, text):
        """"""
        paragraphs = []
        paragraphs_first_pass = text.split('\n')
        for p in paragraphs_first_pass:
            paragraphs_second_pass = re.split('\s{4,}', p)
            paragraphs += paragraphs_second_pass

        # Remove empty strings from list
        paragraphs = [p for p in paragraphs if p]
        return paragraphs
264174
def convert_examples_to_features(examples, tokenizer, max_seq_length,
                                 is_training):
    """"""

    # Swag is a multiple choice task. To perform this task using Bert,
    # we will use the formatting proposed in "Improving Language
    # Understanding by Generative Pre-Training" and suggested by
    # @jacobdevlin-google in this issue
    # https://github.com/google-research/bert/issues/38.
    #
    # Each choice will correspond to a sample on which we run the
    # inference. For a given Swag example, we will create the 4
    # following inputs:
    # - [CLS] context [SEP] choice_1 [SEP]
    # - [CLS] context [SEP] choice_2 [SEP]
    # - [CLS] context [SEP] choice_3 [SEP]
    # - [CLS] context [SEP] choice_4 [SEP]
    # The model will output a single value for each input. To get the
    # final decision of the model, we will run a softmax over these 4
    # outputs.
    features = []
    for example_index, example in enumerate(examples):
        context_tokens = tokenizer.tokenize(example.context_sentence)
        start_ending_tokens = tokenizer.tokenize(example.start_ending)

        choices_features = []
        for ending_index, ending in enumerate(example.endings):
            # We create a copy of the context tokens in order to be
            # able to shrink it according to ending_tokens
            context_tokens_choice = context_tokens[:]
            ending_tokens = start_ending_tokens + tokenizer.tokenize(ending)
            # Modifies `context_tokens_choice` and `ending_tokens` in
            # place so that the total length is less than the
            # specified length.  Account for [CLS], [SEP], [SEP] with
            # "- 3"
            _truncate_seq_pair(context_tokens_choice, ending_tokens, max_seq_length - 3)

            tokens = ["[CLS]"] + context_tokens_choice + ["[SEP]"] + ending_tokens + ["[SEP]"]
            segment_ids = [0] * (len(context_tokens_choice) + 2) + [1] * (len(ending_tokens) + 1)

            input_ids = tokenizer.convert_tokens_to_ids(tokens)
            input_mask = [1] * len(input_ids)

            # Zero-pad up to the sequence length.
            padding = [0] * (max_seq_length - len(input_ids))
            input_ids += padding
            input_mask += padding
            segment_ids += padding

            assert len(input_ids) == max_seq_length
            assert len(input_mask) == max_seq_length
            assert len(segment_ids) == max_seq_length

            choices_features.append((tokens, input_ids, input_mask, segment_ids))

        label = example.label
        if example_index < 5:
            logger.info("*** Example ***")
            logger.info("swag_id: {}".format(example.swag_id))
            for choice_idx, (tokens, input_ids, input_mask, segment_ids) in enumerate(choices_features):
                logger.info("choice: {}".format(choice_idx))
                logger.info("tokens: {}".format(' '.join(tokens)))
                logger.info("input_ids: {}".format(' '.join(map(str, input_ids))))
                logger.info("input_mask: {}".format(' '.join(map(str, input_mask))))
                logger.info("segment_ids: {}".format(' '.join(map(str, segment_ids))))
            if is_training:
                logger.info("label: {}".format(label))

        features.append(
            InputFeatures(
                example_id = example.swag_id,
                choices_features = choices_features,
                label = label
            )
        )

    return features
441823
def process_part(self, char):
        ''''''
        if char in self.whitespace or char == self.eol_char:
            # End of the part.
            self.parts.append( ''.join(self.part) )
            self.part = []
            # Switch back to processing a delimiter.
            self.process_char = self.process_delimiter
            if char == self.eol_char:
                self.complete = True
            return
        if char in self.quote_chars:
            # Store the quote type (' or ") and switch to quote processing.
            self.inquote = char
            self.process_char = self.process_quote
            return
        self.part.append(char)