segment words with jieba
197966
def _segment_words(sen):
    '''
    
    '''
    words, tags = [], []
    m = _tokenizer.cut(sen, HMM=True)  # HMM更好的识别新词
    for x in m:
        words.append(x.word)
        tags.append(x.flag)
    return words, tags
1206439
public static List<List<Term>> seg2sentence(String text)
    {
        List<List<Term>> sentenceList = SEGMENT.seg2sentence(text);
        for (List<Term> sentence : sentenceList)
        {
            ListIterator<Term> listIterator = sentence.listIterator();
            while (listIterator.hasNext())
            {
                if (!CoreStopWordDictionary.shouldInclude(listIterator.next()))
                {
                    listIterator.remove();
                }
            }
        }

        return sentenceList;
    }
71533
function splitWord (word) {
  Util.assert(word);

  if (word.length < MAX_SAMPLE_SIZE) {
    return [word];
  }

  let chunks = [];
  for (let ii = 0; ii < word.length - MAX_SAMPLE_SIZE + 1; ++ii) {
    chunks.push(word.substring(ii, ii + MAX_SAMPLE_SIZE));
  }
  return chunks;
}
603036
def annotate_segments(self, Z):
        
        # We need a way to go from compressed idices to original indices
        P = Z.copy()
        P[~np.isfinite(P)] = -1
        _, mapping = np.unique(np.cumsum(P >= 0), return_index=True)

        dZ = Z.compressed()
        uniq, idx = np.unique(dZ, return_inverse=True)
        segments = []
        for i, mean_cn in enumerate(uniq):
            if not np.isfinite(mean_cn):
                continue
            for rr in contiguous_regions(idx == i):
                segments.append((mean_cn, mapping[rr]))

        return segments
247204
function( input, charset ) {

    if( /iso-?2022-?jp/i.test( charset ) ) {
      charset = 'shift_jis'
    }

    if( charset ) {
      return MIME.Iconv.decode( Buffer.from( input, 'base64' ), charset )
    } else {
      return Buffer.from( input, 'base64' ).toString()
    }

  }
584405
private List<List<Term>> seg2sentence(String text) {
        List<String> sentenceList = toSentenceList(text);
        List<List<Term>> resultList = new ArrayList<List<Term>>(sentenceList.size());
        for (String sentence : sentenceList) {
            resultList.add(analysis.parseStr(sentence).recognition(sr).getTerms());
        }
        return resultList;
    }
871707
def cut(self, text):
        
        remain = text
        while remain:
            matched = ''
            # 一次加一个字的匹配
            for index in range(len(remain)):
                word = remain[:index + 1]
                if word in self._prefix_set:
                    matched = word
                else:
                    # 前面的字符串是个词语
                    if matched:
                        yield matched
                        matched = ''
                        remain = remain[index:]
                    else:  # 前面为空
                        yield word
                        remain = remain[index + 1:]
                    # 有结果了，剩余的重新开始匹配
                    break
            else:  # 整个文本就是一个词语
                yield remain
                break
376546
protected void segmentAfterRule(String sentence, String normalized, List<String> wordList)
    {
        if (!enableRuleBasedSegment)
        {
            segmenter.segment(sentence, normalized, wordList);
            return;
        }
        int start = 0;
        int end = start;
        byte preType = typeTable[normalized.charAt(end)];
        byte curType;
        while (++end < normalized.length())
        {
            curType = typeTable[normalized.charAt(end)];
            if (curType != preType)
            {
                if (preType == CharType.CT_NUM)
                {
                    // 浮点数识别
                    if ("，,．.".indexOf(normalized.charAt(end)) != -1)
                    {
                        if (end + 1 < normalized.length())
                        {
                            if (typeTable[normalized.charAt(end + 1)] == CharType.CT_NUM)
                            {
                                continue;
                            }
                        }
                    }
                    else if ("年月日时分秒".indexOf(normalized.charAt(end)) != -1)
                    {
                        preType = curType; // 交给统计分词
                        continue;
                    }
                }
                pushPiece(sentence, normalized, start, end, preType, wordList);
                start = end;
            }
            preType = curType;
        }
        if (end == normalized.length())
            pushPiece(sentence, normalized, start, end, preType, wordList);
    }
157699
def _segment_with_tokens(text, tokens):
        """"""
        list_form = []
        text_ptr = 0
        for token in tokens:
            inter_token_string = []
            while not text[text_ptr:].startswith(token):
                inter_token_string.append(text[text_ptr])
                text_ptr += 1
                if text_ptr >= len(text):
                    raise ValueError("Tokenization produced tokens that do not belong in string!")
            text_ptr += len(token)
            if inter_token_string:
                list_form.append(''.join(inter_token_string))
            list_form.append(token)
        if text_ptr < len(text):
            list_form.append(text[text_ptr:])
        return list_form
18236
def from_tokens(cls, text, toks):
        
        sentence_pb = CoreNLP_pb2.Sentence()
        sentence_pb.characterOffsetBegin = 0
        sentence_pb.characterOffsetEnd = len(text)
        sentence_pb.sentenceIndex = 0
        sentence_pb.tokenOffsetBegin = 0
        sentence_pb.tokenOffsetEnd = len(toks)

        # Track progress in sentence and tokens.
        char_idx = 0
        tok_idx = 0

        buf = ""
        token_pb = None
        while char_idx < len(text):
            tok = toks[tok_idx]
            # Scan to the beginning of the token.
            if text[char_idx] != tok[0]:
                buf += text[char_idx]
                char_idx += 1
            # Aha! we have found the token. Assert that they match.
            else:
                assert text[char_idx:char_idx+len(tok)] == tok, "text did not match a token"
                # Create a new Token from this text.
                if token_pb: token_pb.after = buf
                token_pb = sentence_pb.token.add()
                token_pb.before = buf

                token_pb.beginChar = char_idx
                token_pb.endChar = char_idx + len(tok)
                # TODO(chaganty): potentially handle -LRB-?
                token_pb.value = tok
                token_pb.word = tok
                token_pb.originalText = text[char_idx:char_idx+len(tok)]

                buf = ""
                char_idx += len(tok)
                tok_idx += 1
        if token_pb: token_pb.after = buf
        assert tok_idx == len(toks), "text does not match all tokens"
        return AnnotatedSentence.from_pb(sentence_pb)
720033
def segment_str(text: str, phoneme_inventory: Set[str] = PHONEMES) -> str:
    

    text = text.lower()
    text = segment_into_tokens(text, phoneme_inventory)
    return text
769757
@Override
    protected List<Term> segSentence(char[] sentence)
    {
        if (sentence.length == 0) return Collections.emptyList();
        List<Term> termList = roughSegSentence(sentence);
        if (!(config.ner || config.useCustomDictionary || config.speechTagging))
            return termList;
        List<Vertex> vertexList = toVertexList(termList, true);
        if (config.speechTagging)
        {
            Viterbi.compute(vertexList, CoreDictionaryTransformMatrixDictionary.transformMatrixDictionary);
            int i = 0;
            for (Term term : termList)
            {
                if (term.nature != null) term.nature = vertexList.get(i + 1).guessNature();
                ++i;
            }
        }
        if (config.useCustomDictionary)
        {
            combineByCustomDictionary(vertexList);
            termList = convert(vertexList, config.offset);
        }
        return termList;
    }
89691
def segment(text: str) -> str:
    

    if not text or not isinstance(text, str):
        return ""

    if re.search(r"[เแ]" + _C + r"[" + "".join(_UV) + r"]" + r"\w", text):
        search = re.findall(r"[เแ]" + _C + r"[" + "".join(_UV) + r"]" + r"\w", text)
        for i in search:
            text = re.sub(i, "/" + i + "/", text)

    if re.search(_C + r"[" + "".join(_UV1) + r"]" + _C + _C + r"ุ" + r"์", text):
        search = re.findall(
            _C + r"[" + "".join(_UV1) + r"]" + _C + _C + r"ุ" + r"์", text
        )
        for i in search:
            text = re.sub(i, "//" + i + "/", text)

    if re.search(_C + _UV2 + _C, text):
        search = re.findall(_C + _UV2 + _C, text)
        for i in search:
            text = re.sub(i, "/" + i + "/", text)
    re.sub("//", "/", text)

    if re.search("เ" + _C + "า" + "ะ", text):
        search = re.findall("เ" + _C + "า" + "ะ", text)
        for i in search:
            text = re.sub(i, "/" + i + "/", text)

    if re.search("เ" + r"\w\w" + "า" + "ะ", text):
        search = re.findall("เ" + r"\w\w" + "า" + "ะ", text)
        for i in search:
            text = re.sub(i, "/" + i + "/", text)
    text = re.sub("//", "/", text)

    if re.search(_C + "[" + "".join(_UV1) + "]" + _C + _C + "์", text):
        search = re.findall(_C + "[" + "".join(_UV1) + "]" + _C + _C + "์", text)
        for i in search:
            text = re.sub(i, "/" + i + "/", text)

    if re.search("/" + _C + "".join(["ุ", "์"]) + "/", text):
        # แก้ไขในกรณี พัน/ธุ์
        search = re.findall("/" + _C + "".join(["ุ", "์"]) + "/", text)
        for i in search:
            ii = re.sub("/", "", i)
            text = re.sub(i, ii + "/", text)

    text = re.sub("//", "/", text)

    return text.split("/")
635478
public static List<Word> segWithStopWords(String text, SegmentationAlgorithm segmentationAlgorithm){
        return SegmentationFactory.getSegmentation(segmentationAlgorithm).seg(text);
    }
160502
function breakwords () {
  return through2.obj(function (chunk, _, cb) {
    const words = chunk.split(' ')
      .map(i => i.trim())
      .filter(Boolean)
      .filter(i => i !== '(LISTEN)') // ugh.
    this.push(words)
    cb()
  })
}
1277217
def segment(text: str, custom_dict: Trie = None) -> List[str]:
    
    if not text or not isinstance(text, str):
        return []

    return list(_multicut(text, custom_dict=custom_dict))
264174
def convert_examples_to_features(examples, tokenizer, max_seq_length,
                                 is_training):
    """"""

    # Swag is a multiple choice task. To perform this task using Bert,
    # we will use the formatting proposed in "Improving Language
    # Understanding by Generative Pre-Training" and suggested by
    # @jacobdevlin-google in this issue
    # https://github.com/google-research/bert/issues/38.
    #
    # Each choice will correspond to a sample on which we run the
    # inference. For a given Swag example, we will create the 4
    # following inputs:
    # - [CLS] context [SEP] choice_1 [SEP]
    # - [CLS] context [SEP] choice_2 [SEP]
    # - [CLS] context [SEP] choice_3 [SEP]
    # - [CLS] context [SEP] choice_4 [SEP]
    # The model will output a single value for each input. To get the
    # final decision of the model, we will run a softmax over these 4
    # outputs.
    features = []
    for example_index, example in enumerate(examples):
        context_tokens = tokenizer.tokenize(example.context_sentence)
        start_ending_tokens = tokenizer.tokenize(example.start_ending)

        choices_features = []
        for ending_index, ending in enumerate(example.endings):
            # We create a copy of the context tokens in order to be
            # able to shrink it according to ending_tokens
            context_tokens_choice = context_tokens[:]
            ending_tokens = start_ending_tokens + tokenizer.tokenize(ending)
            # Modifies `context_tokens_choice` and `ending_tokens` in
            # place so that the total length is less than the
            # specified length.  Account for [CLS], [SEP], [SEP] with
            # "- 3"
            _truncate_seq_pair(context_tokens_choice, ending_tokens, max_seq_length - 3)

            tokens = ["[CLS]"] + context_tokens_choice + ["[SEP]"] + ending_tokens + ["[SEP]"]
            segment_ids = [0] * (len(context_tokens_choice) + 2) + [1] * (len(ending_tokens) + 1)

            input_ids = tokenizer.convert_tokens_to_ids(tokens)
            input_mask = [1] * len(input_ids)

            # Zero-pad up to the sequence length.
            padding = [0] * (max_seq_length - len(input_ids))
            input_ids += padding
            input_mask += padding
            segment_ids += padding

            assert len(input_ids) == max_seq_length
            assert len(input_mask) == max_seq_length
            assert len(segment_ids) == max_seq_length

            choices_features.append((tokens, input_ids, input_mask, segment_ids))

        label = example.label
        if example_index < 5:
            logger.info("*** Example ***")
            logger.info("swag_id: {}".format(example.swag_id))
            for choice_idx, (tokens, input_ids, input_mask, segment_ids) in enumerate(choices_features):
                logger.info("choice: {}".format(choice_idx))
                logger.info("tokens: {}".format(' '.join(tokens)))
                logger.info("input_ids: {}".format(' '.join(map(str, input_ids))))
                logger.info("input_mask: {}".format(' '.join(map(str, input_mask))))
                logger.info("segment_ids: {}".format(' '.join(map(str, segment_ids))))
            if is_training:
                logger.info("label: {}".format(label))

        features.append(
            InputFeatures(
                example_id = example.swag_id,
                choices_features = choices_features,
                label = label
            )
        )

    return features