将句子切分为词
@param sentence 句子
@return 词集合
429577
private List<Word> segSentence(final String sentence){
        if(sentence.length() == 1){
            if(KEEP_WHITESPACE){
                List<Word> result = new ArrayList<>(1);
                result.add(new Word(KEEP_CASE ? sentence : sentence.toLowerCase()));
                return result;
            }else{
                if(!Character.isWhitespace(sentence.charAt(0))){
                    List<Word> result = new ArrayList<>(1);
                    result.add(new Word(KEEP_CASE ? sentence : sentence.toLowerCase()));
                    return result;
                }
            }
        }
        if(sentence.length() > 1){
            List<Word> list = segImpl(sentence);
            if(list != null){
                if(PERSON_NAME_RECOGNIZE){
                    list = PersonName.recognize(list);
                }
                return list;
            }else{
                LOGGER.error("文本 "+sentence+" 没有获得分词结果");
            }
        }
        return Collections.emptyList();
    }
408801
private static void splitInSentences(HashSet<String> sentencesSet, String text) {
		
		if (inputType == inputTypes.PARAGRAPH) { // split in multiple sentences
			LinkedList<LinkedList<Word>> sentences = Splitting.tokenizeSentences(LVMorphologyReaderAndWriter.getAnalyzer(), text);
			for (LinkedList<Word> sentence : sentences) 
			{
				if(sentence != null)
				{
					String madeSentence = "";
					for(int w = 0; w < sentence.size(); w++)
					{
						madeSentence = madeSentence + " " + sentence.get(w);
					}
					madeSentence = madeSentence.trim();
					//pārbaudam, vai teikuma atšķiribas nav punkts teikuma beigas
					if(madeSentence.length() > 0 && madeSentence.lastIndexOf(".") >= 0) madeSentence = madeSentence.substring(0, madeSentence.length() - 1);
					sentencesSet.add(madeSentence.trim());
				}
			}
		} 
	}
1166880
def chunked(sentence):
    
    # For example, to construct a training vector with the head of previous chunks as a feature.
    # Doing this with Sentence.chunks would discard the punctuation marks and conjunctions
    # (Sentence.chunks only yields Chunk objects), which amy be useful features.
    chunks = []
    for word in sentence:
        if word.chunk is not None:
            if len(chunks) == 0 or chunks[-1] != word.chunk:
                chunks.append(word.chunk)
        else:
            ch = Chink(sentence)
            ch.append(word.copy(ch))
            chunks.append(ch)
    return chunks
200092
private void processLeaf(Leaf leaf, List<String> sentence, List<Span> names) {

    if (leaf != null && leftContractionPart == null) {

      int startOfNamedEntity = -1;
      String leafTag = leaf.getSecondaryTag();

      if (leafTag != null) {
        if (leafTag.contains("<sam->")) {
          String[] lexemes = underlinePattern.split(leaf.getLexeme());
          if (lexemes.length > 1) {
            for (int i = 0; i < lexemes.length - 1; i++) {
              sentence.add(lexemes[i]);
              
              String[] conts = ContractionUtility.expand(lexemes[i]);
              if(conts != null) {
                int end = sentence.size();
                int start = end - 1;
                Span s = new Span(start, end, "default");
                names.add(s);
                {
                  Span[] ss = {s};
                  System.out.println(Arrays.toString(Span.spansToStrings(ss, sentence.toArray(new String[sentence.size()]))));
                }
              }
            }
          }
          leftContractionPart = lexemes[lexemes.length - 1];
          return;
        }
        // if (leaf.getLexeme().contains("_") && leaf.getLexeme().length() > 3)
        // {
        // String tag = leaf.getFunctionalTag();
        // if (tags != null) {
        // if (tags.contains(tag)) {
        // namedEntityTag = leaf.getFunctionalTag();
        // }
        // } else {
        // namedEntityTag = leaf.getFunctionalTag();
        // }
        // }
      }

      // if (contraction) {
      // startOfNamedEntity = sentence.size();
      // }
      //
      sentence.addAll(Arrays.asList(leaf.getLexeme()));// .split("_")
      //
      // if (contraction) {
      // names
      // .add(new Span(startOfNamedEntity, sentence.size()));
      // }

    } else {
      // will handle the contraction
      String tag = leaf.getSecondaryTag();
      String right = leaf.getLexeme();
      if (tag != null && tag.contains("<-sam>")) {
        String[] parts = underlinePattern.split(leaf.getLexeme());
        if(parts != null) {
          // try to join only the first
          String c = PortugueseContractionUtility.toContraction(
              leftContractionPart, parts[0]);

          if (c != null) {
            sentence.add(c);
            names.add(new Span(sentence.size() - 1, sentence.size(), "default"));
          }
          
          for (int i = 1; i < parts.length; i++) {
            sentence.add(parts[i]);
          }
        } else {
          right = leaf.getLexeme();
          String c = PortugueseContractionUtility.toContraction(
              leftContractionPart, right);

          if (c != null) {
            sentence.add(c);
            names.add(new Span(sentence.size() - 1, sentence.size(), "default"));
          } else {
            System.err.println("missing " + leftContractionPart + " + " + right);
            sentence.add(leftContractionPart);
            sentence.add(right);
          }
        }
      } else {
        System.err.println("unmatch" + leftContractionPart + " + " + right);
      }
      leftContractionPart = null;
    }

  }
1168418
public static String sentences(int sentenceCount, boolean supplemental) {
        List<String> sentenceList = new ArrayList<String>();
        for (int i = 0; i < sentenceCount; i++) {
            sentenceList.add(sentence(3, supplemental, 6));
        }

        String sentenceString = StringUtils.join(sentenceList, " ");
        return sentenceString;

    }
33103
public static void parsePattern(List<NR> nrList, List<Vertex> vertexList, final WordNet wordNetOptimum, final WordNet wordNetAll)
    {
        // 拆分UV
        ListIterator<Vertex> listIterator = vertexList.listIterator();
        StringBuilder sbPattern = new StringBuilder(nrList.size());
        NR preNR = NR.A;
        boolean backUp = false;
        int index = 0;
        for (NR nr : nrList)
        {
            ++index;
            Vertex current = listIterator.next();
//            logger.trace("{}/{}", current.realWord, nr);
            switch (nr)
            {
                case U:
                    if (!backUp)
                    {
                        vertexList = new ArrayList<Vertex>(vertexList);
                        listIterator = vertexList.listIterator(index);
                        backUp = true;
                    }
                    sbPattern.append(NR.K.toString());
                    sbPattern.append(NR.B.toString());
                    preNR = B;
                    listIterator.previous();
                    String nowK = current.realWord.substring(0, current.realWord.length() - 1);
                    String nowB = current.realWord.substring(current.realWord.length() - 1);
                    listIterator.set(new Vertex(nowK));
                    listIterator.next();
                    listIterator.add(new Vertex(nowB));
                    continue;
                case V:
                    if (!backUp)
                    {
                        vertexList = new ArrayList<Vertex>(vertexList);
                        listIterator = vertexList.listIterator(index);
                        backUp = true;
                    }
                    if (preNR == B)
                    {
                        sbPattern.append(NR.E.toString());  //BE
                    }
                    else
                    {
                        sbPattern.append(NR.D.toString());  //CD
                    }
                    sbPattern.append(NR.L.toString());
                    // 对串也做一些修改
                    listIterator.previous();
                    String EorD = current.realWord.substring(0, 1);
                    String L = current.realWord.substring(1, current.realWord.length());
                    listIterator.set(new Vertex(EorD));
                    listIterator.next();
                    listIterator.add(new Vertex(L));
                    continue;
                default:
                    sbPattern.append(nr.toString());
                    break;
            }
            preNR = nr;
        }
        String pattern = sbPattern.toString();
//        logger.trace("模式串：{}", pattern);
//        logger.trace("对应串：{}", vertexList);
//        if (pattern.length() != vertexList.size())
//        {
//            logger.warn("人名识别模式串有bug", pattern, vertexList);
//            return;
//        }
        final Vertex[] wordArray = vertexList.toArray(new Vertex[0]);
        final int[] offsetArray = new int[wordArray.length];
        offsetArray[0] = 0;
        for (int i = 1; i < wordArray.length; ++i)
        {
            offsetArray[i] = offsetArray[i - 1] + wordArray[i - 1].realWord.length();
        }
        trie.parseText(pattern, new AhoCorasickDoubleArrayTrie.IHit<NRPattern>()
        {
            @Override
            public void hit(int begin, int end, NRPattern value)
            {
//            logger.trace("匹配到：{}", keyword);
                StringBuilder sbName = new StringBuilder();
                for (int i = begin; i < end; ++i)
                {
                    sbName.append(wordArray[i].realWord);
                }
                String name = sbName.toString();
//            logger.trace("识别出：{}", name);
                // 对一些bad case做出调整
                switch (value)
                {
                    case BCD:
                        if (name.charAt(0) == name.charAt(2)) return; // 姓和最后一个名不可能相等的
//                        String cd = name.substring(1);
//                        if (CoreDictionary.contains(cd))
//                        {
//                            EnumItem<NR> item = PersonDictionary.dictionary.get(cd);
//                            if (item == null || !item.containsLabel(Z)) return; // 三字名字但是后两个字不在词典中，有很大可能性是误命中
//                        }
                        break;
                }
                if (isBadCase(name)) return;

                // 正式算它是一个名字
                if (HanLP.Config.DEBUG)
                {
                    System.out.printf("识别出人名：%s %s\n", name, value);
                }
                int offset = offsetArray[begin];
                wordNetOptimum.insert(offset, new Vertex(Predefine.TAG_PEOPLE, name, ATTRIBUTE, WORD_ID), wordNetAll);
            }
        });
    }
1022914
def is_participle_clause_fragment(sentence):
    """"""
    # short circuit if sentence or fragment doesn't start with a participle
    # past participles can sometimes look like adjectives -- ie, Tired
    if not _begins_with_one_of(sentence, ['VBG', 'VBN', 'JJ']):
        return 0.0

    if _begins_with_one_of(sentence, ['JJ']):
        doc = nlp(sentence)
        fw = [w for w in doc][0]
        # Beautiful toy birds
        if fw.dep_ == 'amod':
            return 0.0

    # short circuit if sentence starts with a gerund and the gerund is the
    # subject.
    
    if _begins_with_one_of(sentence, ['VBG']):
        doc = nlp(sentence)
        fw = [w for w in doc][0]
        # Running is fun
        if fw.dep_.endswith('subj'):
            return 0.0

        fc = [c for c in doc.noun_chunks]
        # Dancing boys can never sing
        if str(fw) in str(fc):
            return 0.0

    positive_prob = models['participle'].predict([_text_to_vector(sentence,
        trigram2idx['participle'], trigram_count['participle'])])[0][1]
    return float(positive_prob)
1374645
public static Map<List<Word>, Float> trigram(List<Word>... sentences){
        Map<List<Word>, Float> map = new HashMap<>();
        //计算多种分词结果的分值
        for(List<Word> sentence : sentences){
            if(map.get(sentence) != null){
                //相同的分词结果只计算一次分值
                continue;
            }
            float score=0;
            //计算其中一种分词结果的分值
            if(sentence.size() > 2){
                for(int i=0; i<sentence.size()-2; i++){
                    String first = sentence.get(i).getText();
                    String second = sentence.get(i+1).getText();
                    String third = sentence.get(i+2).getText();
                    float trigramScore = getScore(first, second, third);
                    if(trigramScore > 0){
                        score += trigramScore;
                    }
                }
            }
            map.put(sentence, score);
        }
        
        return map;
    }
1166599
public Sentence next() throws IOException
    {
        gisb.clear();
        int c, pos = -1;
        
        while ( (c = readNext()) != -1 ) {
            //clear the whitespace of the begainning
            if ( StringUtil.isWhitespace(c) ) continue;
            if ( c == '\n' || c == '\t' || c == '…' ) continue;
            if ( StringUtil.isCnPunctuation(c) ) {
                switch ( (char)c ) {
                case '“':
                case '【':
                case '（':
                case '《':
                case '｛':
                    break;
                default: continue;
                }
            }
            
            if ( StringUtil.isEnPunctuation(c)) {
                switch ( (char)c ) {
                case '"':
                /*case '[':
                case '(':
                case '{':
                case '<':*/
                    break;
                default: continue;
                }
            }
             
            pos = idx;
            gisb.clear().append((char)c);
            
            while ( (c = readNext()) != -1 ) {
                boolean endTag = false;
                
                /*
                 * here, define the sentence end tag
                 * punctuation like the following:
                 * .。\n;；?？!！:： 
                */
                switch ((char)c) {
                case '"': gisb.append('"'); readUntil('"');  break;
                case '“': gisb.append('“'); readUntil('”');  break;
                case '【': gisb.append('【'); readUntil('】'); break;
                case '《': gisb.append('《'); readUntil('》'); break;
                case '.': {
                    int chr = readNext();
                    gisb.append((char)c);
                    if ( StringUtil.isEnLetter(chr) )  {
                        reader.unread(chr);
                        continue;
                    } else {
                        endTag = true;
                    }
                    
                    break;
                }
                case '。':
                case ';':
                case '；':
                case '?':
                case '？':
                case '!':
                case '！':
                case '…': {
                    endTag = true;
                    gisb.append((char)c);
                    break;
                }
                case ':':
                case '：':
                case '\n': {
                    endTag = true;
                    break;
                }
                default:
                    gisb.append((char)c);
                }
                
                if ( endTag ) break;
            }

            //clear the whitespace from the back
            for ( int i = gisb.length() - 1; i >= 0; i-- ) {
                char chr = gisb.charAt(i);
                if ( chr == ' ' || chr == '\t' ) gisb.deleteCharAt(i);
                else break;
            }
            
            if ( gisb.length() <= 1 ) continue;
            return new Sentence(gisb.toString(), pos);
        }
                
        return null;
    }
376546
protected void segmentAfterRule(String sentence, String normalized, List<String> wordList)
    {
        if (!enableRuleBasedSegment)
        {
            segmenter.segment(sentence, normalized, wordList);
            return;
        }
        int start = 0;
        int end = start;
        byte preType = typeTable[normalized.charAt(end)];
        byte curType;
        while (++end < normalized.length())
        {
            curType = typeTable[normalized.charAt(end)];
            if (curType != preType)
            {
                if (preType == CharType.CT_NUM)
                {
                    // 浮点数识别
                    if ("，,．.".indexOf(normalized.charAt(end)) != -1)
                    {
                        if (end + 1 < normalized.length())
                        {
                            if (typeTable[normalized.charAt(end + 1)] == CharType.CT_NUM)
                            {
                                continue;
                            }
                        }
                    }
                    else if ("年月日时分秒".indexOf(normalized.charAt(end)) != -1)
                    {
                        preType = curType; // 交给统计分词
                        continue;
                    }
                }
                pushPiece(sentence, normalized, start, end, preType, wordList);
                start = end;
            }
            preType = curType;
        }
        if (end == normalized.length())
            pushPiece(sentence, normalized, start, end, preType, wordList);
    }
117096
public static boolean hasNature(List<Term> sentence, Nature nature)
    {
        for (Term term : sentence)
        {
            if (term.nature == nature)
            {
                return true;
            }
        }

        return false;
    }
640102
private boolean isRealSentence(AnalyzedSentence sentence) {
    AnalyzedTokenReadings[] tokens = sentence.getTokensWithoutWhitespace();
    if (tokens.length > 0) {
      AnalyzedTokenReadings lastToken = tokens[tokens.length - 1];
      return lastToken.hasPosTag("PKT") && StringUtils.equalsAny(lastToken.getToken(), ".", "?", "!");
    }
    return false;
  }
951157
private void processLeaf(Leaf leaf, List<String> sentence, List<Span> names) {

    if (leaf != null && leftContractionPart == null) {

      String namedEntityTag = null;
      int startOfNamedEntity = -1;

      String leafTag = leaf.getSecondaryTag();

      if (leafTag != null) {
        if (leafTag.contains("<sam->")) {
          String[] lexemes = leaf.getLexeme().split("_");
          if (lexemes.length > 1) {
            for (int i = 0; i < lexemes.length - 1; i++) {
              sentence.add(lexemes[i]);
            }
          }
          leftContractionPart = lexemes[lexemes.length - 1];
          return;
        }
        if (leaf.getLexeme().contains("_") && leaf.getLexeme().length() > 3) {
          String tag = leaf.getFunctionalTag();
          if (tags != null) {
            if (tags.contains(tag)) {
              namedEntityTag = leaf.getFunctionalTag();
            }
          } else {
            namedEntityTag = leaf.getFunctionalTag();
          }
        }
      }

      if (namedEntityTag != null) {
        startOfNamedEntity = sentence.size();
      }

      sentence.addAll(Arrays.asList(leaf.getLexeme().split("_")));

      if (namedEntityTag != null) {
        names
            .add(new Span(startOfNamedEntity, sentence.size(), namedEntityTag));
      }

    } else {
      // will handle the contraction
      String tag = leaf.getSecondaryTag();
      String right = leaf.getLexeme();
      if (tag != null && tag.contains("<-sam>")) {
        right = leaf.getLexeme();
        String c = PortugueseContractionUtility.toContraction(
            leftContractionPart, right);

        if (c != null) {
          sentence.add(c);
        } else {
          System.err.println("missing " + leftContractionPart + " + " + right);
          sentence.add(leftContractionPart);
          sentence.add(right);
        }

      } else {
        System.err.println("unmatch" + leftContractionPart + " + " + right);
      }
      leftContractionPart = null;
    }

  }
495548
def search(self, sentence):
        
        if sentence.__class__.__name__ == "Sentence":
            pass
        elif isinstance(sentence, list) or sentence.__class__.__name__ == "Text":
            a=[]; [a.extend(self.search(s)) for s in sentence]; return a
        elif isinstance(sentence, basestring):
            sentence = Sentence(sentence)
        elif isinstance(sentence, Match) and len(sentence) > 0:
            sentence = sentence[0].sentence.slice(sentence[0].index, sentence[-1].index + 1)
        a = []
        v = self._variations()
        u = {}
        m = self.match(sentence, _v=v)
        while m:
            a.append(m)
            m = self.match(sentence, start=m.words[-1].index+1, _v=v, _u=u)
        return a
877151
public static String sentence(int wordCount, boolean supplemental, int randomWordsToAdd) {
        List<String> wordList = words(wordCount + RandomUtils.nextInt(randomWordsToAdd), supplemental);
        String sentenceString = StringUtils.join(wordList, " ");

        return StringUtils.capitalize(sentenceString + ".");
    }
329471
String[] tokenize(String sentence) {
    TokenizerME tokenizer = new TokenizerME(tokenModel);
    String cleanString = sentence.replace('’', '\'');  // this is the type of apostrophe that OpenNLP expects
    return tokenizer.tokenize(cleanString);
  }
1178342
public Segment toSegment()
    {
        return new Segment()
        {
            @Override
            protected List<Term> segSentence(char[] sentence)
            {
                List<String> wordList = segment(new String(sentence));
                List<Term> termList = new LinkedList<Term>();
                for (String word : wordList)
                {
                    termList.add(new Term(word, null));
                }
                return termList;
            }
        }.enableCustomDictionary(false);
    }