判断一个词语是否停用..

@param term
@return
481887
public boolean filter(Term term) {

        if (!stop.isEmpty() && (stop.contains(term.getName()))) {
            return true;
        }

        if (!natureStop.isEmpty() && (natureStop.contains(term.natrue().natureStr))) {
            return true;
        }

        if (!regexList.isEmpty()) {
            for (Pattern stopwordPattern : regexList) {
                if (stopwordPattern.matcher(term.getName()).matches()) {
                    return true;
                }
            }
        }

        return false;
    }
666435
private Node parseSubExp(TokenType term) {
        Node node = parseBranch(term);

        if (token.type == term) {
            return node;
        } else if (token.type == TokenType.ALT) {
            ListNode top = ListNode.newAlt(node, null);
            ListNode t = top;
            while (token.type == TokenType.ALT) {
                fetchToken();
                node = parseBranch(term);

                t.setTail(ListNode.newAlt(node, null));
                t = t.tail;
            }

            if (token.type != term) parseSubExpError(term);
            return top;
        } else {
            parseSubExpError(term);
            return null; //not reached
        }
    }
1211967
public static void parseNature(Term term) {
		if (!Nature.NW.equals(term.natrue())) {
			return;
		}

		String name = term.getName();

		if (name.length() <= 3) {
			return;
		}

		// 是否是外国人名
		if (DATDictionary.foreign(term)) {
			term.setNature(NatureLibrary.getNature("nrf"));
			return;
		}

		List<Term> subTerm = term.getSubTerm();

		// TODO:判断是否是机构名

	}
472516
public EquivalentTerm convert(Term t) {
        if (t == null) return null;
        return termCvtr.convert(t);
    }
1165842
private List<Word>[] fullSeg(String text){
        //文本长度
        final int textLen = text.length();
        //以每一个字作为词的开始，所能切分的词
        final List<String>[] sequence = new LinkedList[textLen];
        if(isParallelSeg()){
            //并行化
            List<Integer> list = new ArrayList<>(textLen);
            for(int i=0; i<textLen; i++){
                list.add(i);
            }
            list.parallelStream().forEach(i->sequence[i] = fullSeg(text, i));
        }else {
            //串行化
            for (int i = 0; i < textLen; i++) {
                sequence[i] = fullSeg(text, i);
            }
        }
        if(LOGGER.isDebugEnabled()){
            LOGGER.debug("全切分中间结果：");
            int i=1;
            for(List<String> list : sequence){
                LOGGER.debug("\t"+(i++)+"、"+list);
            }
        }
        //树叶
        List<Node> leaf = new LinkedList<>();
        for(String word : sequence[0]){
            //树根
            Node node = new Node(word);
            //把全切分中间结果（二维数组）转换为合理切分
            buildNode(node, sequence, word.length(), leaf);
        }
        //清理无用数据
        for(int j=0; j<sequence.length; j++){
            sequence[j].clear();
            sequence[j] = null;
        }
        //从所有树叶开始反向遍历出全切分结果
        List<Word>[] res = toWords(leaf);
        leaf.clear();
        return res;
    }
1145002
public T get(Object term) {
        // Check that an index vector does not already exist.
        T v = termToItem.get(term);
        if (v == null) {
            synchronized (this) {
                // Confirm that some other thread has not created an index
                // vector for this term.
                v = termToItem.get(term);
                if (v == null) {
                    // Generate the index vector for this term and store it.
                    v = generator.generate();
                    termToItem.put((String) term, v);
                }
            }
        }
        return v;
    }
1069207
public List<Term> seg(String text)
    {
        char[] charArray = text.toCharArray();
        if (HanLP.Config.Normalization)
        {
            CharTable.normalization(charArray);
        }
        if (config.threadNumber > 1 && charArray.length > 10000)    // 小文本多线程没意义，反而变慢了
        {
            List<String> sentenceList = SentencesUtil.toSentenceList(charArray);
            String[] sentenceArray = new String[sentenceList.size()];
            sentenceList.toArray(sentenceArray);
            //noinspection unchecked
            List<Term>[] termListArray = new List[sentenceArray.length];
            final int per = sentenceArray.length / config.threadNumber;
            WorkThread[] threadArray = new WorkThread[config.threadNumber];
            for (int i = 0; i < config.threadNumber - 1; ++i)
            {
                int from = i * per;
                threadArray[i] = new WorkThread(sentenceArray, termListArray, from, from + per);
                threadArray[i].start();
            }
            threadArray[config.threadNumber - 1] = new WorkThread(sentenceArray, termListArray, (config.threadNumber - 1) * per, sentenceArray.length);
            threadArray[config.threadNumber - 1].start();
            try
            {
                for (WorkThread thread : threadArray)
                {
                    thread.join();
                }
            }
            catch (InterruptedException e)
            {
                logger.severe("线程同步异常：" + TextUtility.exceptionToString(e));
                return Collections.emptyList();
            }
            List<Term> termList = new LinkedList<Term>();
            if (config.offset || config.indexMode > 0)  // 由于分割了句子，所以需要重新校正offset
            {
                int sentenceOffset = 0;
                for (int i = 0; i < sentenceArray.length; ++i)
                {
                    for (Term term : termListArray[i])
                    {
                        term.offset += sentenceOffset;
                        termList.add(term);
                    }
                    sentenceOffset += sentenceArray[i].length();
                }
            }
            else
            {
                for (List<Term> list : termListArray)
                {
                    termList.addAll(list);
                }
            }

            return termList;
        }
//        if (text.length() > 10000)  // 针对大文本，先拆成句子，后分词，避免内存峰值太大
//        {
//            List<Term> termList = new LinkedList<Term>();
//            if (config.offset || config.indexMode)
//            {
//                int sentenceOffset = 0;
//                for (String sentence : SentencesUtil.toSentenceList(charArray))
//                {
//                    List<Term> termOfSentence = segSentence(sentence.toCharArray());
//                    for (Term term : termOfSentence)
//                    {
//                        term.offset += sentenceOffset;
//                        termList.add(term);
//                    }
//                    sentenceOffset += sentence.length();
//                }
//            }
//            else
//            {
//                for (String sentence : SentencesUtil.toSentenceList(charArray))
//                {
//                    termList.addAll(segSentence(sentence.toCharArray()));
//                }
//            }
//
//            return termList;
//        }
        return segSentence(charArray);
    }
74119
@Override
    public List<String> getKeywords(List<Term> termList, int size)
    {
        clear();
        add(termList);
        Collection<TermFrequency> topN = top(size);
        List<String> r = new ArrayList<String>(topN.size());
        for (TermFrequency termFrequency : topN)
        {
            r.add(termFrequency.getTerm());
        }
        return r;
    }
621704
public static String genSequence4Tags(String[] wordArray){
		StringBuilder sb = new StringBuilder();
		for(int i=0; i<wordArray.length; i++) {
			String word = wordArray[i];
			for(int j=0; j<word.length(); j++) {
				char c = word.charAt(j);
				if(Chars.getType(c)==Chars.CharType.B){
					System.err.println(word + " :包含空格(将序列标签转为BMES)");					
				}
				sb.append(c);
				sb.append('\t');
				if(j == 0) {
					if(word.length() == 1)
						sb.append('S');
					else
						sb.append('B');
				} else if(j == word.length()-1) {
					sb.append('E');
				} else {
					sb.append('M');
				}
				sb.append('\n');
			}
		}
		sb.append('\n');
		return sb.toString();
	}
633114
private void foreign(Term[] terms) {
        List<Term> list = new ArrayList<>();

        Term term = null;
        Term temp = null;

        boolean mid = false;

        for (int i = 0; i < terms.length; i++) {
            term = terms[i];
            if (term == null) {
                continue;
            }

            if (DATDictionary.foreign(term)) {
                mid = "·".equals(term.getName());

                if (mid && list.size() == 0) { // mid 不能出现在名字首位
                    mid = false;
                    continue;
                }

                list.add(term);
                continue;
            }

            if (mid) {
                list = list.subList(0, list.size() - 1);
                mid = false;
            }

            if (list.size() == 0) {
                continue;
            }

            if (list.size() == 1) { //只有一个就别识别了
                list.clear();
                continue;
            }

            TermUtil.insertTerm(terms, list, TermNatures.NRF);

            list.clear();

        }

    }
930065
public Long getCount(String type, Map<String, ?> terms) {
		if (terms == null) {
			return 0L;
		}
		MultivaluedMap<String, String> params = new MultivaluedHashMap<>();
		LinkedList<String> list = new LinkedList<>();
		for (Map.Entry<String, ? extends Object> term : terms.entrySet()) {
			String key = term.getKey();
			Object value = term.getValue();
			if (value != null) {
				list.add(key.concat(Config.SEPARATOR).concat(value.toString()));
			}
		}
		if (!terms.isEmpty()) {
			params.put("terms", list);
		}
		params.putSingle(Config._TYPE, type);
		params.putSingle("count", "true");
		Pager pager = new Pager();
		getItems(find("terms", params), pager);
		return pager.getCount();
	}
1126236
function(term, any) {
                    var max = Math.max.apply(null, term.terms),
                        count = 0, idx;

                    if(!any) {
                        for(var i=0; i<term.terms.length; i++) {
                            if(term.terms[i].equals(max)) {
                                idx = i; count++;
                            }
                            if(count > 1) return;
                        }
                    }
                    if(any) {
                        for(i=0; i<term.terms.length; i++) 
                            if(term.terms[i].equals(max)) {
                                idx = i; break;
                            }
                    }
                    return [max, idx, term];
                }
134259
public List<Term> getAllTerms() {
        List<Term> ret = new ArrayList<Term>();
        ret.add(subject);

        List<Term> subjectTerms = subject.getTerms();
        if (subjectTerms != null) {
            ret.addAll(subjectTerms);
            for (final Term term : subjectTerms) {
                ret.addAll(term.getAllTerms());
            }
        }

        if (object != null) {
            ret.addAll(object.getAllTerms());
        }
        return ret;
    }
1182644
private boolean isNoiseWord(String term)
   {
      int len = term.length();
      if (minWordLen > 0 && len < minWordLen)
      {
         return true;
      }
      if (maxWordLen > 0 && len > maxWordLen)
      {
         return true;
      }
      if (stopWords != null && stopWords.contains(term))
      {
         return true;
      }
      return false;
   }
79617
private String computeMorse(BytesRef term) {
    StringBuilder stringBuilder = new StringBuilder();
    int i = term.offset + prefixOffset;
    for (; i < term.length; i++) {
      if (ALPHABET_MORSE.containsKey(term.bytes[i])) {
        stringBuilder.append(ALPHABET_MORSE.get(term.bytes[i]) + " ");
      } else if(term.bytes[i]!=0x00){
        return null;
      } else {
        break;
      }
    }
    return stringBuilder.toString();
  }
1356160
public static Word create(String param)
    {
        if (param == null) return null;
        int cutIndex = param.lastIndexOf('/');
        if (cutIndex <= 0 || cutIndex == param.length() - 1)
        {
            logger.warning("使用 " + param + "创建单个单词失败");
            return null;
        }

        return new Word(param.substring(0, cutIndex), param.substring(cutIndex + 1));
    }
167679
public static int getPosition(Term subTerm, Term term) {
		int startingIndex = -1;
		int j = 0;
		for(int i=0; i<term.getWords().size(); i++) {
			if(term.getWords().get(i).equals(subTerm.getWords().get(j))) {
				j++;
				if(startingIndex == -1) 
					startingIndex = i;
			} else {
				startingIndex = -1;
				j = 0;
			}
			if(j == subTerm.getWords().size())
				return startingIndex;
		}
		return -1;
	}