判定相似度的方式：Jaccard相似性系数
@param words1 词列表1
@param words2 词列表2
@return 相似度分值
965987
@Override
    protected double scoreImpl(List<Word> words1, List<Word> words2) {
        if(words1.isEmpty() && words2.isEmpty()){
            return 1.0;
        }
        //HashSet的contains性能要大于ArrayList的contains
        Set<Word> words2Set = new HashSet<>();
        words2Set.addAll(words2);
        //求交集
        Set<String> intersectionSet = new ConcurrentSkipListSet<>();
        words1.parallelStream().forEach(word -> {
            if (words2Set.contains(word)) {
                intersectionSet.add(word.getText());
            }
        });
        //交集的大小
        int intersectionSize = intersectionSet.size();
        //求并集
        Set<String> unionSet = new HashSet<>();
        words1.forEach(word -> unionSet.add(word.getText()));
        words2.forEach(word -> unionSet.add(word.getText()));
        //并集的大小
        int unionSize = unionSet.size();
        //相似度分值
        double score = intersectionSize/(double)unionSize;
        if(LOGGER.isDebugEnabled()) {
            LOGGER.debug("交集的大小：" + intersectionSize);
            LOGGER.debug("并集的大小：" + unionSize);
            LOGGER.debug("相似度分值=" + intersectionSize + "/(double)" + unionSize + "=" + score);
        }
        return score;
    }
785101
public static Map<List<Word>, Float> bigram(List<Word>... sentences){
        Map<List<Word>, Float> map = new HashMap<>();
        Map<String, Float> bigramScores = new HashMap<>();
        //两个连续的bigram补偿粗粒度分值
        //如：美国, 加州, 大学，如果美国, 加州和加州, 大学有分值
        //则美国加州大学也会获得分值
        Map<String, Float> twoBigramScores = new HashMap<>();
        //1、计算多种分词结果的分值
        for(List<Word> sentence : sentences){
            if(map.get(sentence) != null){
                continue;
            }
            float score=0;
            //计算其中一种分词结果的分值
            if(sentence.size() > 1){
                String last="";
                for(int i=0; i<sentence.size()-1; i++){
                    String first = sentence.get(i).getText();
                    String second = sentence.get(i+1).getText();
                    float bigramScore = getScore(first, second);
                    if(bigramScore > 0){
                        if(last.endsWith(first)){
                            twoBigramScores.put(last+second, bigramScores.get(last)+bigramScore);
                            last="";
                        }
                        last = first+second;
                        bigramScores.put(last, bigramScore);
                        score += bigramScore;
                    }
                }
            }
            map.put(sentence, score);
        }
        //2、利用获得的二元模型分值重新计算分词结果的分值
        //补偿细粒度切分获得分值而粗粒度切分未获得分值的情况
        //计算多种分词结果的分值
        if(bigramScores.size() > 0 || twoBigramScores.size() > 0){
            for(List<Word> sentence : map.keySet()){
                //计算其中一种分词结果的分值
                for(Word word : sentence){
                    Float bigramScore = bigramScores.get(word.getText());
                    Float twoBigramScore = twoBigramScores.get(word.getText());
                    Float[] array = {bigramScore, twoBigramScore};
                    for(Float score : array){
                        if(score !=null && score > 0){
                            if(LOGGER.isDebugEnabled()) {
                                LOGGER.debug(word.getText() + " 获得分值：" + score);
                            }
                            float value = map.get(sentence);
                            value += score;
                            map.put(sentence, value);
                        }                    
                    }
                }
            }
        }
        
        return map;
    }
1147636
@SafeVarargs
    public final WordBuilder<I> append(Word<? extends I>... words) {
        if (words.length == 0) {
            return this;
        }

        int allLen = 0;
        for (Word<? extends I> w : words) {
            allLen += w.length();
        }

        ensureAdditionalCapacity(allLen);

        for (Word<? extends I> word : words) {
            int wLen = word.length();
            word.writeToArray(0, array, length, wLen);
            length += wLen;
        }

        return this;
    }
1214221
public Word<I> findSeparatingWord(final S s1, final S s2) {

        final FastMealyState<O> n1 = this.nodeToObservationMap.get(s1);
        final FastMealyState<O> n2 = this.nodeToObservationMap.get(s2);

        return NearLinearEquivalenceTest.findSeparatingWord(this.observationTree, n1, n2, this.alphabet, true);
    }
210653
public void addWords(String... words){
		HashSet<String> wordsSet = CollectionUtil.newHashSet(words);
		for (String word : wordsSet) {
			addWord(word);
		}
	}
1040857
@SuppressWarnings("unchecked")
    @Nonnull
    protected Word<I> concatInternal(Word<? extends I>... words) {
        if (words.length == 0) {
            return this;
        }

        int len = length();

        int totalSize = len;
        for (Word<? extends I> word : words) {
            totalSize += word.length();
        }

        Object[] array = new Object[totalSize];
        writeToArray(0, array, 0, len);
        int currOfs = len;
        for (Word<? extends I> w : words) {
            int wLen = w.length();
            w.writeToArray(0, array, currOfs, wLen);
            currOfs += wLen;
        }

        return new SharedWord<>(array);
    }
434120
def basic_word_sim(word1, word2):
    
    return sum([1 for c in word1 if c in word2]) / max(len(word1), len(word2))
1040855
public static boolean compareStringLists(List<String> list1, List<String> list2) {
        if (list1 == null && list2 == null)
            return true;
        if (list1 == null || list2 == null)
            return false;
        if (list1.size() != list2.size())
            return false;
        for (int i = 0; i < list1.size(); i++) {
            if (!compareStrings(list1.get(i), list2.get(i))) {
                return false;
            }
        }

        return true;
    }
92892
public static List<Word> split(Word word){
        String value = GENERIC_TRIE.get(word.getText());
        if(value==null){
            return null;
        }
        List<Word> words = new ArrayList<>();
        for(String val : value.split("\\s+")){
            words.add(new Word(val));
        }
        if(words.isEmpty()){
            return null;
        }
        return words;
    }
804171
public List<Period> _Combine( List<Period> periods1, List<Period> periods2 )
	{
		List<Period> result = new ArrayList<Period>();

		int count1 = periods1.size();
		int count2 = periods2.size();

		int i = 0;
		int j = 0;
		while( i < count1 && j < count2 )
		{
			if( periods1.get( i ).getFrom().compareTo( periods2.get( j ).from ) <= 0 )
			{
				result.add( periods1.get( i ) );
				i++;
			}
			else
			{
				result.add( periods2.get( j ) );
				j++;
			}
		}
		while( i < count1 )
		{
			result.add( periods1.get( i ) );
			i++;
		}
		while( j < count2 )
		{
			result.add( periods2.get( j ) );
			j++;
		}

		return result;
	}
1094274
@Override
    public Collection<String> wordsNearestSum(INDArray words, int top) {

        if (lookupTable instanceof InMemoryLookupTable) {
            InMemoryLookupTable l = (InMemoryLookupTable) lookupTable;
            INDArray syn0 = l.getSyn0();
            INDArray weights = syn0.norm2(0).rdivi(1).muli(words);
            INDArray distances = syn0.mulRowVector(weights).sum(1);
            INDArray[] sorted = Nd4j.sortWithIndices(distances, 0, false);
            INDArray sort = sorted[0];
            List<String> ret = new ArrayList<>();

            // FIXME: int cast
            if (top > sort.length())
                top = (int) sort.length();
            //there will be a redundant word
            int end = top;
            for (int i = 0; i < end; i++) {
                String add = vocabCache.wordAtIndex(sort.getInt(i));
                if (add == null || add.equals("UNK") || add.equals("STOP")) {
                    end++;
                    if (end >= sort.length())
                        break;
                    continue;
                }
                ret.add(vocabCache.wordAtIndex(sort.getInt(i)));
            }
            return ret;
        }

        Counter<String> distances = new Counter<>();

        for (String s : vocabCache.words()) {
            INDArray otherVec = lookupTable.vector(s);
            double sim = Transforms.cosineSim(words, otherVec);
            distances.incrementCount(s, (float) sim);
        }

        distances.keepTopNElements(top);
        return distances.keySet();
    }
249505
@Override
    protected double scoreImpl(List<Word> words1, List<Word> words2){
        double score = super.scoreImpl(words1, words2);

        score += (scalingFactor * commonPrefixLength() * (1.0 - score));

        return score;
    }
923930
@Override
    protected double scoreImpl(List<Word> words1, List<Word> words2) {
        //用词频来标注词的权重
        taggingWeightWithWordFrequency(words1, words2);
        //构造权重快速搜索容器
        Map<String, Float> weights1 = toFastSearchMap(words1);
        Map<String, Float> weights2 = toFastSearchMap(words2);
        //所有的不重复词
        Set<Word> words = new HashSet<>();
        words.addAll(words1);
        words.addAll(words2);
        //向量的维度为words的大小，每一个维度的权重是词频
        //a.b
        AtomicFloat ab = new AtomicFloat();
        //|a|的平方
        AtomicFloat aa = new AtomicFloat();
        //|b|的平方
        AtomicFloat bb = new AtomicFloat();
        //计算
        words
            .parallelStream()
            .forEach(word -> {
                Float x1 = weights1.get(word.getText());
                Float x2 = weights2.get(word.getText());
                if (x1 != null && x2 != null) {
                    //x1x2
                    float oneOfTheDimension = x1 * x2;
                    //+
                    ab.addAndGet(oneOfTheDimension);
                }
                if (x1 != null) {
                    //(x1)^2
                    float oneOfTheDimension = x1 * x1;
                    //+
                    aa.addAndGet(oneOfTheDimension);
                }
                if (x2 != null) {
                    //(x2)^2
                    float oneOfTheDimension = x2 * x2;
                    //+
                    bb.addAndGet(oneOfTheDimension);
                }
            });
        //|a|
        double aaa = Math.sqrt(aa.doubleValue());
        //|b|
        double bbb = Math.sqrt(bb.doubleValue());
        //使用BigDecimal保证精确计算浮点数
        //|a|*|b|
        //double aabb = aaa * bbb;
        BigDecimal aabb = BigDecimal.valueOf(aaa).multiply(BigDecimal.valueOf(bbb));
        //similarity=a.b/|a|*|b|
        //double cos = ab.get() / aabb.doubleValue();
        double cos = BigDecimal.valueOf(ab.get()).divide(aabb, 9, BigDecimal.ROUND_HALF_UP).doubleValue();
        return cos;
    }
1224118
@Override
    public double similarScore(List<Word> words1, List<Word> words2) {
        if(words1 == null || words2 == null){
            //只要有一个文本为null，规定相似度分值为0，表示完全不相等
            return 0.0;
        }
        if(words1.isEmpty() && words2.isEmpty()){
            //如果两个文本都为空，规定相似度分值为1，表示完全相等
            return 1.0;
        }
        if(words1.isEmpty() || words2.isEmpty()){
            //如果一个文本为空，另一个不为空，规定相似度分值为0，表示完全不相等
            return 0.0;
        }
        //输出词列表信息
        if(LOGGER.isDebugEnabled()) {
            LOGGER.debug("词列表1：");
            LOGGER.debug("\t" + words1);
            LOGGER.debug("词列表2：");
            LOGGER.debug("\t" + words2);
        }
        double score = scoreImpl(words1, words2);
        if(LOGGER.isDebugEnabled()){
            LOGGER.debug("分值："+score);
        }
        score = (int)(score*1000000+0.5)/(double)1000000;
        if(LOGGER.isDebugEnabled()){
            LOGGER.debug("取六位小数，四舍五入，分值："+score);
        }
        return score;
    }
1187624
public function wordSimEmbedding($word1, $word2, $options=array()){

        $data = array();
        
        $data['word_1'] = $word1;
        $data['word_2'] = $word2;

        $data = array_merge($data, $options);
        $data = mb_convert_encoding(json_encode($data), 'GBK', 'UTF8');

        return $this->request($this->wordSimEmbeddingUrl, $data);
    }
578716
public static <T> List<T> intersection(final List<? extends T> list1, final List<? extends T> list2) {
		List<? extends T> smaller = list1;
		List<? extends T> larger = list2;
		if (list1.size() > list2.size()) {
			smaller = list2;
			larger = list1;
		}

		// 克隆一个可修改的副本
		List<T> newSmaller = new ArrayList<T>(smaller);
		List<T> result = new ArrayList<T>(smaller.size());
		for (final T e : larger) {
			if (newSmaller.contains(e)) {
				result.add(e);
				newSmaller.remove(e);
			}
		}
		return result;
	}
909762
def hamming_distance(word1, word2):
    
    from operator import ne
    if len(word1) != len(word2):
        raise WrongLengthException('The words need to be of the same length!')

    return sum(map(ne, word1, word2))