计算相似度分值
@param words1 词列表1
@param words2 词列表2
@return 相似度分值
335063
@Override
    protected double scoreImpl(List<Word> words1, List<Word> words2) {
        //计算词列表1总的字符数
        AtomicInteger words1Length = new AtomicInteger();
        words1.parallelStream().forEach(word -> words1Length.addAndGet(word.getText().length()));
        //计算词列表2总的字符数
        AtomicInteger words2Length = new AtomicInteger();
        words2.parallelStream().forEach(word -> words2Length.addAndGet(word.getText().length()));
        //计算词列表1和词列表2共有的词的总的字符数
        words1.retainAll(words2);
        AtomicInteger intersectionLength = new AtomicInteger();
        words1.parallelStream().forEach(word -> {
            intersectionLength.addAndGet(word.getText().length());
        });
        double score = intersectionLength.get()/(double)Math.max(words1Length.get(), words2Length.get());
        if(LOGGER.isDebugEnabled()) {
            LOGGER.debug("词列表1总的字符数：" + words1Length.get());
            LOGGER.debug("词列表2总的字符数：" + words2Length.get());
            LOGGER.debug("词列表1和2共有的词的总的字符数：" + intersectionLength.get());
            LOGGER.debug("相似度分值=" + intersectionLength.get() + "/(double)Math.max(" + words1Length.get() + ", " + words1Length.get() + ")=" + score);
        }
        return score;
    }
536475
@Override
    public Collection<String> wordsNearest(INDArray words, int top) {
        return modelUtils.wordsNearest(words, top);
    }
1208873
public List<String> getWords() {
        List<String> result = new ArrayList<>();
        List<Word> words = WordParser.parse(question.replace("?", "").replace("？", ""));
        for (Word word : words) {
            result.add(word.getText());
        }
        return result;
    }
352322
public static Word combine(List<Word> words){
        if(words==null || words.size() < 2){
            return null;
        }
        String key = "";
        for(Word word : words){
            key += word.getText();
            key += " ";
        }
        key=key.trim();
        String value = GENERIC_TRIE.get(key);
        if(value==null){
            return null;
        }
        return new Word(value);
    }
578716
public static <T> List<T> intersection(final List<? extends T> list1, final List<? extends T> list2) {
		List<? extends T> smaller = list1;
		List<? extends T> larger = list2;
		if (list1.size() > list2.size()) {
			smaller = list2;
			larger = list1;
		}

		// 克隆一个可修改的副本
		List<T> newSmaller = new ArrayList<T>(smaller);
		List<T> result = new ArrayList<T>(smaller.size());
		for (final T e : larger) {
			if (newSmaller.contains(e)) {
				result.add(e);
				newSmaller.remove(e);
			}
		}
		return result;
	}
212670
def triples(self, words):
        

        if len(words) < 3:
            return

        for i in range(len(words) - 2):
            yield (words[i], words[i+1], words[i+2])
436153
public static Trie<String> toStringTrie(List<String> words) {
        final TrieBuilder<String> builder = new TrieBuilder<>();
        for (String word : words) {
            builder.add(word, "");
        }
        return builder.build();
    }
52569
@Override
    protected double scoreImpl(List<Word> words1, List<Word> words2) {
        //用词频来标注词的权重
        taggingWeightWithWordFrequency(words1, words2);
        //构造权重快速搜索容器
        Map<String, Float> weights1 = toFastSearchMap(words1);
        Map<String, Float> weights2 = toFastSearchMap(words2);
        //所有的不重复词
        Set<Word> words = new HashSet<>();
        words.addAll(words1);
        words.addAll(words2);
        //向量的维度为words的大小，每一个维度的权重是词频
        //manhattanDistance=|x1-x2|+|y1-y2|
        AtomicFloat manhattanDistance = new AtomicFloat();
        //计算
        words
            .parallelStream()
            .forEach(word -> {
                Float x1 = weights1.get(word.getText());
                Float x2 = weights2.get(word.getText());
                if (x1 == null) {
                    x1 = 0f;
                }
                if (x2 == null) {
                    x2 = 0f;
                }
                //|x1-x2|
                float oneOfTheDimension = Math.abs(x1 - x2);
                //+
                manhattanDistance.addAndGet(oneOfTheDimension);
            });
        double score = 0;
        if(manhattanDistance.get() == 0){
            //距离为0，表示完全相同
            score = 1.0;
        }else {
            //使用BigDecimal保证精确计算浮点数
            //score = 1 / (double)(manhattanDistance.get()+1);
            score = BigDecimal.valueOf(1).divide(BigDecimal.valueOf(manhattanDistance.get()+1), 9, BigDecimal.ROUND_HALF_UP).doubleValue();
        }
        if(LOGGER.isDebugEnabled()){
            LOGGER.debug("文本1和文本2的曼哈顿距离："+manhattanDistance.get());
            LOGGER.debug("文本1和文本2的相似度分值：1 / (double)("+manhattanDistance.get()+"+1)="+score);
        }
        return score;
    }
4808
protected Double computeAssociation(SemanticSpace sspace, 
                                        String word1, String word2) {
        Vector v1 = sspace.getVector(word1);
        Vector v2 = sspace.getVector(word2);
        if (v1 == null || v2 == null)
            return null;
        
        // Find the ranks of each of the two words to each other
        double rank1 = findRank(sspace, word1, word2);
        double rank2 = findRank(sspace, word2, word1);
        return 2d / (rank1 + rank2);
    }
389834
private List<Word>[] toWords(List<Node> leaf){
        List<Word>[] result = new ArrayList[leaf.size()];
        int i = 0;
        if(LOGGER.isDebugEnabled()) {
            LOGGER.debug("全切分结果：");
        }
        for(Node node : leaf){
            result[i++] = toWords(node);
            if(LOGGER.isDebugEnabled()) {
                LOGGER.debug("\t" + i + "：" + result[i - 1]);
            }
        }
        return result;
    }
1214221
public Word<I> findSeparatingWord(final S s1, final S s2) {

        final FastMealyState<O> n1 = this.nodeToObservationMap.get(s1);
        final FastMealyState<O> n2 = this.nodeToObservationMap.get(s2);

        return NearLinearEquivalenceTest.findSeparatingWord(this.observationTree, n1, n2, this.alphabet, true);
    }
1284844
@Override
    public Collection<String> wordsNearestSum(INDArray words, int top) {
        return modelUtils.wordsNearestSum(words, top);
    }
702394
private int calcCountSubstringsWords(String str, HashSet<String> words) {
    int wordCount = 0;
    int N = str.length();
    for (int i = 0; i < N - 1; i++)
      for (int j = i + 2; j < N + 1; j++) {
        if (words.contains(str.substring(i, j)))
          wordCount += 1;
      }
    return wordCount;
  }
772243
public JSONObject wordSimEmbedding(String word1, String word2, HashMap<String, Object> options) {
        AipRequest request = new AipRequest();
        preOperation(request);
        
        request.addBody("word_1", word1);
        
        request.addBody("word_2", word2);
        
        if (options != null) {
            request.addBody(options);
        }
        request.setUri(NlpConsts.WORD_SIM_EMBEDDING);
        request.addHeader(Headers.CONTENT_ENCODING, HttpCharacterEncoding.ENCODE_GBK);
        request.addHeader(Headers.CONTENT_TYPE, HttpContentType.JSON_DATA);
        request.setBodyFormat(EBodyFormat.RAW_JSON);
        postOperation(request);
        return requestServer(request);
    }
909762
def hamming_distance(word1, word2):
    
    from operator import ne
    if len(word1) != len(word2):
        raise WrongLengthException('The words need to be of the same length!')

    return sum(map(ne, word1, word2))
986759
public static List<Word> refine(List<Word> words){
        if(LOGGER.isDebugEnabled()) {
            LOGGER.debug("对分词结果进行refine之前：{}", words);
        }
        List<Word> result = new ArrayList<>(words.size());
        //一：拆词
        for(Word word : words){
            List<Word> splitWords = WordRefiner.split(word);
            if(splitWords==null){
                result.add(word);
            }else{
                if(LOGGER.isDebugEnabled()) {
                    LOGGER.debug("词： " + word.getText() + " 被拆分为：" + splitWords);
                }
                result.addAll(splitWords);
            }
        }
        if(LOGGER.isDebugEnabled()) {
            LOGGER.debug("对分词结果进行refine阶段的拆词之后：{}", result);
        }
        //二：组词
        if(result.size()<2){
            return result;
        }
        int combineMaxLength = WordConfTools.getInt("word.refine.combine.max.length", 3);
        if(combineMaxLength < 2){
            combineMaxLength = 2;
        }
        List<Word> finalResult = new ArrayList<>(result.size());
        for(int i=0; i<result.size(); i++){
            List<Word> toCombineWords = null;
            Word combinedWord = null;
            for(int j=2; j<=combineMaxLength; j++){
                int to = i+j;
                if(to > result.size()){
                    to = result.size();
                }
                toCombineWords = result.subList(i, to);
                combinedWord = WordRefiner.combine(toCombineWords);
                if(combinedWord != null){
                    i += j;
                    i--;
                    break;
                }
            }
            if(combinedWord == null){
                finalResult.add(result.get(i));
            }else{
                if(LOGGER.isDebugEnabled()) {
                    LOGGER.debug("词： " + toCombineWords + " 被合并为：" + combinedWord);
                }
                finalResult.add(combinedWord);
            }
        }
        if(LOGGER.isDebugEnabled()) {
            LOGGER.debug("对分词结果进行refine阶段的组词之后：{}", finalResult);
        }
        return finalResult;
    }
1158999
public List<Blob> getBlobFromStrings(String blobName1, String blobName2) {
    // [START getBlobFromStrings]
    List<Blob> blobs = bucket.get(blobName1, blobName2);
    for (Blob blob : blobs) {
      if (blob == null) {
        // the blob was not found
      }
    }
    // [END getBlobFromStrings]
    return blobs;
  }