Simple measure of similarity: Number of letters in common / max length
434120
def basic_word_sim(word1, word2):
    
    return sum([1 for c in word1 if c in word2]) / max(len(word1), len(word2))
47682
def similarity(string_a, string_b)
      string_a, string_b = prep_strings(string_a, string_b)
      return 100.0 if string_a == string_b
      score = 0
      total_weight = algorithms.values.inject { |sum, weight| sum + weight }
      algorithms.each do |algorithm, weight|
        next unless weight.positive?
        score+= string_a.send("#{algorithm}_similarity", string_b) * weight
      end
      score / total_weight
    end
454051
def similarity(self, other):
        """"""
        numerator, denominator = sorted((self.value, other.value))
        try:
            ratio = float(numerator) / denominator
        except ZeroDivisionError:
            ratio = 0.0 if numerator else 1.0
        similarity = self.Similarity(ratio)
        return similarity
743184
def compare(self, statement_a, statement_b):
        
        # Make both strings lowercase
        document_a = self.nlp(statement_a.text.lower())
        document_b = self.nlp(statement_b.text.lower())

        statement_a_lemmas = set([
            token.lemma_ for token in document_a if not token.is_stop
        ])
        statement_b_lemmas = set([
            token.lemma_ for token in document_b if not token.is_stop
        ])

        # Calculate Jaccard similarity
        numerator = len(statement_a_lemmas.intersection(statement_b_lemmas))
        denominator = float(len(statement_a_lemmas.union(statement_b_lemmas)))
        ratio = numerator / denominator

        return ratio
1190433
def segment(self, document):
        
        assert(len(document) > 0 and len([d for d in document if not isinstance(d, str)]) == 0)
        if len(document) < 3:
            return [1] + [0 for _ in range(len(document) - 1)]
        # step 1, preprocessing
        n = len(document)
        self.window = min(self.window, n)
        cnts = [Counter(self.tokenizer.tokenize(document[i])) for i in range(n)]

        # step 2, compute similarity matrix
        self.sim = np.zeros((n, n))
        for i in range(n):
            for j in range(i, n):
                self.sim[i][j] = cosine_sim(cnts[i], cnts[j])
                self.sim[j][i] = self.sim[i][j]

        # step 3, compute rank matrix & sum matrix
        self.rank = np.zeros((n, n))
        for i in range(n):
            for j in range(i, n):
                r1 = max(0, i - self.window + 1)
                r2 = min(n - 1, i + self.window - 1)
                c1 = max(0, j - self.window + 1)
                c2 = min(n - 1, j + self.window - 1)
                sublist = self.sim[r1:(r2 + 1), c1:(c2+1)].flatten()
                lowlist = [x for x in sublist if x < self.sim[i][j]]
                self.rank[i][j] = 1.0 * len(lowlist) / ((r2 - r1 + 1) * (c2 - c1 + 1))
                self.rank[j][i] = self.rank[i][j]

        self.sm = np.zeros((n, n))
        # O(n^4) solution
        # for i in xrange(n):
        #     for j in xrange(i, n):
        #         self.sm[i][j] = sum(self.rank[i:(j + 1), i:(j + 1)].flatten())
        #         self.sm[j][i] = self.sm[i][j]
        # O(n^2) solution
        prefix_sm = np.zeros((n, n))
        for i in range(n):
            for j in range(n):
                prefix_sm[i][j] = self.rank[i][j]
                if i - 1 >= 0: prefix_sm[i][j] += prefix_sm[i - 1][j]
                if j - 1 >= 0: prefix_sm[i][j] += prefix_sm[i][j - 1]
                if i - 1 >= 0 and j - 1 >= 0: prefix_sm[i][j] -= prefix_sm[i - 1][j - 1]
        for i in range(n):
            for j in range(i, n):
                if i == 0:
                    self.sm[i][j] = prefix_sm[j][j]
                else:
                    self.sm[i][j] = prefix_sm[j][j] - prefix_sm[i - 1][j] \
                                    - prefix_sm[j][i - 1] + prefix_sm[i - 1][i - 1]
                self.sm[j][i] = self.sm[i][j]

        # step 4, determine boundaries
        D = 1.0 * self.sm[0][n - 1] / (n * n)
        darr, region_arr, idx = [D], [Region(0, n - 1, self.sm)], []
        sum_region, sum_area = float(self.sm[0][n - 1]), float(n * n)
        for i in range(n - 1):
            mx, pos = -1e9, -1
            for j, region in enumerate(region_arr):
                if region.l == region.r:
                    continue
                region.split(self.sm)
                den = sum_area - region.area + region.lch.area + region.rch.area
                cur = (sum_region - region.tot + region.lch.tot + region.rch.tot) / den
                if cur > mx:
                    mx, pos = cur, j
            assert(pos >= 0)
            tmp = region_arr[pos]
            region_arr[pos] = tmp.rch
            region_arr.insert(pos, tmp.lch)
            sum_region += tmp.lch.tot + tmp.rch.tot - tmp.tot
            sum_area += tmp.lch.area + tmp.rch.area - tmp.area
            darr.append(sum_region / sum_area)
            idx.append(tmp.best_pos)

        dgrad = [(darr[i + 1] - darr[i]) for i in range(len(darr) - 1)]

        # optional step, smooth gradient
        smooth_dgrad = [dgrad[i] for i in range(len(dgrad))]
        if len(dgrad) > 1:
            smooth_dgrad[0] = (dgrad[0] * 2 + dgrad[1]) / 3.0
            smooth_dgrad[-1] = (dgrad[-1] * 2 + dgrad[-2]) / 3.0
        for i in range(1, len(dgrad) - 1):
            smooth_dgrad[i] = (dgrad[i - 1] + 2 * dgrad[i] + dgrad[i + 1]) / 4.0
        dgrad = smooth_dgrad

        avg, stdev = np.average(dgrad), np.std(dgrad)
        cutoff = avg + self.std_coeff * stdev
        assert(len(idx) == len(dgrad))
        above_cutoff_idx = [i for i in range(len(dgrad)) if dgrad[i] >= cutoff]
        if len(above_cutoff_idx) == 0: boundary = []
        else: boundary = idx[:max(above_cutoff_idx) + 1]
        ret = [0 for _ in range(n)]
        for i in boundary:
            ret[i] = 1
            # boundary should not be too close
            for j in range(i - 1, i + 2):
                if j >= 0 and j < n and j != i and ret[j] == 1:
                    ret[i] = 0
                    break
        return [1] + ret[:-1]
47609
def similar_items(self, itemid, N=10):
        
        if itemid >= self.similarity.shape[0]:
            return []

        return sorted(list(nonzeros(self.similarity, itemid)), key=lambda x: -x[1])[:N]
1043232
def similarity_by_path(sense1: "wn.Synset", sense2: "wn.Synset", option: str = "path") -> float:
    
    if option.lower() in ["path", "path_similarity"]: # Path similarities.
        return max(wn.path_similarity(sense1, sense2, if_none_return=0),
                   wn.path_similarity(sense2, sense1, if_none_return=0))
    elif option.lower() in ["wup", "wupa", "wu-palmer", "wu-palmer"]: # Wu-Palmer
        return max(wn.wup_similarity(sense1, sense2, if_none_return=0),
                   wn.wup_similarity(sense2, sense1, if_none_return=0))
    elif option.lower() in ['lch', "leacock-chordorow"]: # Leacock-Chodorow
        if sense1.pos != sense2.pos: # lch can't do diff POS
            return 0
        return wn.lch_similarity(sense1, sense2, if_none_return=0)
1114744
def random_letters(self, length=16):
        """"""
        return self.random_choices(
            getattr(string, 'letters', string.ascii_letters),
            length=length,
        )
699732
def find_common_beginning(string_list, boundary_char = None):
    
    
    common=''
    
    # by definition there is nothing common to 1 item...
    if len(string_list) > 1:
        shortestLen = min([len(el) for el in string_list])
        
        for idx in range(shortestLen):
            chars = [s[idx] for s in string_list]
            if chars.count(chars[0]) != len(chars): # test if any chars differ
                break
            common+=chars[0]
    
        
    if boundary_char is not None:
        try:
            end_idx = common.rindex(boundary_char)
            common = common[0:end_idx+1]
        except ValueError:
            common = ''
    
    return common
864318
def text_list_to_colors(names):
    '''
    
    '''
    # STEP A: compute strings distance between all combnations of strings
    Dnames = np.zeros( (len(names), len(names)) )
    for i in range(len(names)):
        for j in range(len(names)):
            Dnames[i,j] = 1 - 2.0 * levenshtein(names[i], names[j]) / float(len(names[i]+names[j]))

    # STEP B: pca dimanesionality reduction to a single-dimension (from the distance space)
    pca = sklearn.decomposition.PCA(n_components = 1)
    pca.fit(Dnames)    
    
    # STEP C: mapping of 1-dimensional values to colors in a jet-colormap
    textToColor = pca.transform(Dnames)
    textToColor = 255 * (textToColor - textToColor.min()) / (textToColor.max() - textToColor.min())
    textmaps = generateColorMap();
    colors = [textmaps[int(c)] for c in textToColor]
    return colors
1233566
private int findRank(SemanticSpace sspace, 
                         String target, String other) {
        Vector v1 = sspace.getVector(target);
        Vector v2 = sspace.getVector(other);
        // Compute the base similarity between the two words
        double baseSim = Similarity.cosineSimilarity(v1, v2);
        int rank = 0;
        // Next, count how many words are more similar to the target than the
        // other word is
        for (String word : sspace.getWords()) {
            Vector v = sspace.getVector(word);
            double sim = Similarity.cosineSimilarity(v1, v);
            if (sim > baseSim)
                rank++;
        }
        return rank;
    }
114306
public static int matchingNibbleLength(byte[] a, byte[] b) {
        int i = 0;
        int length = a.length < b.length ? a.length : b.length;
        while (i < length) {
            if (a[i] != b[i])
                return i;
            i++;
        }
        return i;
    }
656956
def evaluate_similarity(args, token_embedding, ctx, logfile=None,
                        global_step=0):
    """"""

    results = []
    for similarity_function in args.similarity_functions:
        evaluator = nlp.embedding.evaluation.WordEmbeddingSimilarity(
            idx_to_vec=token_embedding.idx_to_vec,
            similarity_function=similarity_function)
        evaluator.initialize(ctx=ctx)
        if not args.no_hybridize:
            evaluator.hybridize()

        # Evaluate all datasets
        for (dataset_name, dataset_kwargs,
             dataset) in iterate_similarity_datasets(args):
            initial_length = len(dataset)
            dataset_coded = [[
                token_embedding.token_to_idx[d[0]],
                token_embedding.token_to_idx[d[1]], d[2]
            ] for d in dataset if d[0] in token_embedding.token_to_idx
                             and d[1] in token_embedding.token_to_idx]
            num_dropped = initial_length - len(dataset_coded)

            # All words are unknown
            if not len(dataset_coded):
                correlation = 0
            else:
                words1, words2, scores = zip(*dataset_coded)
                pred_similarity = evaluator(
                    mx.nd.array(words1, ctx=ctx), mx.nd.array(words2, ctx=ctx))
                sr = stats.spearmanr(pred_similarity.asnumpy(),
                                     np.array(scores))
                correlation = sr.correlation

            logging.info(
                'Spearman rank correlation on %s (%s pairs) %s with %s:\t%s',
                dataset.__class__.__name__, len(dataset_coded),
                str(dataset_kwargs), similarity_function, correlation)

            result = dict(
                task='similarity',
                dataset_name=dataset_name,
                dataset_kwargs=dataset_kwargs,
                similarity_function=similarity_function,
                spearmanr=correlation,
                num_dropped=num_dropped,
                global_step=global_step,
            )
            log_similarity_result(logfile, result)
            results.append(result)

    return results
321294
def similarity(self, other):
        """"""
        ratio = SequenceMatcher(a=self.value, b=other.value).ratio()
        similarity = self.Similarity(ratio)
        return similarity
62694
def similarity(state_a, state_b):
        
        count_a = Counter(state_a.history.bbl_addrs)
        count_b = Counter(state_b.history.bbl_addrs)
        normal_distance = sum((count_a.get(addr, 0) - count_b.get(addr, 0)) ** 2
                              for addr in set(list(count_a.keys()) + list(count_b.keys()))) ** 0.5
        return 1.0 / (1 + normal_distance)
713474
def diff_commonOverlap(self, text1, text2):
    
    # Cache the text lengths to prevent multiple calls.
    text1_length = len(text1)
    text2_length = len(text2)
    # Eliminate the null case.
    if text1_length == 0 or text2_length == 0:
      return 0
    # Truncate the longer string.
    if text1_length > text2_length:
      text1 = text1[-text2_length:]
    elif text1_length < text2_length:
      text2 = text2[:text1_length]
    text_length = min(text1_length, text2_length)
    # Quick check for the worst case.
    if text1 == text2:
      return text_length

    # Start by looking for a single character match
    # and increase length until no match is found.
    # Performance analysis: https://neil.fraser.name/news/2010/11/04/
    best = 0
    length = 1
    while True:
      pattern = text1[-length:]
      found = text2.find(pattern)
      if found == -1:
        return best
      length += found
      if found == 0 or text1[-length:] == text2[:length]:
        best = length
        length += 1
246182
def _infer_spaces(s):
    
    s = s.lower()

    # Find the best match for the i first characters, assuming cost has
    # been built for the i-1 first characters.
    # Returns a pair (match_cost, match_length).
    def best_match(i):
        candidates = enumerate(reversed(cost[max(0, i - MAXWORD):i]))
        return min((c + WORDCOST.get(s[i-k-1: i], 9e999), k + 1)
                    for k, c in candidates)

    # Build the cost array.
    cost = [0]
    for i in range(1, len(s) + 1):
        c, k = best_match(i)
        cost.append(c)

    # Backtrack to recover the minimal-cost string.
    out = []
    i = len(s)
    while i > 0:
        c, k = best_match(i)
        assert c == cost[i]
        out.append(s[i-k:i])
        i -= k

    return u" ".join(reversed(out))