Create a similarity matrix, using diff as the similarity measure, based on the difference of WORDS (not characters)
 (only counts insertions and deletions, not substitution and transposition).
326628
def calculate_with_diff
      progress_bar = nil
      if @config[:show_progress]
        progress_bar = ProgressBar.create format: '%a |%B| %p%% %e', length: 80, smoothing: 0.5,
                                         total: @sources.length*(@sources.length-1)/2
      end

      matrix = {}
      @sources.each_with_index do |a,i|
        a_id = id_of(a)
        a_text = text_of(a)

        @sources[i + 1..-1].each do |b|
          b_id   = id_of(b)
          b_text = text_of(b)

          stats = FastHtmlDiff::DiffBuilder.new(a_text, b_text).statistics

          # http://en.wikipedia.org/wiki/Dice%27s_coefficient
          total_count = 2 * stats[:matches][:words] + stats[:insertions][:words] + stats[:deletions][:words]
          similarity = 2 * stats[:matches][:words] / total_count.to_f

          # Build the similarity matrix,
          matrix[a_id] ||= {a_id => 1}
          matrix[a_id][b_id] = similarity
          matrix[b_id] ||= {b_id => 1}
          matrix[b_id][a_id] = similarity

          progress_bar.increment unless progress_bar.nil?
        end
      end
      matrix
    end
753839
private Diff generateDiff(final char[] revA, final char[] revB)
		throws UnsupportedEncodingException
	{

		blockCount = 0;
		queueA = new ArrayList<DiffBlock>();
		queueB = new ArrayList<DiffBlock>();

		revABlocked = new boolean[revA.length];
		revBBlocked = new boolean[revB.length];

		int revAStartIndex = 0, revAEndIndex = revA.length - 1;
		int revBStartIndex = 0, revBEndIndex = revB.length - 1;

		while (revAStartIndex <= revAEndIndex && revBStartIndex <= revBEndIndex
				&& revA[revAStartIndex] == revB[revBStartIndex]) {

			revABlocked[revAStartIndex] = true;
			revBBlocked[revBStartIndex] = true;
			revAStartIndex++;
			revBStartIndex++;
		}

		// First Block
		if (revAStartIndex != 0) {
			queueA.add(new DiffBlock(this.blockCount, 0, revAStartIndex, 0,
					revBStartIndex, true));
			queueB.add(new DiffBlock(this.blockCount, 0, revAStartIndex, 0,
					revBStartIndex, false));
			this.blockCount++;
		}

		while (revAStartIndex < revAEndIndex && revBStartIndex < revBEndIndex
				&& revA[revAEndIndex] == revB[revBEndIndex]) {

			revABlocked[revAEndIndex] = true;
			revBBlocked[revBEndIndex] = true;
			revAEndIndex--;
			revBEndIndex--;
		}

		// Last Block
		if (revAEndIndex + 1 != revA.length) {
			queueA.add(new DiffBlock(this.blockCount, revAEndIndex + 1,
					revA.length, revBEndIndex + 1, revB.length, true));
			queueB.add(new DiffBlock(this.blockCount, revAEndIndex + 1,
					revA.length, revBEndIndex + 1, revB.length, false));
			this.blockCount++;
		}

		scan(revA, revAStartIndex, revAEndIndex);

		ArrayList<Integer> list;
		char c;

		int i = revBStartIndex;
		while (i < revBEndIndex) {

			c = revB[i];
			list = positions.get(c);

			if (list != null && findLongestMatch(revA, list, revB, i)) {

				i += longestMatch_size;
			}
			else {
				i++;
			}
		}

		int j;
		for (i = revAStartIndex; i <= revAEndIndex; i++) {
			if (!revABlocked[i]) {
				j = i;
				while (i + 1 <= revAEndIndex && !revABlocked[++i]) {
					;
				}

				if (i + 1 > revAEndIndex) {
					i++;
				}

				queueA.add(new DiffBlock(-1, j, i, -1, -1, true));
			}
		}

		for (i = revBStartIndex; i <= revBEndIndex; i++) {
			if (!revBBlocked[i]) {
				j = i;
				while (i + 1 <= revBEndIndex && !revBBlocked[++i]) {
					;
				}

				if (i + 1 > revBEndIndex) {
					i++;
				}

				queueB.add(new DiffBlock(-1, -1, -1, j, i, false));
			}
		}

		Collections.sort(queueA);
		Collections.sort(queueB);

		return blocks.manage(revA, revB, queueA, queueB);
	}
113816
def generate_diff_subparser(subparsers):
    """"""
    parser = subparsers.add_parser(
        'diff', description=constants.DIFF_DESCRIPTION,
        epilog=constants.DIFF_EPILOG, formatter_class=ParagraphFormatter,
        help=constants.DIFF_HELP)
    parser.set_defaults(func=ngram_diff)
    group = parser.add_mutually_exclusive_group()
    group.add_argument('-a', '--asymmetric', help=constants.ASYMMETRIC_HELP,
                       metavar='LABEL')
    utils.add_common_arguments(parser)
    utils.add_db_arguments(parser)
    utils.add_corpus_arguments(parser)
    utils.add_query_arguments(parser)
174197
def diff(a, b):
        
        delta = diff(a, b)
        if not a:
            return delta
        for item in delta:
            if isinstance(a[item], list):
                if delta[item] == []:
                    delta[item] = {JSONUtils.DIFF_DELETE : a[item]}
                elif isinstance(delta[item], dict):
                    for key in delta[item].keys():
                        if str(key) == JSONUtils.DIFF_DELETE:
                            delta_upd = {JSONUtils.DIFF_DELETE : []}
                            for i in delta[item][key]:
                                delta_upd[JSONUtils.DIFF_DELETE].append(a[item][i])
                                delta[item] = delta_upd
        return delta
17943
def dicom_diff(file1, file2):
    

    datasets = compressed_dicom.read_file(file1), compressed_dicom.read_file(file2)

    rep = []

    for dataset in datasets:
        lines = (str(dataset.file_meta)+"\n"+str(dataset)).split('\n')
        lines = [line + '\n' for line in lines]  # add the newline to the end
        rep.append(lines)

    diff = difflib.Differ()
    for line in diff.compare(rep[0], rep[1]):
        if (line[0] == '+') or (line[0] == '-'):
            sys.stdout.write(line)
1060234
def diffstat(self):
    
    names = []
    insert = []
    delete = []
    delta = 0    # size change in bytes
    namelen = 0
    maxdiff = 0  # max number of changes for single file
                 # (for histogram width calculation)
    for patch in self.items:
      i,d = 0,0
      for hunk in patch.hunks:
        for line in hunk.text:
          if line.startswith(b'+'):
            i += 1
            delta += len(line)-1
          elif line.startswith(b'-'):
            d += 1
            delta -= len(line)-1
      names.append(patch.target)
      insert.append(i)
      delete.append(d)
      namelen = max(namelen, len(patch.target))
      maxdiff = max(maxdiff, i+d)
    output = ''
    statlen = len(str(maxdiff))  # stats column width
    for i,n in enumerate(names):
      # %-19s | %-4d %s
      format = " %-" + str(namelen) + "s | %" + str(statlen) + "s %s\n"

      hist = ''
      # -- calculating histogram --
      width = len(format % ('', '', ''))
      histwidth = max(2, 80 - width)
      if maxdiff < histwidth:
        hist = "+"*insert[i] + "-"*delete[i]
      else:
        iratio = (float(insert[i]) / maxdiff) * histwidth
        dratio = (float(delete[i]) / maxdiff) * histwidth

        # make sure every entry gets at least one + or -
        iwidth = 1 if 0 < iratio < 1 else int(iratio)
        dwidth = 1 if 0 < dratio < 1 else int(dratio)
        #print(iratio, dratio, iwidth, dwidth, histwidth)
        hist = "+"*int(iwidth) + "-"*int(dwidth)
      # -- /calculating +- histogram --
      output += (format % (tostr(names[i]), str(insert[i] + delete[i]), hist))

    output += (" %d files changed, %d insertions(+), %d deletions(-), %+d bytes"
               % (len(names), sum(insert), sum(delete), delta))
    return output
42389
def cosine_similarity(sent1: str, sent2: str) -> float:
    

    WORD = re.compile(r'\w+')
    def get_cosine(vec1, vec2):
        intersection = set(vec1.keys()) & set(vec2.keys())
        numerator = sum([vec1[x] * vec2[x] for x in intersection])

        sum1 = sum([vec1[x]**2 for x in vec1.keys()])
        sum2 = sum([vec2[x]**2 for x in vec2.keys()])
        denominator = math.sqrt(sum1) * math.sqrt(sum2)

        if not denominator:
            return 0.0
        else:
            return float(numerator) / denominator

    def text_to_vector(text):
        words = WORD.findall(text)
        return Counter(words)

    vector1 = text_to_vector(sent1)
    vector2 = text_to_vector(sent2)
    cosine = get_cosine(vector1, vector2)

    return cosine
865851
protected void diff_charsToLines(LinkedList<Diff> diffs,
            List<String> lineArray) {
        StringBuilder text;
        for (Diff diff : diffs) {
            text = new StringBuilder();
            for (int y = 0; y < diff.text.length(); y++) {
                text.append(lineArray.get(diff.text.charAt(y)));
            }
            diff.text = text.toString();
        }
    }
512699
def similarity_by_infocontent(sense1: "wn.Synset", sense2: "wn.Synset", option: str) -> float:
    

    if sense1.pos != sense2.pos: # infocontent sim can't do diff POS.
        return 0

    if option in ['res', 'resnik']:
        if sense1.pos not in wnic_bnc_resnik_add1.ic:
            return 0
        return wn.res_similarity(sense1, sense2, wnic_bnc_resnik_add1)
    #return min(wn.res_similarity(sense1, sense2, wnic.ic(ic)) \
    #             for ic in info_contents)

    elif option in ['jcn', "jiang-conrath"]:
        if sense1.pos not in wnic_bnc_add1.ic:
            return 0
        return wn.jcn_similarity(sense1, sense2, wnic_bnc_add1)

    elif option in ['lin']:
        if sense1.pos not in wnic_bnc_add1.ic:
            return 0
        return wn.lin_similarity(sense1, sense2, wnic_bnc_add1)
79907
def _diff(d1,d2):
    '''
        
    '''
    d = {}
    ds = _diff_internal(d1,d2)
    for key in ds['vdiff']:
        d[key] = d1[key]
    for key in ds['kdiff']:
        d[key] = d1[key]
    return(d)
512525
def diff(self, other, **kwargs):
        
        from normalize.diff import diff
        return diff(self, other, **kwargs)
174901
def compare(s1, s2, seg=True, ignore=False, stopwords=False):
    '''
    
    '''
    if s1 == s2: return 1.0
    
    s1_words = []
    s2_words = []

    if seg:
        s1 = [x for x in jieba.cut(s1, cut_all=False, HMM=False)]
        s2 = [x for x in jieba.cut(s2, cut_all=False, HMM=False)]
    else:
        s1 = s1.split()
        s2 = s2.split()

    # check stopwords
    if not stopwords:
        global _stopwords
        for x in s1: 
            if not x in _stopwords:
                s1_words.append(x)
        for x in s2:
            if not x in _stopwords:
                s2_words.append(x)
    else:
        s1_words = s1 
        s2_words = s2

    assert len(s1) > 0 and len(s2) > 0, "The length of s1 and s2 should > 0."
    return _similarity_distance(s1_words, s2_words, ignore)
1014000
def record_diff(lhs, rhs):
    ""
    delta = {}
    for k in set(lhs).union(rhs):
        from_ = lhs[k]
        to_ = rhs[k]
        if from_ != to_:
            delta[k] = {'from': from_, 'to': to_}

    return delta
101547
def structural_similarity(document_1, document_2):
    
    try:
        document_1 = lxml.html.parse(StringIO(document_1))
        document_2 = lxml.html.parse(StringIO(document_2))
    except Exception as e:
        print(e)
        return 0

    tags1 = get_tags(document_1)
    tags2 = get_tags(document_2)
    diff = difflib.SequenceMatcher()
    diff.set_seq1(tags1)
    diff.set_seq2(tags2)

    return diff.ratio()
1332109
void cleanupMerge(LinkedList<Change> diffs) {
        diffs.add(new Change(Operation.EQUAL, ""));  // Add a dummy entry at the end.
        ListIterator<Change> pointer = diffs.listIterator();
        int count_delete = 0;
        int count_insert = 0;
        String text_delete = "";
        String text_insert = "";
        Change thisDiff = pointer.next();
        Change prevEqual = null;
        int commonlength;
        while (thisDiff != null) {
            switch (thisDiff.operation) {
                case INSERT:
                    count_insert++;
                    text_insert += thisDiff.text;
                    prevEqual = null;
                    break;
                case DELETE:
                    count_delete++;
                    text_delete += thisDiff.text;
                    prevEqual = null;
                    break;
                case EQUAL:
                    if (count_delete + count_insert > 1) {
                        boolean both_types = count_delete != 0 && count_insert != 0;
                        // Delete the offending records.
                        pointer.previous();  // Reverse direction.
                        while (count_delete-- > 0) {
                            pointer.previous();
                            pointer.remove();
                        }
                        while (count_insert-- > 0) {
                            pointer.previous();
                            pointer.remove();
                        }
                        if (both_types) {
                            // Factor out any common prefixies.
                            commonlength = commonPrefix(text_insert, text_delete);
                            if (commonlength != 0) {
                                if (pointer.hasPrevious()) {
                                    thisDiff = pointer.previous();
                                    assert thisDiff.operation == Operation.EQUAL
                                            : "Previous diff should have been an equality.";
                                    thisDiff.text += text_insert.substring(0, commonlength);
                                    pointer.next();
                                } else {
                                    pointer.add(new Change(Operation.EQUAL,
                                                           text_insert.substring(0, commonlength)));
                                }
                                text_insert = text_insert.substring(commonlength);
                                text_delete = text_delete.substring(commonlength);
                            }
                            // Factor out any common suffixies.
                            commonlength = commonSuffix(text_insert, text_delete);
                            if (commonlength != 0) {
                                thisDiff = pointer.next();
                                thisDiff.text = text_insert.substring(text_insert.length()
                                                                      - commonlength) + thisDiff.text;
                                text_insert = text_insert.substring(0, text_insert.length()
                                                                       - commonlength);
                                text_delete = text_delete.substring(0, text_delete.length()
                                                                       - commonlength);
                                pointer.previous();
                            }
                        }
                        // Insert the merged records.
                        if (text_delete.length() != 0) {
                            pointer.add(new Change(Operation.DELETE, text_delete));
                        }
                        if (text_insert.length() != 0) {
                            pointer.add(new Change(Operation.INSERT, text_insert));
                        }
                        // Step forward to the equality.
                        thisDiff = pointer.hasNext() ? pointer.next() : null;
                    } else if (prevEqual != null) {
                        // Merge this equality with the previous one.
                        prevEqual.text += thisDiff.text;
                        pointer.remove();
                        thisDiff = pointer.previous();
                        pointer.next();  // Forward direction
                    }
                    count_insert = 0;
                    count_delete = 0;
                    text_delete = "";
                    text_insert = "";
                    prevEqual = thisDiff;
                    break;
            }
            thisDiff = pointer.hasNext() ? pointer.next() : null;
        }
        if (diffs.getLast().text.length() == 0) {
            diffs.removeLast();  // Remove the dummy entry at the end.
        }

    /*
     * Second pass: look for single edits surrounded on both sides by equalities
     * which can be shifted sideways to eliminate an equality.
     * e.g: A<ins>BA</ins>C -> <ins>AB</ins>AC
     */
        boolean changes = false;
        // Create a new iterator at the start.
        // (As opposed to walking the current one back.)
        pointer = diffs.listIterator();
        Change prevDiff = pointer.hasNext() ? pointer.next() : null;
        thisDiff = pointer.hasNext() ? pointer.next() : null;
        Change nextDiff = pointer.hasNext() ? pointer.next() : null;
        // Intentionally ignore the first and last element (don't need checking).
        while (nextDiff != null) {
            if (prevDiff.operation == Operation.EQUAL &&
                nextDiff.operation == Operation.EQUAL) {
                // This is a single edit surrounded by equalities.
                if (thisDiff.text.endsWith(prevDiff.text)) {
                    // Shift the edit over the previous equality.
                    thisDiff.text = prevDiff.text
                                    + thisDiff.text.substring(0, thisDiff.text.length()
                                                                 - prevDiff.text.length());
                    nextDiff.text = prevDiff.text + nextDiff.text;
                    pointer.previous(); // Walk past nextDiff.
                    pointer.previous(); // Walk past thisDiff.
                    pointer.previous(); // Walk past prevDiff.
                    pointer.remove(); // Delete prevDiff.
                    pointer.next(); // Walk past thisDiff.
                    thisDiff = pointer.next(); // Walk past nextDiff.
                    nextDiff = pointer.hasNext() ? pointer.next() : null;
                    changes = true;
                } else if (thisDiff.text.startsWith(nextDiff.text)) {
                    // Shift the edit over the next equality.
                    prevDiff.text += nextDiff.text;
                    thisDiff.text = thisDiff.text.substring(nextDiff.text.length())
                                    + nextDiff.text;
                    pointer.remove(); // Delete nextDiff.
                    nextDiff = pointer.hasNext() ? pointer.next() : null;
                    changes = true;
                }
            }
            prevDiff = thisDiff;
            thisDiff = nextDiff;
            nextDiff = pointer.hasNext() ? pointer.next() : null;
        }
        // If shifts were made, the diff needs reordering and another shift sweep.
        if (changes) {
            cleanupMerge(diffs);
        }
    }
793348
def equal(self, line1, line2):
        '''
        
        '''

        eqLine = line1 == line2
        if eqLine:
            return BooleanPlus(True, False)
        else:
            unchanged_count = self.count_similar_words(line1, line2)
            similarity_percent = (
                (2.0 * unchanged_count) /
                (len(line1.split()) + len(line2.split()))
            )
            if similarity_percent >= 0.50:
                return BooleanPlus(True, True)
            return BooleanPlus(False, False)
1193315
def _rows_differ(row, _row):
    '''
    
    '''
    row_copy = copy.deepcopy(row)
    _row_copy = copy.deepcopy(_row)
    # Strip id from all panels in both rows, since they are always generated.
    for panel in row_copy['panels']:
        if 'id' in panel:
            del panel['id']
    for _panel in _row_copy['panels']:
        if 'id' in _panel:
            del _panel['id']
    diff = DictDiffer(row_copy, _row_copy)
    return diff.changed() or diff.added() or diff.removed()