synonyms and existing_ids are part of an object bug that can create duplicates if in the same batch
962283
def merge(new, old):
    '''  '''

    for k, vals in new.items():

        if k == 'synonyms':
            new_synonyms = vals
            if old['synonyms']:
                old_literals = [syn['literal'].lower().strip() for syn in old['synonyms']]
                for new_synonym in new_synonyms:
                    if new_synonym['literal'].lower().strip() not in old_literals:
                        old['synonyms'].append(new_synonym) # default is a list in SciCrunch, that's why this works without initing old['synonyms']
            else:
                old['synonyms'].extend(new['synonyms'])

        elif k == 'existing_ids':
            iris = [e['iri'] for e in old['existing_ids']]
            for new_existing_id in vals:

                new_existing_id['preferred'] = 0

                if 'change' not in list(new_existing_id):  # notion that you want to add it
                    new_existing_id['change'] = False

                if new_existing_id.get('delete') == True:
                    if new_existing_id['iri'] in iris:
                        new_existing_ids = []
                        for e in old['existing_ids']:
                            if e['iri'] != new_existing_id['iri']:
                                new_existing_ids.append(e)
                        old['existing_ids'] = new_existing_ids
                    else:
                        print(new_existing_id)
                        sys.exit("You want to delete an iri that doesn't exist")

                elif new_existing_id.get('replace') == True:
                    if not new_existing_id.get('old_iri'):
                        sys.exit(
                            'Need to have old_iri as a key to have a ref for replace'
                        )
                    old_iri = new_existing_id.pop('old_iri')
                    if old_iri in iris:
                        new_existing_ids = []
                        for e in old['existing_ids']:
                            if e['iri'] == old_iri:
                                if new_existing_id.get('curie'):
                                    e['curie'] = new_existing_id['curie']
                                if new_existing_id.get('iri'):
                                    e['iri'] = new_existing_id['iri']
                            new_existing_ids.append(e)
                        old['existing_ids'] = new_existing_ids
                    else:
                        print(new_existing_id)
                        sys.exit("You want to replace an iri that doesn't exist", '\n', new)

                else:
                    if new_existing_id['iri'] not in iris and new_existing_id['change'] == True:
                        sys.exit('You want to change iri that doesnt exist ' + str(new))
                    elif new_existing_id['iri'] not in iris and new_existing_id['change'] == False:
                        old['existing_ids'].append(new_existing_id)
                    elif new_existing_id['iri'] in iris and new_existing_id['change'] == True:
                        new_existing_ids = []
                        for e in old['existing_ids']:
                            if e['iri'] == new_existing_id['iri']:
                                if not new_existing_id.get('curie'):
                                    new_existing_id['curie'] = e['curie']
                                new_existing_ids.append(new_existing_id)
                            else:
                                new_existing_ids.append(e)
                        old['existing_ids'] = new_existing_ids
                    elif new_existing_id['iri'] in iris and new_existing_id['change'] == False:
                        pass  # for sanity readability
                    else:
                        sys.exit('Something broke while merging in existing_ids')

        elif k in ['definition', 'superclasses', 'id', 'type', 'comment', 'label', 'uid', 'ontologies']:
            old[k] = vals

        # TODO: still need to mark them... but when batch elastic for update works
        # old['uid'] = 34142  # DEBUG: need to mark as mine manually until all Old terms are fixed

    ''' REMOVE REPEATS; needs to exist due to server overloads '''
    if old.get('synonyms'):
        visited = {}
        new_synonyms = []
        for synonym in old['synonyms']:
            if not visited.get(synonym.get('literal')):
                new_synonyms.append(synonym)
                visited[synonym['literal']] = True
        old['synonyms'] = new_synonyms

    visited = {}
    new_existing_ids = []
    for e in old['existing_ids']:
        if not visited.get(e['iri']):
            new_existing_ids.append(e)
            visited[e['iri']] = True
    old['existing_ids'] = new_existing_ids
    old = preferred_change(old)

    return old
655685
private static void checkDuplicates(ImmutableList<SQLPPTriplesMap> mappings)
            throws DuplicateMappingException {

        Set<SQLPPTriplesMap> mappingSet = new HashSet<>(mappings);

        int duplicateCount = mappings.size() - mappingSet.size();

        /**
         * If there are some triplesMaps, finds them
         */
        if (duplicateCount > 0) {
            Set<String> duplicateIds = new HashSet<>();
            int remaining = duplicateCount;
            for (SQLPPTriplesMap mapping : mappings) {
                if (mappingSet.contains(mapping)) {
                    mappingSet.remove(mapping);
                }
                /**
                 * Duplicate
                 */
                else {
                    duplicateIds.add(mapping.getId());
                    if (--remaining == 0)
                        break;
                }
            }
            //TODO: indicate the source
            throw new DuplicateMappingException(String.format("Found %d duplicates in the following ids: %s",
                    duplicateCount, duplicateIds.toString()));
        }
    }
970726
def _get_distinct_objs(objs):
    
    ids = set()
    res = []
    for obj in objs:
        if not id(obj) in ids:
            ids.add(id(obj))
            res.append(obj)
    return res
82476
def update_with_diff(self, identifier, new_instance):
        
        with self.flushing():
            instance = self.retrieve(identifier)
            before = Version(instance)
            self.merge(instance, new_instance)
            instance.updated_at = instance.new_timestamp()
            after = Version(instance)
        return instance, before - after
1372693
def merge_dicts(*dicts, **copy_check):
    '''
    
    '''
    merged = {}
    if not dicts:
        return merged
    for index, merge_dict in enumerate(dicts):
        if index == 0 and not copy_check.get('copy'):
            merged = merge_dict
        else:
            merged.update(merge_dict)
    return merged
1347618
def _additional(self, idents, kwargs):
        ''''''
        if kwargs:
            for name, value in list(kwargs.items()):
                if not isinstance(value, list):
                    raise ValueError('Additional arguments must be lists of \
same length as idents')
                for i in range(len(value)):
                    self[idents[i]][name] = value[i]
1173385
function nameDuplicates(duplicates) {
    // enumerate duplicate hash groups in sorted order
    var hashes = Object.keys(duplicates).sort();

    // process each group
    processGroup(0);
    function processGroup(i) {
      if(i === hashes.length) {
        // done, create JSON-LD array
        return createArray();
      }

      // name each group member
      var group = duplicates[hashes[i]];
      var results = [];
      nameGroupMember(group, 0);
      function nameGroupMember(group, n) {
        if(n === group.length) {
          // name bnodes in hash order
          results.sort(function(a, b) {
            a = a.hash;
            b = b.hash;
            return (a < b) ? -1 : ((a > b) ? 1 : 0);
          });
          for(var r in results) {
            // name all bnodes in path namer in key-entry order
            // Note: key-order is preserved in javascript
            for(var key in results[r].pathNamer.existing) {
              namer.getName(key);
            }
          }
          return processGroup(i + 1);
        }

        // skip already-named bnodes
        var bnode = group[n];
        if(namer.isNamed(bnode)) {
          return nameGroupMember(group, n + 1);
        }

        // hash bnode paths
        var pathNamer = new UniqueNamer('_:b');
        pathNamer.getName(bnode);
        _hashPaths(bnode, bnodes, namer, pathNamer,
          function(err, result) {
            if(err) {
              return callback(err);
            }
            results.push(result);
            nameGroupMember(group, n + 1);
          });
      }
    }
  }
760923
def create_document_batches(jsonlines, id_field, max_batch_size=CLOUDSEARCH_MAX_BATCH_SIZE):
    
    batch = []
    fixed_initial_size = 2

    def create_entry(line):
        try:
            record = json.loads(line)
        except:
            raise ValueError('Could not parse JSON from: %s' % line)
        key = record[id_field]
        return '{"type":"add","id":%s,"fields":%s}' % (json.dumps(key), line)

    current_size = fixed_initial_size
    for line in jsonlines:
        entry = create_entry(line)
        entry_size = len(entry) + 1

        if max_batch_size > (current_size + entry_size):
            current_size += entry_size
            batch.append(entry)
        else:
            yield '[' + ','.join(batch) + ']'
            batch = [entry]
            current_size = fixed_initial_size + entry_size

    if batch:
        yield '[' + ','.join(batch) + ']'
618172
def remove_duplicates(apps, schema_editor):
    

    # Get the model
    EntityRelationship = apps.get_model('entity', 'EntityRelationship')

    # Find the duplicates
    duplicates = EntityRelationship.objects.all().order_by(
        'sub_entity_id',
        'super_entity_id'
    ).values(
        'sub_entity_id',
        'super_entity_id'
    ).annotate(
        Count('sub_entity_id'),
        Count('super_entity_id'),
        max_id=Max('id')
    ).filter(
        super_entity_id__count__gt=1
    )

    # Loop over the duplicates and delete
    for duplicate in duplicates:
        EntityRelationship.objects.filter(
            sub_entity_id=duplicate['sub_entity_id'],
            super_entity_id=duplicate['super_entity_id']
        ).exclude(
            id=duplicate['max_id']
        ).delete()
244126
private void copyObjectAndUpdateStats(int id, IndexInput in, Long inRef,
      IndexOutput out) throws IOException {
    int mtasId;
    int objectFlags;
    // read
    in.seek(inRef);
    mtasId = in.readVInt();
    assert id == mtasId : "wrong id detected while copying object";
    objectFlags = in.readVInt();
    out.writeVInt(mtasId);
    out.writeVInt(objectFlags);
    if ((objectFlags
        & MtasCodecPostingsFormat.MTAS_OBJECT_HAS_PARENT) == MtasCodecPostingsFormat.MTAS_OBJECT_HAS_PARENT) {
      out.writeVInt(in.readVInt());
    }
    if ((objectFlags
        & MtasCodecPostingsFormat.MTAS_OBJECT_HAS_POSITION_RANGE) == MtasCodecPostingsFormat.MTAS_OBJECT_HAS_POSITION_RANGE) {
      int minPos = in.readVInt();
      int maxPos = in.readVInt();
      out.writeVInt(minPos);
      out.writeVInt(maxPos);
      tokenStatsAdd(minPos, maxPos);
    } else if ((objectFlags
        & MtasCodecPostingsFormat.MTAS_OBJECT_HAS_POSITION_SET) == MtasCodecPostingsFormat.MTAS_OBJECT_HAS_POSITION_SET) {
      int size = in.readVInt();
      out.writeVInt(size);
      SortedSet<Integer> list = new TreeSet<>();
      int previousPosition = 0;
      for (int t = 0; t < size; t++) {
        int pos = in.readVInt();
        out.writeVInt(pos);
        previousPosition = (pos + previousPosition);
        list.add(previousPosition);
      }
      assert list.size() == size : "duplicate positions in set are not allowed";
      tokenStatsAdd(list.first(), list.last());
    } else {
      int pos = in.readVInt();
      out.writeVInt(pos);
      tokenStatsAdd(pos, pos);
    }
    if ((objectFlags
        & MtasCodecPostingsFormat.MTAS_OBJECT_HAS_OFFSET) == MtasCodecPostingsFormat.MTAS_OBJECT_HAS_OFFSET) {
      out.writeVInt(in.readVInt());
      out.writeVInt(in.readVInt());
    }
    if ((objectFlags
        & MtasCodecPostingsFormat.MTAS_OBJECT_HAS_REALOFFSET) == MtasCodecPostingsFormat.MTAS_OBJECT_HAS_REALOFFSET) {
      out.writeVInt(in.readVInt());
      out.writeVInt(in.readVInt());
    }
    if ((objectFlags
        & MtasCodecPostingsFormat.MTAS_OBJECT_HAS_PAYLOAD) == MtasCodecPostingsFormat.MTAS_OBJECT_HAS_PAYLOAD) {
      int length = in.readVInt();
      out.writeVInt(length);
      byte[] payload = new byte[length];
      in.readBytes(payload, 0, length);
      out.writeBytes(payload, payload.length);
    }
    out.writeVLong(in.readVLong());
  }
11347
def duplicates(base, items):
    
    for item in items:
        if item.similarity(base) and not item.equality(base):
            yield item
1266263
def insert_many(self, it):
        """"""
        unique_indexes = self._uniqueIndexes  # [ind for ind in self._indexes.values() if ind.is_unique]
        NO_SUCH_ATTR = object()
        new_objs = list(it)
        if unique_indexes:
            for ind in unique_indexes:
                ind_attr = ind.attr
                new_keys = dict((getattr(obj, ind_attr, NO_SUCH_ATTR), obj) for obj in new_objs)
                if not ind.accept_none and (None in new_keys or NO_SUCH_ATTR in new_keys):
                    raise KeyError("unique key cannot be None or blank for index %s" % ind_attr, 
                                    [ob for ob in new_objs if getattr(ob, ind_attr, NO_SUCH_ATTR) is None])
                if len(new_keys) < len(new_objs):
                    raise KeyError("given sequence contains duplicate keys for index %s" % ind_attr)
                for key in new_keys:
                    if key in ind:
                        obj = new_keys[key]
                        raise KeyError("duplicate unique key value '%s' for index %s" % (getattr(obj, ind_attr), ind_attr), 
                                       new_keys[key])
                    
        for obj in new_objs:
            self.obs.append(obj)
            for attr, ind in self._indexes.items():
                obval = getattr(obj, attr)
                ind[obval] = obj

        return self
1104178
private function checkDefinitions(): void {
    // Add many to many stuff if necessary
    foreach($this->unchecked as $def) {
      foreach($def->many_to_many as $relation) {
        //echo "Processing many to many for {$def->name}\n";

        // Set up left keys
        $related = $this->defs[$relation->entity_left];

        if(empty($relation->id_left)) {
          //echo "Setting {$relation->alias_join} id_left to {$related->id->alias}\n";
          $relation->idLeft($related->id->alias);
        }

        if(empty($relation->key_left)) {
          //echo "Setting {$relation->alias_join} key_left to " . keyify($related->name) . "\n";
          $relation->keyLeft(keyify($related->name));
        }
        //

        // Set up right keys
        $related = $this->defs[$relation->entity_right];

        if(empty($relation->id_right)) {
          //echo "Setting {$relation->alias_join} id_right to {$related->id->alias}\n";
          $relation->idRight($related->id->alias);
        }

        if(empty($relation->key_right)) {
          //echo "Setting {$relation->alias_join} key_right to " . keyify($related->name) . "\n";
          $relation->keyRight(keyify($related->name));
        }
        //

        // Left side relations
        $related = $this->defs[$relation->entity_left];

        //echo "Adding {$related->name} hasManyThrough {$relation->alias_join} to {$relation->alias_right}\n";
        $related->hasManyThrough(pluralize($relation->alias_right), $relation->entity_join, $relation->entity_right)
          ->idLocal($relation->id_left)
          ->idForeign($relation->id_right)
          ->keyLocal($relation->key_left)
          ->keyForeign($relation->key_right)
        ;

        if(!array_key_exists($relation->alias_join, $def->has_many)) {
          //echo "Adding {$related->name} hasMany {$def->name}\n";
          $related->hasMany($relation->alias_join, $def->full_name)
            ->localKey  ($relation->id_left)
            ->foreignKey($relation->key_left)
          ;
        }

        if(!array_key_exists($relation->alias_left, $def->belongs_to)) {
          //echo "Adding {$def->name} belongsTo {$related->name}\n";
          $def->belongsTo($relation->alias_left, $related->full_name);
        }
        //

        // Right side relations
        $related = $this->defs[$relation->entity_right];

        //echo "Adding {$related->name} hasManyThrough {$relation->alias_join} to {$relation->alias_left}\n";
        $related->hasManyThrough(pluralize($relation->alias_left), $relation->entity_join, $relation->entity_left)
          ->idLocal($relation->id_right)
          ->idForeign($relation->id_left)
          ->keyLocal($relation->key_right)
          ->keyForeign($relation->key_left)
        ;

        if(!array_key_exists($relation->alias_join, $def->has_many)) {
          //echo "Adding {$related->name} hasMany {$def->name}\n";
          $related->hasMany($relation->alias_join, $def->full_name)
            ->localKey  ($relation->id_right)
            ->foreignKey($relation->key_right)
          ;
        }

        if(!array_key_exists($relation->alias_right, $def->belongs_to)) {
          //echo "Adding {$def->name} belongsTo {$related->name}\n";
          $def->belongsTo($relation->alias_right, $related->full_name);
        }
        //
      }
    }

    // Add has many stuff if necessary
    foreach($this->unchecked as $def) {
      foreach($def->has_many as $relation) {
        //echo "Processing has many for {$def->name}\n";

        $related = $this->defs[$relation->foreign_entity];

        if(empty($relation->local_key)) {
          //echo "Setting {$relation->alias} local_key to {$def->id->alias}\n";
          $relation->localKey($def->id->alias);
        }

        if(empty($relation->foreign_key)) {
          //echo "Setting {$relation->alias} foreign_key to " . keyify($related->name) . "\n";
          $relation->foreignKey(keyify($related->name));
        }
      }
    }

    // Add belongs to stuff if necessary
    foreach($this->unchecked as $def) {
      foreach($def->belongs_to as $relation) {
        //echo "Processing belongs to for {$def->name}\n";

        $related = $this->defs[$relation->foreign_entity];

        if(empty($relation->local_key)) {
          //echo "Setting {$relation->alias} local_key to " . keyify($related->name) . "\n";
          $relation->localKey(keyify($related->name));
        }

        if(empty($relation->foreign_key)) {
          //echo "Setting {$relation->alias} foreign_key to {$def->id->alias}\n";
          $relation->foreignKey($related->id->alias);
        }

        if(!array_key_exists($relation->local_key, $related->required)) {
          //echo "Adding {$def->name} column {$relation->local_key} of type {$related->id->type}\n";
          $def->required($relation->local_key, $related->id->type);
        }
      }
    }

    // We've checked everything in the queue.  Clear it out.
    $this->unchecked = [];
  }
106243
public JSONObject batchSynonyms(List<JSONObject> objects, boolean forwardToReplicas) throws AlgoliaException {
    return batchSynonyms(objects, forwardToReplicas, false);
  }
148314
def PrependUnique(self, delete_existing=0, **kw):
        
        kw = copy_non_reserved_keywords(kw)
        for key, val in kw.items():
            if SCons.Util.is_List(val):
                val = _delete_duplicates(val, not delete_existing)
            if key not in self._dict or self._dict[key] in ('', None):
                self._dict[key] = val
            elif SCons.Util.is_Dict(self._dict[key]) and \
                 SCons.Util.is_Dict(val):
                self._dict[key].update(val)
            elif SCons.Util.is_List(val):
                dk = self._dict[key]
                if not SCons.Util.is_List(dk):
                    dk = [dk]
                if delete_existing:
                    dk = [x for x in dk if x not in val]
                else:
                    val = [x for x in val if x not in dk]
                self._dict[key] = val + dk
            else:
                dk = self._dict[key]
                if SCons.Util.is_List(dk):
                    # By elimination, val is not a list.  Since dk is a
                    # list, wrap val in a list first.
                    if delete_existing:
                        dk = [x for x in dk if x not in val]
                        self._dict[key] = [val] + dk
                    else:
                        if not val in dk:
                            self._dict[key] = [val] + dk
                else:
                    if delete_existing:
                        dk = [x for x in dk if x not in val]
                    self._dict[key] = val + dk
        self.scanner_map_delete(kw)
17509
private void copyObjectIDsToBatch(BatchResult batchResult, DBObjectBatch dbObjBatch) {
        if (batchResult.getResultObjectCount() < dbObjBatch.getObjectCount()) {
            m_logger.warn("Batch result returned fewer objects ({}) than input batch ({})",
                          batchResult.getResultObjectCount(), dbObjBatch.getObjectCount());
        }
        Iterator<ObjectResult> resultIter = batchResult.getResultObjects().iterator();
        Iterator<DBObject> objectIter = dbObjBatch.getObjects().iterator();
        while (resultIter.hasNext()) {
            if (!objectIter.hasNext()) {
                m_logger.warn("Batch result has more objects ({}) than input batch ({})!",
                              batchResult.getResultObjectCount(), dbObjBatch.getObjectCount());
                break;
            }
            ObjectResult objResult = resultIter.next();
            DBObject dbObj = objectIter.next();
            if (Utils.isEmpty(dbObj.getObjectID())) {
                dbObj.setObjectID(objResult.getObjectID());
            } else if (!dbObj.getObjectID().equals(objResult.getObjectID())) {
                m_logger.warn("Batch results out of order: expected ID '{}', got '{}'",
                              dbObj.getObjectID(), objResult.getObjectID());
            }
        }
    }
1043949
function () {
      var doUniqueConnectionIdsCheck = function (compound) {
        var duplicateConnectionIds = [];
        compound.connections.forEach(function (connection) {
          var connectionId = connection.connectionId;

          function filterByConnectionId (connection) {
            return connection.connectionId === connectionId;
          }

          var arrayByConnectionId = compound.connections.filter(filterByConnectionId);
          if (arrayByConnectionId.length > 1 && duplicateConnectionIds.indexOf(connectionId) < 0) {
            duplicateConnectionIds.push(connectionId);
          }
        });
        duplicateConnectionIds.forEach(function (connectionId) {
          errors.push('RuleViolation identified: Artifact \'' + compound.artifactId +
            '\' contains multiple connections with \'connectionId\'=\'' + connectionId + '\'');
        });
      };
      //
      if (doc.artifacts && doc.artifacts.compoundComponents) {
        doc.artifacts.compoundComponents.forEach(doUniqueConnectionIdsCheck);
      }
    }