Will compare the triplestore and elasticsearch index to ensure that
        that elasticsearch and triplestore items match. elasticsearch records
        that are not in the triplestore will be deleteed
430993
def validate_index(self, rdf_class):
        
        es_ids = set(self.get_es_ids())
        tstore_ids = set([item[1]
                          for item in self.get_uri_list(no_status=True)])
        diff = es_ids - tstore_ids
        if diff:
            pdb.set_trace()
            action_list = self.es_worker.make_action_list(diff,
                                                          action_type="delete")
            results = self.es_worker.bulk_save(action_list)
250284
def _CompareStores(self, storage_reader, compare_storage_reader):
    
    storage_counters = self._CalculateStorageCounters(storage_reader)
    compare_storage_counters = self._CalculateStorageCounters(
        compare_storage_reader)

    # TODO: improve comparison, currently only total numbers are compared.

    return storage_counters == compare_storage_counters
673595
public boolean isSolrSearch() {

        if (VFS.equals(m_source)) {
            // VFS search selected
            return false;
        }
        // index selected and query entered --> Solr search else VFS
        return (CmsStringUtil.isNotEmptyOrWhitespaceOnly(m_source)
            && CmsStringUtil.isNotEmptyOrWhitespaceOnly(m_query));
    }
930182
protected Record<V> removeImpl (int key, boolean checkShrink)
    {
        int index = keyToIndex(key);

        // go through the chain looking for a match
        for (Record<V> prev = null, rec = _buckets[index]; rec != null; rec = rec.next) {
            if (rec.key == key) {
                if (prev == null) {
                    _buckets[index] = rec.next;
                } else {
                    prev.next = rec.next;
                }
                _size--;
                if (checkShrink) {
                    checkShrink();
                }
                return rec;
            }
            prev = rec;
        }

        return null;
    }
784361
def items_differ(jsonitems, dbitems, subfield_dict):
    

    # short circuit common cases
    if len(jsonitems) == len(dbitems) == 0:
        # both are empty
        return False
    elif len(jsonitems) != len(dbitems):
        # if lengths differ, they're definitely different
        return True

    original_jsonitems = jsonitems
    jsonitems = copy.deepcopy(jsonitems)
    keys = jsonitems[0].keys()

    # go over dbitems looking for matches
    for dbitem in dbitems:
        order = getattr(dbitem, 'order', None)
        match = None
        for i, jsonitem in enumerate(jsonitems):
            # check if all keys (excluding subfields) match
            for k in keys:
                if k not in subfield_dict and getattr(dbitem, k) != jsonitem.get(k, None):
                    break
            else:
                # all fields match so far, possibly equal, just check subfields now
                for k in subfield_dict:
                    jsonsubitems = jsonitem[k]
                    dbsubitems = list(getattr(dbitem, k).all())
                    if items_differ(jsonsubitems, dbsubitems, subfield_dict[k][2]):
                        break
                else:
                    # if the dbitem sets 'order', then the order matters
                    if order is not None and int(order) != original_jsonitems.index(jsonitem):
                        break
                    # these items are equal, so let's mark it for removal
                    match = i
                    break

        if match is not None:
            # item exists in both, remove from jsonitems
            jsonitems.pop(match)
        else:
            # exists in db but not json
            return True

    # if we get here, jsonitems has to be empty because we asserted that the length was
    # the same and we found a match for each thing in dbitems, here's a safety check just in case
    if jsonitems:       # pragma: no cover
        return True

    return False
1077420
def clear_es():
        """"""
        # TODO: should receive a catalog slug.
        ESHypermap.es.indices.delete(ESHypermap.index_name, ignore=[400, 404])
        LOGGER.debug('Elasticsearch: Index cleared')
410003
def clean_store(self, store, display_progress=True):
        
        spec = self._get_store_spec(store)
        progress = tqdm if display_progress else lambda x: x
        if spec['protocol'] == 'file':
            folder = os.path.join(spec['location'], self.database)
            delete_list = set(os.listdir(folder)).difference(self.fetch('hash'))
            print('Deleting %d unused items from %s' % (len(delete_list), folder), flush=True)
            for f in progress(delete_list):
                os.remove(os.path.join(folder, f))
        elif spec['protocol'] == 's3':
            try:
                S3Folder(database=self.database, **spec).clean(self.fetch('hash'))
            except TypeError:
                raise DataJointError('External store {store} configuration is incomplete.'.format(store=store))
228641
def HasIndex(self, index):
        
        for i in self.Items:
            if i.index == index:
                return True
        return False
725476
def on_mismatch(self, pair):
        
        remote_entry = pair.remote
        if self.options.get("delete_unmatched") and remote_entry:
            self._log_action("delete", "unmatched", ">", remote_entry)
            if remote_entry.is_dir():
                self._remove_dir(remote_entry)
            else:
                self._remove_file(remote_entry)
        else:
            self._log_action("skip", "unmatched", "-", pair.any_entry, min_level=4)
719700
def _find_matching_collections_externally(collections, record):
    
    index, doc_type = RecordIndexer().record_to_index(record)
    body = {"doc": record.dumps()}
    results = current_search_client.percolate(
        index=index,
        doc_type=doc_type,
        allow_no_indices=True,
        ignore_unavailable=True,
        body=body
    )
    prefix_len = len('collection-')
    for match in results['matches']:
        collection_name = match['_id']
        if collection_name.startswith('collection-'):
            name = collection_name[prefix_len:]
            if name in collections:
                yield collections[name]['ancestors']
    raise StopIteration
525821
def remove_stale_indexes_from_bika_catalog(portal):
    
    logger.info("Removing stale indexes and metadata from bika_catalog ...")
    cat_id = "bika_catalog"
    indexes_to_remove = [
        "getAnalyst",
        "getAnalysts",
        "getAnalysisService",
        "getClientOrderNumber",
        "getClientReference",
        "getClientSampleID",
        "getContactTitle",
        "getDateDisposed",
        "getDateExpired",
        "getDateOpened",
        "getDatePublished",
        "getInvoiced",
        "getPreserver",
        "getSamplePointTitle",
        "getSamplePointUID",
        "getSampler",
        "getScheduledSamplingSampler",
        "getSamplingDate",
        "getWorksheetTemplateTitle",
        "BatchUID",
    ]
    metadata_to_remove = [
        "getAnalysts",
        "getClientOrderNumber",
        "getClientReference",
        "getClientSampleID",
        "getContactTitle",
        "getSamplePointTitle",
        "getAnalysisService",
        "getDatePublished",
    ]
    for index in indexes_to_remove:
        del_index(portal, cat_id, index)

    for metadata in metadata_to_remove:
        del_metadata(portal, cat_id, metadata)
    commit_transaction(portal)
33730
def index(models=None, name=None, force=False, keep=False):
    '''
    
    '''
    index_name = name or default_index_name()

    doc_types_names = [m.__name__.lower() for m in adapter_catalog.keys()]
    models = [model.lower().rstrip('s') for model in (models or [])]
    for model in models:
        if model not in doc_types_names:
            log.error('Unknown model %s', model)
            sys.exit(-1)

    log.info('Initiliazing index "{0}"'.format(index_name))
    if es.indices.exists(index_name):
        if IS_TTY and not force:
            msg = 'Index {0} will be deleted, are you sure?'
            click.confirm(msg.format(index_name), abort=True)
        es.indices.delete(index_name)

    es.initialize(index_name)

    with handle_error(index_name, keep):

        disable_refresh(index_name)
        for adapter in iter_adapters():
            if not models or adapter.doc_type().lower() in models:
                index_model(index_name, adapter)
            else:
                log.info('Copying {0} objects to the new index'.format(
                         adapter.model.__name__))
                # Need upgrade to Elasticsearch-py 5.0.0 to write:
                # es.reindex({
                #     'source': {'index': es.index_name, 'type': adapter.doc_type()},
                #     'dest': {'index': index_name}
                # })
                #
                # http://elasticsearch-py.readthedocs.io/en/master/api.html#elasticsearch.Elasticsearch.reindex
                # This method (introduced in Elasticsearch 2.3 but only in Elasticsearch-py 5.0.0)
                # triggers a server-side documents copy.
                # Instead we use this helper for meant for backward compatibility
                # but with poor performance as copy is client-side (scan+bulk)
                es_reindex(es.client, es.index_name, index_name, scan_kwargs={
                    'doc_type': adapter.doc_type()
                })

        enable_refresh(index_name)
    # At this step, we don't want error handler to delete the index
    # in case of error
    set_alias(index_name, delete=not keep)
950223
function checkIndexFieldsMatch(indexFields, sortOrder, fields) {
  if (sortOrder) {
    // array has to be a strict subarray of index array. furthermore,
    // the sortOrder fields need to all be represented in the index
    var sortMatches = utils.oneArrayIsStrictSubArrayOfOther(sortOrder, indexFields);
    var selectorMatches = utils.oneArrayIsSubArrayOfOther(fields, indexFields);

    return sortMatches && selectorMatches;
  }

  // all of the user's specified fields still need to be
  // on the left side of the index array, although the order
  // doesn't matter
  return utils.oneSetIsSubArrayOfOther(fields, indexFields);
}
991097
boolean updateIndexEntry(S userStorable, S oldUserStorable) throws PersistException {
        Storable newIndexEntry = makeIndexEntry(userStorable);

        if (oldUserStorable != null) deleteOldEntry: {
            Storable oldIndexEntry;
            try {
                oldIndexEntry = makeIndexEntry(oldUserStorable);
            } catch (PersistException e) {
                Throwable cause = e.getCause();
                if (cause instanceof IllegalArgumentException) {
                    // Can be caused by a corrupt master record, which is
                    // attempting do assign an illegal value to the index. There's
                    // no way to find the old index entry to delete.
                    break deleteOldEntry;
                }
                throw e;
            }

            if (oldIndexEntry.equalPrimaryKeys(newIndexEntry)) {
                // Index entry didn't change, so nothing to do. If the index
                // entry has a version, it will lag behind the master's version
                // until the index entry changes, at which point the version
                // will again match the master.
                return true;
            }

            oldIndexEntry.tryDelete();
        }

        return insertIndexEntry(userStorable, newIndexEntry);
    }
195777
private boolean checkMatch() {
        if (matchList == null) {
            return true;
        }
        final int matchSize = matchList.size();
        final int ancestorSize = topicIdList.size();
        final List<String> tail = topicIdList.subList(ancestorSize - matchSize, ancestorSize);
        return matchList.equals(tail);
    }
311647
public boolean deleteBulkDocs(List<String> indexList, List<String> typeList,
            QueryBuilder filterQueryBuilder) {
        return indexList.stream().flatMap(index -> typeList.stream()
                .map(type -> deleteBulkDocs(index, type, filterQueryBuilder)))
                .noneMatch(BulkResponse::hasFailures);
    }
448329
def check_elasticsearch(record, *args, **kwargs):
    
    def can(self):
        """Try to search for given record."""
        search = request._methodview.search_class()
        search = search.get_record(str(record.id))
        return search.count() == 1

    return type('CheckES', (), {'can': can})()