Write items into ElasticSearch.

        :param items: Pandas DataFrame
330419
def write(self, items):
        
        if self._read_only:
            raise IOError("Cannot write, Connector created as Read Only")

        if len(items) == 0:
            logger.info(self.__log_prefix + " Nothing to write")
            return

        # Uploading info to the new ES
        rows = items.to_dict("index")
        docs = []
        for row_index in rows.keys():
            row = rows[row_index]
            item_id = row[self.AUTHOR_ORG] + '_' + row[self.PROJECT] + '_' \
                + row[self.TIMEFRAME] + '_' + row[self.AUTHOR_UUID]
            item_id = item_id.replace(' ', '').lower()

            doc = {
                "_index": self._es_index,
                "_type": "item",
                "_id": item_id,
                "_source": row
            }
            docs.append(doc)

        # TODO uncomment following lines for incremental version
        # # Delete old data if exists to ensure refreshing in case of deleted commits
        # timeframe = docs[0]['_source']['timeframe']
        # org = docs[0]['_source']['author_org_name']
        # project = docs[0]['_source']['project']
        # s = Search(using=self._es_conn, index=self._es_index)
        # s = s.filter('term', project=project)
        # s = s.filter('term', author_org_name=org)
        # s = s.filter('term', timeframe=timeframe)
        # response = s.execute()
        #
        # if response.hits.total > 0:
        #     response = s.delete()
        #     logger.info("[Onion] Deleted " + str(response.deleted) + " items for refreshing: " + timeframe + " "
        #                 + org + " " + project)

        # TODO exception and error handling
        helpers.bulk(self._es_conn, docs)
        logger.info(self.__log_prefix + " Written: " + str(len(docs)))
1225345
def extract_index(index_data, global_index=False):
    '''
    
    '''
    parsed_data = {}
    keys = []

    for key, value in six.iteritems(index_data):
        for item in value:
            for field, data in six.iteritems(item):
                if field == 'hash_key':
                    parsed_data['hash_key'] = data
                elif field == 'hash_key_data_type':
                    parsed_data['hash_key_data_type'] = data
                elif field == 'range_key':
                    parsed_data['range_key'] = data
                elif field == 'range_key_data_type':
                    parsed_data['range_key_data_type'] = data
                elif field == 'name':
                    parsed_data['name'] = data
                elif field == 'read_capacity_units':
                    parsed_data['read_capacity_units'] = data
                elif field == 'write_capacity_units':
                    parsed_data['write_capacity_units'] = data
                elif field == 'includes':
                    parsed_data['includes'] = data
                elif field == 'keys_only':
                    parsed_data['keys_only'] = True

    if parsed_data['hash_key']:
        keys.append(
            HashKey(
                parsed_data['hash_key'],
                data_type=parsed_data['hash_key_data_type']
            )
        )
    if parsed_data.get('range_key'):
        keys.append(
            RangeKey(
                parsed_data['range_key'],
                data_type=parsed_data['range_key_data_type']
            )
        )
    if (
            global_index and
            parsed_data['read_capacity_units'] and
            parsed_data['write_capacity_units']):
        parsed_data['throughput'] = {
            'read':     parsed_data['read_capacity_units'],
            'write':    parsed_data['write_capacity_units']
        }
    if parsed_data['name'] and keys:
        if global_index:
            if parsed_data.get('keys_only') and parsed_data.get('includes'):
                raise SaltInvocationError('Only one type of GSI projection can be used.')

            if parsed_data.get('includes'):
                return GlobalIncludeIndex(
                    parsed_data['name'],
                    parts=keys,
                    throughput=parsed_data['throughput'],
                    includes=parsed_data['includes']
                )
            elif parsed_data.get('keys_only'):
                return GlobalKeysOnlyIndex(
                    parsed_data['name'],
                    parts=keys,
                    throughput=parsed_data['throughput'],
                )
            else:
                return GlobalAllIndex(
                    parsed_data['name'],
                    parts=keys,
                    throughput=parsed_data['throughput']
                )
        else:
            return AllIndex(
                parsed_data['name'],
                parts=keys
            )
763475
def flatten(items,enter=lambda x:isinstance(x, list)):
    # http://stackoverflow.com/a/40857703
    # https://github.com/ctmakro/canton/blob/master/canton/misc.py
    """"""
    for x in items:
        if enter(x):
            yield from flatten(x)
        else:
            yield x
773392
def insert_many(self, items):
    
    return SessionContext.session.execute(
        self.insert(values=[
            to_dict(item, self.c)
            for item in items
        ]),
    ).rowcount
979681
def get_items_as_list(items,keys,items_names='styles'):
	
	if type(items)!=dict:
		if type(items)==list:
			if len(items)!=len(keys):
				raise Exception('List of {0} is not the same length as keys'.format(items_names))
			else:
				items=dict(zip(keys,items))
		else:
			items=dict(zip(keys,[items]*len(keys)))
	return items
36360
def get_elastic_items(self, elastic_scroll_id=None, _filter=None, ignore_incremental=False):
        

        headers = {"Content-Type": "application/json"}

        if not self.elastic:
            return None
        url = self.elastic.index_url
        # 1 minute to process the results of size items
        # In gerrit enrich with 500 items per page we need >1 min
        # In Mozilla ES in Amazon we need 10m
        max_process_items_pack_time = "10m"  # 10 minutes
        url += "/_search?scroll=%s&size=%i" % (max_process_items_pack_time,
                                               self.scroll_size)

        if elastic_scroll_id:
            """ Just continue with the scrolling """
            url = self.elastic.url
            url += "/_search/scroll"
            scroll_data = {
                "scroll": max_process_items_pack_time,
                "scroll_id": elastic_scroll_id
            }
            query_data = json.dumps(scroll_data)
        else:
            # If using a perceval backends always filter by repository
            # to support multi repository indexes
            filters_dict = self.get_repository_filter_raw(term=True)
            if filters_dict:
                filters = json.dumps(filters_dict)
            else:
                filters = ''

            if self.filter_raw:
                for fltr in self.filter_raw_dict:
                    filters += '''
                        , {"term":
                            { "%s":"%s"  }
                        }
                    ''' % (fltr['name'], fltr['value'])

            if _filter:
                filter_str = '''
                    , {"terms":
                        { "%s": %s }
                    }
                ''' % (_filter['name'], _filter['value'])
                # List to string conversion uses ' that are not allowed in JSON
                filter_str = filter_str.replace("'", "\"")
                filters += filter_str

            # The code below performs the incremental enrichment based on the last value of `metadata__timestamp`
            # in the enriched index, which is calculated in the TaskEnrich before enriching the single repos that
            # belong to a given data source. The old implementation of the incremental enrichment, which consisted in
            # collecting the last value of `metadata__timestamp` in the enriched index for each repo, didn't work
            # for global data source (which are collected globally and only partially enriched).
            if self.from_date and not ignore_incremental:
                date_field = self.get_incremental_date()
                from_date = self.from_date.isoformat()

                filters += '''
                    , {"range":
                        {"%s": {"gte": "%s"}}
                    }
                ''' % (date_field, from_date)
            elif self.offset and not ignore_incremental:
                filters += '''
                    , {"range":
                        {"offset": {"gte": %i}}
                    }
                ''' % self.offset

            # Order the raw items from the old ones to the new so if the
            # enrich process fails, it could be resume incrementally
            order_query = ''
            order_field = None
            if self.perceval_backend:
                order_field = self.get_incremental_date()
            if order_field is not None:
                order_query = ', "sort": { "%s": { "order": "asc" }} ' % order_field

            filters_should = ''
            if self.filter_raw_should:
                for fltr in self.filter_raw_should_dict:
                    filters_should += '''
                        {"prefix":
                            { "%s":"%s"  }
                        },''' % (fltr['name'], fltr['value'])

                filters_should = filters_should.rstrip(',')
                query_should = '{"bool": {"should": [%s]}}' % filters_should
                filters += ", " + query_should

            # Fix the filters string if it starts with "," (empty first filter)
            if filters.lstrip().startswith(','):
                filters = filters.lstrip()[1:]

            query = """
            {
                "query": {
                    "bool": {
                        "filter": [%s]
                    }
                } %s
            }
            """ % (filters, order_query)

            logger.debug("Raw query to %s\n%s", self.elastic.anonymize_url(url),
                         json.dumps(json.loads(query), indent=4))
            query_data = query

        rjson = None
        try:
            res = self.requests.post(url, data=query_data, headers=headers)
            res.raise_for_status()
            rjson = res.json()
        except Exception:
            # The index could not exists yet or it could be empty
            logger.warning("No results found from %s", self.elastic.anonymize_url(url))

        return rjson
421956
def itemat(iterable, index):
    

    result = None

    handleindex = True

    if isinstance(iterable, dict):
        handleindex = False

    else:
        try:
            result = iterable[index]
        except TypeError:
            handleindex = False

    if not handleindex:
        iterator = iter(iterable)

        if index < 0:  # ensure index is positive
            index += len(iterable)

        while index >= 0:
            try:
                value = next(iterator)

            except StopIteration:
                raise IndexError(
                    "{0} index {1} out of range".format(
                        iterable.__class__, index
                    )
                )

            else:
                if index == 0:
                    result = value
                    break
                index -= 1

    return result
1320648
public SuggestIndexResponse index(final SuggestItem[] items) {
        // TODO parallel?
        final SuggestItem[] array = Stream.of(items).filter(item -> !item.isBadWord(badWords)).toArray(n -> new SuggestItem[n]);

        try {
            final long start = System.currentTimeMillis();
            final SuggestWriterResult result = suggestWriter.write(client, settings, index, array, true);
            return new SuggestIndexResponse(items.length, items.length, result.getFailures(), System.currentTimeMillis() - start);
        } catch (Exception e) {
            throw new SuggestIndexException("Failed to write items[" + items.length + "] to " + index, e);
        }
    }
1297509
function(_items) {
        var list = new List();
        if(_items) list.push.apply(list, _items);
        return list;
      }
878386
def load_items(self, items):
        
        loaded_items = {}
        requests = collections.deque(create_batch_get_chunks(items))
        while requests:
            request = requests.pop()
            try:
                response = self.dynamodb_client.batch_get_item(RequestItems=request)
            except botocore.exceptions.ClientError as error:
                raise BloopException("Unexpected error while loading items.") from error

            # Accumulate results
            for table_name, table_items in response.get("Responses", {}).items():
                loaded_items.setdefault(table_name, []).extend(table_items)

            # Push additional request onto the deque.
            # "UnprocessedKeys" is {} if this request is done
            if response["UnprocessedKeys"]:
                requests.append(response["UnprocessedKeys"])
        return loaded_items
143480
def encode_items(items):
    
    encoded = []
    for key, values in items:
        for value in values:
            encoded.append((key.encode('utf-8'), value.encode('utf-8')))
    return encoded
1083318
def _chunks(self, items, limit):
        
        for i in range(0, len(items), limit):
            yield items[i:i + limit]
123996
def create_terms_index(es, index_name: str):
    """"""

    with open(mappings_terms_fn, "r") as f:
        mappings_terms = yaml.load(f, Loader=yaml.SafeLoader)

    try:
        es.indices.create(index=index_name, body=mappings_terms)

    except Exception as e:
        log.error(f"Could not create elasticsearch terms index: {e}")
1083006
def _build_str_from_time_items(items):
    
    if not items:
        return None
    items = [int(item) for item in items if item]
    items = items + [0 for _ in xrange(6-len(items))]
    return '%d-%02d-%02d %02d:%02d:%02d' % (items[0], items[1], items[2], items[3], items[4], items[5])
349186
function insertAt(items, item, index) {
  var copy = items.slice();
  copy.splice(index, 0, item);
  return copy;
}
683931
def _update_items(self, items):
        
        self.items = items
        self.map = {item: idx for (idx, item) in enumerate(items)}
33086
def fetch(elastic, backend, limit=None, search_after_value=None, scroll=True):
    

    logging.debug("Creating a elastic items generator.")

    elastic_scroll_id = None
    search_after = search_after_value

    while True:
        if scroll:
            rjson = get_elastic_items(elastic, elastic_scroll_id, limit)
        else:
            rjson = get_elastic_items_search(elastic, search_after, limit)

        if rjson and "_scroll_id" in rjson:
            elastic_scroll_id = rjson["_scroll_id"]

        if rjson and "hits" in rjson:
            if not rjson["hits"]["hits"]:
                break
            for hit in rjson["hits"]["hits"]:
                item = hit['_source']
                if 'sort' in hit:
                    search_after = hit['sort']
                try:
                    backend._fix_item(item)
                except Exception:
                    pass
                yield item
        else:
            logging.error("No results found from %s", elastic.index_url)
            break

    return