Lazily retrieve and paginate report results and build Record instances from returned data
540072
def _evaluate(self):
        """"""
        if self._elements:
            for element in self._elements:
                yield element
        else:
            for page in itertools.count():
                raw_elements = self._retrieve_raw_elements(page)

                for raw_element in raw_elements:
                    element = self._parse_raw_element(raw_element)
                    self._elements.append(element)
                    yield element
                    if self.__limit and len(self._elements) >= self.__limit:
                        break

                if any([
                    len(raw_elements) < self.page_size,
                    (self.__limit and len(self._elements) >= self.__limit)
                ]):
                    break
394735
def get(self, limit=None, page=None):
        
        has_more = False
        self.bounds.paginate = True
        limit_paginate, offset = self.bounds.get(limit, page)
        self.default_val = []
        results = self._query('get')

        if limit_paginate:
            self.bounds.paginate = False
            if len(results) == limit_paginate:
                has_more = True
                results.pop(-1)

        it = ResultsIterator(results, orm_class=self.orm_class, has_more=has_more, query=self)
        return self.iterator_class(it)
102688
def genrows(cursor: Cursor, arraysize: int = 1000) \
        -> Generator[List[Any], None, None]:
    
    # http://code.activestate.com/recipes/137270-use-generators-for-fetching-large-db-record-sets/  # noqa
    while True:
        results = cursor.fetchmany(arraysize)
        if not results:
            break
        for result in results:
            yield result
984736
def paginate(self, request, collection):
        
        p = Paginator(request, self, collection)
        return p.paginator and p or UpdatedList(collection)
671276
def to_paginated_list(self, result, _ns, _operation, **kwargs):
        
        items, context = self.parse_result(result)
        headers = dict()
        paginated_list = PaginatedList(
            items=items,
            _page=self,
            _ns=_ns,
            _operation=_operation,
            _context=context,
        )
        return paginated_list, headers
418015
def _query_iterator(self, result, chunksize, columns, coerce_float=True,
                        parse_dates=None):
        """"""

        while True:
            data = result.fetchmany(chunksize)
            if not data:
                break
            else:
                self.frame = DataFrame.from_records(
                    data, columns=columns, coerce_float=coerce_float)

                self._harmonize_columns(parse_dates=parse_dates)

                if self.index is not None:
                    self.frame.set_index(self.index, inplace=True)

                yield self.frame
1224903
private Collection<String> getPages(String startDate, String endDate, List<Dimension> dimensions,
      ApiDimensionFilter countryFilter, Queue<Pair<String, FilterOperator>> toProcess, int rowLimit) {
    String country = GoogleWebmasterFilter.countryFilterToString(countryFilter);

    ConcurrentLinkedDeque<String> allPages = new ConcurrentLinkedDeque<>();
    int r = 0;
    while (r <= GET_PAGES_RETRIES) {
      ++r;
      log.info(String.format("Get pages at round %d with size %d.", r, toProcess.size()));
      ConcurrentLinkedDeque<Pair<String, FilterOperator>> nextRound = new ConcurrentLinkedDeque<>();
      ExecutorService es = Executors.newFixedThreadPool(10,
          ExecutorsUtils.newDaemonThreadFactory(Optional.of(log), Optional.of(this.getClass().getSimpleName())));

      while (!toProcess.isEmpty()) {
        submitJob(toProcess.poll(), countryFilter, startDate, endDate, dimensions, es, allPages, nextRound, rowLimit);
      }
      //wait for jobs to finish and start next round if necessary.
      try {
        es.shutdown();
        boolean terminated = es.awaitTermination(5, TimeUnit.MINUTES);
        if (!terminated) {
          es.shutdownNow();
          log.warn("Timed out while getting all pages for country-{} at round {}. Next round now has size {}.", country,
              r, nextRound.size());
        }
      } catch (InterruptedException e) {
        throw new RuntimeException(e);
      }

      if (nextRound.isEmpty()) {
        break;
      }
      toProcess = nextRound;
      coolDown(r, PAGES_GET_COOLDOWN_TIME);
    }
    if (r == GET_PAGES_RETRIES + 1) {
      throw new RuntimeException(
          String.format("Getting all pages reaches the maximum number of retires %d. Date range: %s ~ %s. Country: %s.",
              GET_PAGES_RETRIES, startDate, endDate, country));
    }
    return allPages;
  }
163121
def list_build_records(page_size=200, page_index=0, sort="", q=""):
    
    data = list_build_records_raw(page_size, page_index, sort, q)
    if data:
        return utils.format_json_list(data)
1272056
def search(self, query, fields=None, page=1, max_records=None, flatten=True):
        """"""
        if fields is None:
            fields = []
        page = int(page)
        pages = float('inf')
        data = {
            "query": query,
            "page": page,
            "fields": fields,
            "flatten": flatten
        }

        count = 0
        while page <= pages:
            payload = self._post(self.search_path, data=data)
            pages = payload['metadata']['pages']
            page += 1
            data["page"] = page

            for result in payload["results"]:
                yield result
                count += 1
                if max_records and count >= max_records:
                    return
14199
def report(self):
        
        output = OrderedDict()

        for arg in GLOBAL_ARGUMENTS:
            output[arg] = getattr(self.args, arg) or self.config.get(arg, None)

        output['title'] = getattr(self.args, 'title') or self.config.get('title', 'Untitled Report')
        output['run_date'] = datetime.now().strftime('%Y-%m-%d')
        output['queries'] = []

        for analytic in self.config.get('queries', []):
            print 'Querying "%s"' % analytic['name']

            results = self.query(
                metrics=analytic['metrics'],
                dimensions=analytic.get('dimensions', []),
                filters=analytic.get('filter', None),
                segment=analytic.get('segment', None),
                sort=analytic.get('sort', []),
                start_index=analytic.get('start-index', 1),
                max_results=analytic.get('max-results', 10)
            )
            
            dimensions_len = len(analytic.get('dimensions', []))

            data = OrderedDict([ 
                ('config', analytic),
                ('sampled', results.get('containsSampledData', False)),
                ('sampleSize', int(results.get('sampleSize', 0))),
                ('sampleSpace', int(results.get('sampleSpace', 0))),
                ('data_types', OrderedDict()),
                ('data', OrderedDict())
            ])
                    
            for column in results['columnHeaders'][dimensions_len:]:
                data['data_types'][column['name']] = column['dataType']

            def cast_data_type(d, dt):
                if dt == 'INTEGER':
                    return int(d)
                elif data_type in ['TIME', 'FLOAT', 'CURRENCY', 'PERCENT']:  
                    return float(d)
                else:
                    raise Exception('Unknown metric data type: %s' % data_type)

            for i, metric in enumerate(analytic['metrics']):
                data['data'][metric] = OrderedDict()
                data_type = data['data_types'][metric]

                if dimensions_len:
                    for row in results.get('rows', []):
                        column = i + dimensions_len
                        label = ','.join(row[:dimensions_len]) 
                        value = cast_data_type(row[column], data_type)

                        data['data'][metric][label] = value 

                data['data'][metric]['total'] = cast_data_type(results['totalsForAllResults'][metric], data_type)

                # Prevent rate-limiting
                sleep(1)

            output['queries'].append(data)

        return output
135081
def lazy_result(f):
    """"""
    @wraps(f)
    def decorated(ctx, param, value):
        return LocalProxy(lambda: f(ctx, param, value))
    return decorated
1031383
def _find(api_method, query, limit, return_handler, first_page_size, **kwargs):
    ''' 
    '''
    num_results = 0

    if "limit" not in query:
        query["limit"] = first_page_size

    while True:
        resp = api_method(query, **kwargs)

        by_parent = resp.get('byParent')
        descriptions = resp.get('describe')
        def format_result(result):
            if return_handler:
                result = dxpy.get_handler(result['id'], project=result.get('project'))
            if by_parent is not None:
                return result, by_parent, descriptions
            else:
                return result

        for i in resp["results"]:
            if num_results == limit:
                return
            num_results += 1
            yield format_result(i)

        # set up next query
        if resp["next"] is not None:
            query["starting"] = resp["next"]
            query["limit"] = min(query["limit"]*2, 1000)
        else:
            return
858171
def run_object_query(client, base_object_query, start_record, limit_to,
                     verbose=False):
    """"""
    if verbose:
        print("[start: %d limit: %d]" % (start_record, limit_to))
    start = datetime.datetime.now()
    result = client.execute_object_query(
        object_query=base_object_query,
        start_record=start_record,
        limit_to=limit_to)
    end = datetime.datetime.now()
    if verbose:
        print("[%s - %s]" % (start, end))
    return result
738529
def _query(self, query_type, query_str, verbose=False):
        
        # First check query cache
        cached = self.query_cache.get(query_str)
        if cached:
            if verbose:
                print('Returning Cached VT Query Results')
            return cached

        # Not in cache so make the actual query
        if query_type == 'file':
            response = requests.get('https://www.virustotal.com/vtapi/v2/file/report',
                                    params={'apikey': self.apikey, 'resource': query_str, 'allinfo': 1})
        else:
            response = requests.post('https://www.virustotal.com/vtapi/v2/url/report',
                                     params={'apikey': self.apikey, 'resource': query_str, 'allinfo': 1})

        # Make sure we got a json blob back
        try:
            vt_output = response.json()
        except ValueError:
            error_msg = 'VirusTotal no valid response, throttling and trying again...'
            if self.throttle:
                if verbose:
                    print(error_msg)
                time.sleep(30)
                return self._query(query_type, query_str)

            return {'vt_error': error_msg}

        # Check for not-found
        if not vt_output or vt_output['response_code'] == 0:
            output = {'query': query_str, 'not_found': True}
            self.query_cache.set(query_str, output)
            return output

        # Exclude some fields (if summary=True)
        output = {field: vt_output[field] for field in vt_output.keys() if field not in self.exclude}

        # Put the file sha in the output
        output['query'] = query_str

        # Organize the scans fields
        scan_results = collections.Counter()
        for scan in vt_output['scans'].values():
            if 'result' in scan:
                if scan['result']:
                    scan_results[scan['result']] += 1
        output['scan_results'] = scan_results.most_common(5)

        # Pull results in Cache
        self.query_cache.set(query_str, output)

        # Return results
        return output
589963
def _iterate_through_pages(get_function, resource, **kwargs):
        

        request_data = True
        data = []
        page = 1

        while request_data:
            temp_response = get_function(page=page, **kwargs)
            if temp_response[resource]['@total'] != '0':
                data.append(temp_response)

            page += 1
            if page > int(
                math.ceil(
                    float(temp_response[resource]['@total']) / float(temp_response[resource]['@per_page'])
                )
            ):
                request_data = False
        return data
535587
def _makeResult(self):
        
        return [reporter(self.stream, self.descriptions, self.verbosity) for reporter in self.resultclass]
508666
def page_models(self, constructor, paging, constraints=None, *, columns=None, order_by=None):
    """"""
    records, count = self.page(constructor.table_name, paging, constraints, columns=columns,
                               order_by=order_by)
    return ([constructor(r) for r in records], count)