Handle truncated responses

    :param callback:
    :param params:
    :param entities:

    :return:
545652
def handle_truncated_response(callback, params, entities):
    
    results = {}
    for entity in entities:
        results[entity] = []
    while True:
        try:
            marker_found = False
            response = callback(**params)
            for entity in entities:
                if entity in response:
                    results[entity] = results[entity] + response[entity]
            for marker_name in ['NextToken', 'Marker', 'PaginationToken']:
                if marker_name in response and response[marker_name]:
                    params[marker_name] = response[marker_name]
                    marker_found = True
            if not marker_found:
                break
        except Exception as e:
            if is_throttled(e):
                time.sleep(1)
            else:
                raise e
    return results
360272
function parseAnnotations(response) {
  let entities = {};
  Object.keys(response).forEach(key => {
    entities[key] = parseAnnotation(response[key]);
  });
  return entities;
}
273971
function (entities, options) {
        var retainEmpty = options && options.retainEmptyValues;

        _.forEach(entities, function (entity) {
            util.cleanEmptyValue(entity, 'description', retainEmpty);
        });

        return entities;
    }
1354014
def _server_end_response_callback(self, respoonse: Response):
        ''''''
        request = self._item_session.request
        response = self._item_session.response

        _logger.info(__(
            _('Fetched ‘{url}’: {status_code} {reason}. '
              'Length: {content_length} [{content_type}].'),
            url=request.url,
            status_code=response.status_code,
            reason=wpull.string.printable_str(response.reason),
            content_length=wpull.string.printable_str(
                response.fields.get('Content-Length', _('none'))),
            content_type=wpull.string.printable_str(
                response.fields.get('Content-Type', _('none'))),
        ))

        self._result_rule.handle_response(self._item_session)

        if response.status_code in WebProcessor.DOCUMENT_STATUS_CODES:
            filename = self._file_writer_session.save_document(response)
            self._processing_rule.scrape_document(self._item_session)
            self._result_rule.handle_document(self._item_session, filename)

        elif response.status_code in WebProcessor.NO_DOCUMENT_STATUS_CODES:
            self._file_writer_session.discard_document(response)
            self._result_rule.handle_no_document(self._item_session)
        else:
            self._file_writer_session.discard_document(response)
            self._result_rule.handle_document_error(self._item_session)
721247
def split_calls(func):
    
    def wrapper(*args, **kwargs):
        #The size limit is 8190 bytes minus url and api to call
        #For example (https://cmsweb-testbed.cern.ch:8443/dbs/prod/global/filechildren), so 192 bytes should be safe.
        size_limit = 8000
        encoded_url = urllib.urlencode(kwargs)
        if len(encoded_url) > size_limit:
            for key, value in kwargs.iteritems():
                ###only one (first) list at a time is splitted,
                ###currently only file lists are supported
                if key in ('logical_file_name', 'block_name', 'lumi_list', 'run_num') and isinstance(value, list):
                    ret_val = []
                    for splitted_param in list_parameter_splitting(data=dict(kwargs), #make a copy, since it is manipulated
                                                                   key=key,
                                                                   size_limit=size_limit):
                        try:
                            ret_val.extend(func(*args, **splitted_param))
                        except (TypeError, AttributeError):#update function call do not return lists
                            ret_val= []
                    return ret_val
            raise dbsClientException("Invalid input",
                                     "The lenght of the urlencoded parameters to API %s \
                                     is exceeding %s bytes and cannot be splitted." % (func.__name__, size_limit))
        else:
            return func(*args, **kwargs)
    return wrapper
1121033
function apiResponded(apiResponse) {
      // Convert XML to JS object
      var parser = new xml2js.Parser({explicitArray:false});
      parser.parseString(apiResponse, function (err, result) {
        // If the method processes the response, give it back
        // Otherwise call the original callback

        if(typeof a.processor !== 'undefined')
          a.processor(result.response, a.callback);
        else
          a.callback(result.response);
      });


    }
1276469
def parse_return_annotation(callback):
    ''''''
    # This has been implemented as a function as the parsing need to be
    # accessible from a decorator used on the function's class method
    responses = {}
    for data in callback.__annotations__['return']:
        tmp = dict(zip(('code', 'message', 'responseModel'), data))
        rm = tmp.get('responseModel', None)  # "None" can be omitted!
        tmp['responseModel'] = rm.name if rm else 'void'
        responses[tmp['code']] = tmp
    return responses
148311
@Override
  public void processResponse(CloseableHttpResponse response) throws IOException, UnexpectedResponseException {
    if (response.getStatusLine().getStatusCode() >= 400) {
      if (response.getEntity() != null) {
        throw new RuntimeException("Failed. " + EntityUtils.toString(response.getEntity())
                                 + " , response: " + ToStringBuilder.reflectionToString(response, ToStringStyle.SHORT_PREFIX_STYLE));
      }
      throw new RuntimeException("Failed. Response: " + ToStringBuilder.reflectionToString(response, ToStringStyle.SHORT_PREFIX_STYLE));
    }
  }
59481
function(err, res) {
        if (err) return callback(err);
        self._rawResponse = res;
        if (!res || !res['cps:reply']) return callback(new Error("Invalid reply"));
        self.seconds = res['cps:reply']['cps:seconds'];
        if (res['cps:reply']['cps:content']) {
            var content = res['cps:reply']['cps:content'];
            self = _.merge(self, content);
        }

        callback(res['cps:reply']['cps:error'], self);
    }
1009336
function (entity, options, callback) {
                if (entity instanceof Entity) {
                    entity.validate(options, callback);
                } else {
                    callback("parameter is not an entity");
                }
            }
2690
private void processBatchRequestResponse(CloseableHttpResponse response) throws IOException,
      UnexpectedResponseException {
    String entityStr = EntityUtils.toString(response.getEntity());
    int statusCode = response.getStatusLine().getStatusCode();
    if (statusCode >= 400) {
      throw new RuntimeException("Failed due to " + entityStr + " (Detail: "
          + ToStringBuilder.reflectionToString(response, ToStringStyle.SHORT_PREFIX_STYLE) + " )");
    }

    JsonObject jsonBody = new JsonParser().parse(entityStr).getAsJsonObject();
    if (!jsonBody.get("hasErrors").getAsBoolean()) {
      return;
    }

    JsonArray results = jsonBody.get("results").getAsJsonArray();
    for (JsonElement jsonElem : results) {
      JsonObject json = jsonElem.getAsJsonObject();
      int subStatusCode = json.get("statusCode").getAsInt();
      if (subStatusCode < 400) {
        continue;
      } else if (subStatusCode == 400
                 && Operation.INSERT_ONLY_NOT_EXIST.equals(operation)) {
        JsonElement resultJsonElem = json.get("result");
        Preconditions.checkNotNull(resultJsonElem, "Error response should contain result property");
        JsonObject resultJsonObject = resultJsonElem.getAsJsonArray().get(0).getAsJsonObject();
        if (isDuplicate(resultJsonObject, subStatusCode)) {
          continue;
        }
      }
      throw new RuntimeException("Failed due to " + jsonBody + " (Detail: "
          + ToStringBuilder.reflectionToString(response, ToStringStyle.SHORT_PREFIX_STYLE) + " )");
    }
  }
295927
def handle_response(response, batch_id, fasta_mapper)
      if response.success?
        handle_success_response(response, batch_id, fasta_mapper)
      else
        handle_failed_response(response)
      end
    end
157716
def _handle_response(self, url, res, suppress_empty=True):
        
        self._last_response = res
        result = {}
        if res.status_code in (200, 201, 202):
            try:
                result = res.json()
            except Exception as e:  # json might be invalid, or store might be down
                e.message += " (_handle_response failed to decode JSON: " + str(res.content) + ")"
                raise  # TODO better exception
        elif res.status_code == 204 and not suppress_empty:
            raise EmptyResponseWarning("%d %s @ %s: %s" % (res.status_code, res.reason, url, res.content), res)
        elif res.status_code >= 500:
            raise ServerException("%d %s @ %s: %s" % (res.status_code, res.reason, url, res.content), res)
        elif res.status_code == 429:
            raise RateLimitingException("%d %s @ %s: %s" % (res.status_code, res.reason, url, res.content), res)
        elif res.status_code >= 400:
            raise ClientRequestException("%d %s @ %s: %s" % (res.status_code, res.reason, url, res.content), res)
        elif res.status_code >= 300:
            raise RedirectionException("%d %s @ %s: %s" % (res.status_code, res.reason, url, res.content), res)
        return result
983618
function handleUncompressed(res, _write, _end, callback) {
  let buffer = new BufferHelper();
  // Rewrite response method and get the content.
  res.write = data => buffer.concat(data);

  res.end = () => {
    let body;
    try {
      body = JSON.parse(buffer.toBuffer().toString());
    } catch (e) {
      body = buffer.toBuffer().toString();
      console.log('JSON.parse error:', e);
    }

    // Custom modified logic
    if (typeof callback === 'function') {
      body = callback(body);
    }

    let finish = _body => {
      // Converts the JSON to buffer.
      let body = new Buffer(JSON.stringify(_body));

      // Call the response method
      _write.call(res, body);
      _end.call(res);
    };

    if (body && body.then) {
      body.then(finish);
    } else {
      finish(body);
    }
  };
}
299413
function (isResponse, request, cb) {
                if (!this._filters) {
                    this.$callback(cb);
                    return;
                }
                var args = {
                    isResponse : isResponse,
                    request : request,
                    cb : cb,
                    filters : this._filters,
                    nbFilters : this._filters.length
                    // store the number of filters so that we do not call filters which were added after the call
                    // to _callFilters
                };
                if (this._filtersToBeLoaded > 0
                        && (request.sender == null || request.sender.classpath != "aria.core.FileLoader")) {
                    this._loadMissingFilters({
                        fn : this._continueCallingFilters,
                        args : args,
                        scope : this
                    });
                } else {
                    this._continueCallingFilters(null, args);
                }
            }
534452
function processResponse(err, res, cb){
  if(!err) {
    if(Array.isArray(res.body)){
      res.body.forEach(function (body) {
        castRecordDateFields(body);
      });
    } else if (_.isObject(res.body)) {
      castRecordDateFields(res.body);
    }
  }
  cb();
}
201413
function(callback)
    {
        if (this._limit != null && this.offset >= this._limit)
        {
            return callback(null, null);
        }

        var self = this;
        var _next = function(err)
        {       
            if (err)
            {
                return callback(err);
            }
        
            if (self.pos < self.data.length)
            {
                var itemData = self.data[self.pos];
                self.pos++;
                self.offset++;
                var cls = self.itemCls;
                callback(null, new cls(self.api, itemData, true));
            }
            else
            {
                callback(null, null);
            }
        };
        
        if (this.data == null || this.pos >= this.data.length && this.truncated)
        {
            this.loadNextPage(_next);
        }
        else
        {
            process.nextTick(_next);
        }
    }