Handle GitHub pagination
@param  {Object} reqOptions - Options to pass to the request. Url will be overidden
@param  {string} link - Link header
@param  {Object[]} memo - Aggregator of result
@return {Promise} a Promise of aggregated result
@private
484967
function paginate(reqOptions, link, memo) {
	const links = link.split(/,\s*/);
	let matches;
	links.some(function(link) {
		matches = link.trim().match(/<(.+)>;\s*rel="next"/);
		return matches !== null;
	});
	// End of pagination
	if(!matches) {
		return Promise.resolve(memo);
	}
	return new Promise(function(resolve, reject) {
		reqOptions.url = matches[1];
		request(reqOptions, function(err, res, body) {
			paginate(reqOptions, res.headers.link, memo.concat(JSON.parse(body))).then(resolve);
		});
	});
}
79862
function github(url, destination) {
  return new Promise((resolve, reject) => {
    (0, _downloadGithubRepo2.default)(url, destination, err => {
      if (err) {
        reject(err);
      } else {
        resolve();
      }
    });
  });
}
7202
function(url, branch) {
  this.url = url;
  if (url.startsWith('https://api.github.com/repos/')) {
    this.path = url.substring(29);
    this.apiUrl = url;
  }
  else if (url.startsWith('https://github.com/')) {
    this.path = url.substring(19);
    this.apiUrl = 'https://api.github.com/repos/' + this.path;
  }
  else {
    throw new Error('Unsupported URL format: ' + url);
  }
  
  this.branch = branch ? branch : 'master';
  
  if (typeof window === 'undefined') {
    this.request = require('request');
    this.requestOptions = {headers: {'User-Agent': 'limberest'}};
  } 
  else {
    this.request = require('browser-request');
    this.requestOptions = {};
  }
}
444331
function buildRoutes(name, mutator, mutatorOptions) {
	var basePath = '/' + name

	// POST - run the content through the mutator and return a revisit.link compatible object
	plugin.route({
		method: 'POST',
		path: basePath + '/service',
		handler: mutatorOptions.rawPayload ? postHandlerRawPayload : postHandler,
		config: {
			validate: {
				payload: revisitorSchema
			},
			response: {
				schema: revisitorSchema
			}
		}
	})

	function postHandler(request, reply) {
		var imgBuf = dataUriToBuffer(request.payload.content.data),
			imgType = imgBuf.type

		mutator(imgBuf, function(err, mutatedBuffer) {
			if (err) {
				return reply(err).code(400)
			}

			var resp = {
				content: {
					data: 'data:image/' + imgType + ';base64,' + mutatedBuffer.toString('base64')
				},
				meta: request.payload.meta
			}

			reply(resp).code(200)
		})
	}

	function postHandlerRawPayload(request, reply) {
		var rawPayload = request.payload

		mutator(rawPayload, function(err, rawPayloadReply) {
			if (err) {
				return reply(err).code(400)
			}

			reply(rawPayloadReply).code(200)
		})
	}
}
1343829
function golr_callback_action(resp){

	    // Return caches for the values we'll collect.
	    var ret_terms = [];
	    var ret_descs = [];
	    var ret_links = [];

	    // Gather document info if available.
	    //var found_docs = resp.documents();		
	    bbop.core.each(resp.documents(),
			   function(doc){
			       var id = doc[id_field];
			       var label = doc[label_field];

			       ret_terms.push(label);
			       ret_descs.push(id);
			       ret_links.push(linker.url(id, link_type));
			   });

	    // Assemble final answer into the OpenSearch JSON
	    // form.
	    var ret_body = [query];
	    ret_body.push(ret_terms);
	    ret_body.push(ret_descs);
	    ret_body.push(ret_links);
	    var ans = {
		status: 200,
		headers: {'Content-Type': 'application/json'},
		body: [bbop.core.to_string(ret_body)]
	    };
	    response.resolve(ans);
	}
307845
function createPagination(req) {
  return function createPagination(pages, page, liClass, anchorClass) {
    var params = qs.parse(url.parse(req.url).query);
    var str = '';

    params.page = 1;
    var clas = (page == 1 && 'active') || '';

    for (var p = 1; p <= pages; p++) {
      params.page = p;
      clas = (page == p && 'active') || '';

      var href = '?' + qs.stringify(params);

      str += `<li class="${liClass} ${clas}"><a class="${anchorClass}" href="${href}">${p}</a></li>`;
    }

    return str;
  };
}
31688
function(url, method = 'get', body = undefined) {
		var deferred = Q.defer();
		let self = this;

		try {
			let req = undefined;

			// Create request object
			switch (method.toLowerCase().trim()) {
				case 'post': {
					req = request.post(url).send(body);
					break;
				}
				case 'patch': {
					req = request.patch(url).send(body);
					break;
				}
				case 'put': {
					req = request.put(url).send(body);
					break;
				}
				case 'delete': {
					req = request.delete(url);
					break;
				}
				case 'get': {
					req = request.get(url);
					break;
				}
				default: {
					throw new Error(`Unsupported HTTP verb: ${method}`);
				}
			}

			self.requestCount++;

			// Set standard stuff and handle response
			req
				.set('Authorization', 'token ' + config.token)
				.then(function(res) {
					self.logRequestSuccess(res);
					deferred.resolve({
						body: res.body,
						headers: res.headers
					});
				}, 
				function(err) {
					// TODO: handle rate limiting (github sends a 403, not a 429)
					// https://developer.github.com/v3/#rate-limiting
					log.error(err);
					deferred.reject(err);
				});
		} catch(err) {
			log.error(err);
			deferred.reject(err);
		}

		return deferred.promise;
	}
1091135
function(args, connectionInfo, processPagedResults) {
  // args.consignmentIds.value MUST already be set
  args.page = {value: 1};
  args.pageSize = {value: 200};
  args.consignmentIdIndex = {value: 0};
  args.consignmentId = {value: args.consignmentIds.value[args.consignmentIdIndex.value]};
  args.getArray = function(){
    return this.consignmentIds.value;
  };
  args.getArrayIndex = function(){
    return this.consignmentIdIndex.value;
  };

  // iterate serially through a promise chain for all args.consignmentIds.value
  return utils.processPromisesSerially(
    args.consignmentIds.value,
    args.consignmentIdIndex.value,
    args,
    function mergeStrategy(newData, previousData){
      log.debug('inside mergeStrategy()');
      //log.silly('newData ', newData);
      //log.silly('previousData ', previousData);
      if (previousData && previousData.length>0) {
        if (newData.length>0) {
          log.debug('previousData.length: ', previousData.length);
          newData = newData.concat(previousData);
          log.debug('combinedData.length: ', newData.length);
        }
        else {
          newData = previousData;
        }
      }
      //log.silly('finalData ', newData);
      log.debug('finalData.length ', newData.length);
      return Promise.resolve(newData); // why do we need a promise?
    },
    function setupNext(updateArgs){
      updateArgs.consignmentIdIndex.value = updateArgs.consignmentIdIndex.value + 1;
      if (updateArgs.consignmentIdIndex.value < updateArgs.consignmentIds.value.length) {
        updateArgs.consignmentId.value = updateArgs.consignmentIds.value[updateArgs.consignmentIdIndex.value];
        log.debug('next is consignmentId: ' + updateArgs.consignmentId.value);
      }
      else {
        updateArgs.consignmentId.value = null;
        log.debug('finished iterating through all the consignmentIds');
      }
      return updateArgs;
    },
    function executeNext(updatedArgs){
      log.debug('executing for consignmentId: ' + updatedArgs.consignmentId.value);
      //log.silly('updatedArgs: ', updatedArgs);
      return fetchAllProductsByConsignment(updatedArgs, connectionInfo, processPagedResults);
    }
  );
}
1020695
function fetchWithRetry(url, options) {
    async function fetchWrapper() {
        const response = await fetch(url, options);
        // don't retry if status is 4xx
        if (response.status >= 400 && response.status < 500) {
            const contentType = response.headers.get('content-type');
            if (contentType && contentType.includes('application/json')) {
                const body = await response.json();
                throw new p_retry_1.default.AbortError(body.message);
            }
            else {
                throw new p_retry_1.default.AbortError(response.statusText);
            }
        }
        return response;
    }
    return p_retry_1.default(fetchWrapper, { retries: RETRY_COUNT });
}
830480
function queue(options, callback) {


        // if skipDuplicates, don't crawl twice the same url
        if (options.skipDuplicates) {
            store.getStore().checkInCrawlHistory(options.url, function(error, isInCrawlHistory) {

              if (isInCrawlHistory) {
                  log.debug({"url" : options.url, "step" : "queue-resquester.queue", "message" :  "Don't crawl this url - Option skipDuplicates=true & the url has already been crawled" });
                  callback();
              }
              else {

                  //numOfUrl++
                  //console.log("Queue length : " + requestQueue.length() + " - Running : " + requestQueue.running() + " - Total Add :" + numOfUrl + " - Total End : " + endUrl + " url : " + options.url );
                  log.debug({"url" : options.url, "step" : "queue-resquester.queue", "message" : "Add in the request queue"});
                  requestQueue.push(options);
                  callback();

              }

            });

        }
        else {
            //numOfUrl++
            //console.log("Queue length : " + requestQueue.length() + " - Running : " + requestQueue.running() + " - Total Add :" + numOfUrl + " - Total End : " + endUrl + " url : " + options.url );

            log.info({"url" : options.url, "step" : "queue-resquester.queue", "message" : "Add in the request queue"});
            requestQueue.push(options);
            callback();
        }

  }
1099185
function pagedGet ({source, method, skip = 0, aggregatedResponse = null, query = null}) {
  const fullQuery = Object.assign({},
    {
      skip: skip,
      limit: pageLimit,
      order: 'sys.createdAt,sys.id'
    },
    query
  )

  return source[method](fullQuery)
    .then((response) => {
      if (!aggregatedResponse) {
        aggregatedResponse = response
      } else {
        aggregatedResponse.items = aggregatedResponse.items.concat(response.items)
      }
      const page = Math.ceil(skip / pageLimit) + 1
      const pages = Math.ceil(response.total / pageLimit)
      logEmitter.emit('info', `Fetched ${aggregatedResponse.items.length} of ${response.total} items (Page ${page}/${pages})`)
      if (skip + pageLimit <= response.total) {
        return pagedGet({source, method, skip: skip + pageLimit, aggregatedResponse, query})
      }
      return aggregatedResponse
    })
}
446488
function doit(originalArgs, originalSettings, jqOptions, bjOptions) {
    var xhr;
    var p = new Promise(function resolver(resolve, reject) {
      xhr = bluetry.perform.call(this, originalSettings, jqOptions, bjOptions);
      function succeded(data, textStatus, jqXHR) {
        resolve(bjOptions.verboseResults ? [data, textStatus, jqXHR] : data);
      }

      function failed(jqXHR, textStatus, errorThrown) {
        var error = makeError(
          jqXHR, textStatus, errorThrown,
          bjOptions.verboseExceptions ? originalArgs : null);

        if (!isNetworkIssue(error)) {
          // As mentioned earlier, errors that are not due to the network cause
          // an immediate rejection: no diagnosis.
          reject(error);
        }
        else {
          // Move to perhaps diagnosing what could be the problem.
          var diagnose = bjOptions.diagnose;
          if (!diagnose || !diagnose.on) {
            // The user did not request diagnosis: fail now.
            reject(error);
          }
          else {
            // Otherwise, we perform the requested diagnosis.  We cannot just
            // call ``reject`` with the return value of ``diagnoseIt``, as the
            // rejection value would be a promise and not an error. (``resolve``
            // assimilates promises, ``reject`` does not).
            resolve(diagnoseIt(error, diagnose));
          }
        }
      }

      xhr.fail(failed).done(succeded);
    });

    return { xhr: xhr, promise: p };
  }
677651
def add_transition(rel, href, options={})
      new_representor_hash = RepresentorHash.new(deep_dup(@representor_hash.to_h))
      options = symbolize_keys(options)
      options.delete(:method) if options[:method] == Transition::DEFAULT_METHOD
      link_values = options.merge({href: href, rel: rel})

      if options[DATA_KEY]
        link_values[Transition::DESCRIPTORS_KEY] = link_values.delete(DATA_KEY)
      end

      new_representor_hash.transitions.push(link_values)
      RepresentorBuilder.new(new_representor_hash)
    end
34298
function prepareLinks(response, rootUrl, queryPath) {
  var selfLink = { "href": rootUrl + queryPath };
  response._links = {};
  response._links.self = selfLink;
}
279764
function getQueuedLink(callback, seconds) {
  var dateField = 'queued.lastAttempt';
  seconds = seconds || DELAY_SECONDS;
  // function to process the link
  var process = function(err, res) {
    var queuedLink = null;
    if (res && res.hits && res.hits.hits.length) {
      var hit = res.hits.hits[0]._source;
      var queued = hit.queued;
      queuedLink = { uri: hit.uri, queued: queued, total: res.hits.total };
      queued.lastAttempt = new Date().toISOString();
      queued.attempts = queued.attempts + 1;
      hit.queued = queued;

      // update accesses
      contentLib.indexContentItem(hit, {state: 'queued', member: queued.member}, function(err, res) {
        // process the link
        if (!err) {
          callback(null, queuedLink);
        } else {
          utils.passingError(err);
        }
      });
    } else {
      callback(err, null);
    }
  };

 // call query with the function
  GLOBAL.svc.indexer.formQuery({query: {
    terms: dateField + ':<now-' + seconds + 's AND state:queued AND queued.attempts:<' + MAX_ATTEMPTS
  }, sort: dateField, sourceFields : ['uri', 'title', 'queued.*'], size : 1 }, process);
}
847629
function purgeADQueue (channel, connectionName, options, messages) {
  const name = options.uniqueName || options.name;
  return new Promise(function (resolve, reject) {
    const messageCount = messages.messages.length;
    if (messageCount > 0) {
      log.info(`Purge operation for queue '${options.name}' on '${connectionName}' is waiting for resolution on ${messageCount} messages`);
      messages.once('empty', function () {
        channel.purgeQueue(name)
          .then(
            result => resolve(result.messageCount),
            reject
          );
      });
    } else {
      channel.purgeQueue(name)
        .then(
          result => resolve(result.messageCount),
          reject
        );
    }
  });
}
698519
function addLocation (state) {
  state.results.forEach(endpoint => {
    const isQueryRequest = ['GET', 'HEAD'].includes(endpoint.method)
    const urlParameterNames = (endpoint.path.match(/:\w+/g) || []).map(name => name.substr(1))
    endpoint.params.forEach(result => {
      if (urlParameterNames.includes(result.name)) {
        result.location = 'url'
        return
      }

      // Only few endpoints define request headers
      // - https://developer.github.com/v3/repos/releases/#upload-a-release-asset (Content-Type, Content-Length)
      // - https://developer.github.com/v3/markdown/#render-a-markdown-document-in-raw-mode (Content-Type)
      if (['Content-Type', 'Content-Length'].includes(result.name)) {
        result.location = 'headers'
        return
      }

      // Some endpoints have incorrect pagination headers, this is being worked on by GitHub.
      // e.g. if https://developer.github.com/v3/projects/#create-a-repository-project no longer
      // shows Link: <https://api.github.com/resource?page=2>; rel="next", in theresponse,
      // this can be removed
      if (['page', 'per_page'].includes(result.name)) {
        result.location = 'query'
        return
      }

      result.location = isQueryRequest ? 'query' : 'body'
    })
  })
}