fetches page and loads html into object before call parse
493715
function(callback){
    var options = this.options,
        cache = this.options.cache,
        logger =  this.options.logger;


    if(this.url){
      logger.log('fetch page: ' + this.url);

      // if there is a cache and it holds the url
      if(cache &&cache.has(this.url)){
        // http status - content located elsewhere, retrieve from there
        this.statusCode = 305;
        this.requestTime = 0;
        this.html = cache.get(this.url);
        logger.log('fetched html from cache: ' + this.url);

        this.parse(function(){
          callback(this);
        })

      // if not get the html from the url  
      }else{

        this.startedRequest = new Date();
        var self = this;

        var requestObj = {
          uri: this.url,
          headers: options.httpHeaders
        }

        request(requestObj, function(requestErrors, response, body){
          if(!requestErrors && response.statusCode === 200){

            self.endedRequest = new Date();
            self.requestTime = self.endedRequest.getTime() - self.startedRequest.getTime();
            logger.log('fetched html from page: ' 
              + self.requestTime + 'ms - ' + self.url);
            
            self.statusCode = response.statusCode;
            self.html = body;

            if(cache){
              cache.set(self.url, body);
            }

            self.parse(function(){
              self.status = "fetched";
              callback(self);
            })

          }else{
            logger.warn('error requesting page: ' + self.url);
            self.error = 'error requesting page: ' + self.url;
            self.status = "errored";
            callback();
          }
        });


      }
    }else{
      logger.warn('no url given');
      this.error = 'no url given';
      this.status = "errored";
      callback();
    }
  }
48448
def _parse_result_page(self, url, payload, only_region=False):
        
        data = []
        try:

            if only_region:
                html = self.scraper._get_html(url)
            else:
                html = self.scraper._post_html(url, payload=payload)

        except RequestException500:

            self.scraper.log.warning(u"Status code 500 on {} with {}".format(url, payload))
            return None


        current_selection = self._get_current_selection(html)

        table = Datatable(html)
        data = []
        for row in table.data:
            region_or_unit_id, region_or_unit_label = row["region_or_unit"]
            if region_or_unit_label in self.regions:
                row["region"] = region_or_unit_label
                row["unit"] = None
            else:
                row["region"] = None
                row["unit"] = region_or_unit_label

            value = row["value"]

            row.pop("value", None)
            row.pop("region_or_unit", None)

            for dim in self.dimensions:
                if dim.id not in row:
                    row[dim.id] = current_selection[dim.id][1] # gets label



            data.append(Result(value, row))
        return data
48144
function parse_html() {
      var reading_js = false, reading_css = false;
      var parser = new htmlparser.Parser({
        onopentag: function(name, attribs){
          if (name === "script" && ! attribs.src && (attribs.type === "text/javascript" || ! attribs.type)){
            reading_js = true;
          }
          else if (name === "style") {
            reading_css = true;
          }
        },
        ontext: function(text){
          if (reading_js) {
            scripts.push(text);
          }
          if (reading_css) {
            styles.push(text);
          }
        },
        onclosetag: function(tagname){
          if (tagname === "script"){
            reading_js = false;
          }
          else if (tagname === "style"){
            reading_css = false;
          }
        }
      });
      parser.write(src);
      parser.end();

      return true;
    }
330003
def parse(page_to_parse):
    """"""
    global _parsed
    if not isinstance(page_to_parse, page.Page):
        raise TypeError("parser.parse requires a parker.Page object.")

    if page_to_parse.content is None:
        raise ValueError("parser.parse requires a fetched parker.Page object.")

    try:
        parsed = _parsed[page_to_parse]
    except KeyError:
        parsed = parsedpage.ParsedPage(
            page=page_to_parse,
            parsed=PyQuery(page_to_parse.content, parser='html')
        )
        _parsed[page_to_parse] = parsed

    return parsed
1315873
function load(path, callback, errorCallback) {
  try {
    var xhr = new XMLHttpRequest();
    xhr.open("GET", path, true);
    xhr.onreadystatechange = function() {
      if (xhr.readyState === 4) {
        if (xhr.status === 200 || xhr.status === 304) {
          try {
            var response = JSON.parse(xhr.responseText);
            callback(response);
          } catch (e) {
            errorCallback(e);
          }
        } else {
          errorCallback(xhr);
        }
      }
    };
    xhr.setRequestHeader("Content-Type", "application/json");
    xhr.setRequestHeader("X-Requested-With", "XMLHttpRequest");
    xhr.send();
  } catch (e) {
    errorCallback(e);
  }
}
327287
function whenReady(funcToExecute) {
    if (isReady()) {
        logger.info('Page is already loaded, instantly executing!');
        funcToExecute();
        return;
    }

    logger.info('Waiting for page to be ready');
    callbacksOnReady.push(funcToExecute);
}
516671
function queryParser( req, res, next ) {
	if ( !req.query ) {
		var val = parseUrl( req ).query;
		req.query = queryparse( val );
	}
	next();
}
255268
async function _html(req, res) {
  const { queueName, queueHost, state } = req.params;
  const {Queues} = req.app.locals;
  const queue = await Queues.get(queueName, queueHost);
  const basePath = req.baseUrl;
  if (!queue) return res.status(404).render('dashboard/templates/queueNotFound', {basePath, queueName, queueHost});

  if (!isValidState(state, queue.IS_BEE)) return res.status(400).json({ message: `Invalid state requested: ${state}` });

  let jobCounts;
  if (queue.IS_BEE) {
    jobCounts = await queue.checkHealth();
    delete jobCounts.newestJob;
  } else {
    jobCounts = await queue.getJobCounts();
  }

  const page = parseInt(req.query.page, 10) || 1;
  const pageSize = parseInt(req.query.pageSize, 10) || 100;

  const startId = (page - 1) * pageSize;
  const endId = startId + pageSize - 1;

  let jobs;
  if (queue.IS_BEE) {
    const page = {};

    if (['failed', 'succeeded'].includes(state)) {
      page.size = pageSize;
    } else {
      page.start = startId;
      page.end = endId;
    }

    jobs = await queue.getJobs(state, page);

    // Filter out Bee jobs that have already been removed by the time the promise resolves
    jobs = jobs.filter((job) => job);
  } else {
    jobs = await queue[`get${_.capitalize(state)}`](startId, endId);
  }

  let pages = _.range(page - 6, page + 7)
    .filter((page) => page >= 1);
  while (pages.length < 12) {
    pages.push(_.last(pages) + 1);
  }
  pages = pages.filter((page) => page <= _.ceil(jobCounts[state] / pageSize));

  return res.render('dashboard/templates/queueJobsByState', {
    basePath,
    queueName,
    queueHost,
    state,
    jobs,
    jobsInStateCount: jobCounts[state],
    disablePagination: queue.IS_BEE && (state === 'succeeded' || state === 'failed'),
    currentPage: page,
    pages,
    pageSize,
    lastPage: _.last(pages)
  });
}
472036
function renderPage(uri, req, res, hrStart) {
  const locals = res.locals,
    extension = getExtension(req),
    renderer = findRenderer(extension);

  // Add request route params, request query params
  // and the request extension to the locals object
  locals.params = req.params;
  locals.query = req.query;
  locals.extension = extension;

  // look up page alias' component instance
  return getDBObject(uri)
    .then(formDataFromLayout(locals, uri))
    .then(data => {
      logTime(hrStart, uri);
      return data;
    })
    .then(({ data, options }) => renderer.render(data, options, res))
    .catch(responses.handleError(res));
}
53083
def loads(s):
    """"""
    _decoder = json.JSONDecoder()

    while s:
        s = s.strip()
        obj, pos = _decoder.raw_decode(s)
        if not pos:
            raise ValueError('no JSON object found at %i' % pos)
        yield obj
        s = s[pos:]
86550
def __load(self, redirect=True, preload=False):
    '''
    
    '''
    query_params = {
      'prop': 'info|pageprops',
      'inprop': 'url',
      'ppprop': 'disambiguation',
      'redirects': '',
    }
    if not getattr(self, 'pageid', None):
      query_params['titles'] = self.title
    else:
      query_params['pageids'] = self.pageid

    request = _wiki_request(query_params)

    query = request['query']
    pageid = list(query['pages'].keys())[0]
    page = query['pages'][pageid]

    # missing is present if the page is missing
    if 'missing' in page:
      if hasattr(self, 'title'):
        raise PageError(self.title)
      else:
        raise PageError(pageid=self.pageid)

    # same thing for redirect, except it shows up in query instead of page for
    # whatever silly reason
    elif 'redirects' in query:
      if redirect:
        redirects = query['redirects'][0]

        if 'normalized' in query:
          normalized = query['normalized'][0]
          assert normalized['from'] == self.title, ODD_ERROR_MESSAGE

          from_title = normalized['to']

        else:
          from_title = self.title

        assert redirects['from'] == from_title, ODD_ERROR_MESSAGE

        # change the title and reload the whole object
        self.__init__(redirects['to'], redirect=redirect, preload=preload)

      else:
        raise RedirectError(getattr(self, 'title', page['title']))

    # since we only asked for disambiguation in ppprop,
    # if a pageprop is returned,
    # then the page must be a disambiguation page
    elif 'pageprops' in page:
      query_params = {
        'prop': 'revisions',
        'rvprop': 'content',
        'rvparse': '',
        'rvlimit': 1
      }
      if hasattr(self, 'pageid'):
        query_params['pageids'] = self.pageid
      else:
        query_params['titles'] = self.title
      request = _wiki_request(query_params)
      html = request['query']['pages'][pageid]['revisions'][0]['*']

      lis = BeautifulSoup(html, 'html.parser').find_all('li')
      filtered_lis = [li for li in lis if not 'tocsection' in ''.join(li.get('class', []))]
      may_refer_to = [li.a.get_text() for li in filtered_lis if li.a]

      raise DisambiguationError(getattr(self, 'title', page['title']), may_refer_to)

    else:
      self.pageid = pageid
      self.title = page['title']
      self.url = page['fullurl']
529848
function fetch(params) {
    let fetchObj = null;
    if (/^http(s{0,1}):\/\//.test(params.url)) {
        const pageLoc = location.href.match(/^https{0,1}:\/\/(.+?)(?=\/|$)/);
        const ajaxLoc = params.url.match(/^https{0,1}:\/\/(.+?)(?=\/|$)/);
        if (pageLoc.length === 2 && ajaxLoc.length === 2 && ajaxLoc[1] === pageLoc[1]) {
            fetchObj = this.ajax(params);
        } else {
            fetchObj = this.jsonp(params);
        }
    } else {
        fetchObj = this.ajax(params);
    }
    return fetchObj;
}
199886
def fetch_next_page(self):
        """"""
        result = self.request_handler.get(url=self.next_page_url).json()
        self.__init__(self.request_handler, result,
                      self.data_type, self.automatic_pagination)
1270142
function(url) {
		return new Promise(function (resolve) {
			self.roptions.url = url;
			request(self.roptions, function (err, response, body) {
				if (!err && response.statusCode === 200) {
					var $ = cheerio.load(body);

					// no results
					if (!$('.result').length) {
						return resolve(result);
					}
					
					$('.result').each(function() {
						var href = $(this).find('a').attr('href');
						if(href.toString().indexOf("http") !== 0) {
							result.push(base + href); 							
						} else {
							result.push(href); 							
						}
					});

					if (num && result.length > num) {
						return resolve(result.slice(0,num));
					}

					// search for current page and select next one
					var page = $('#nextPage').attr('href');
					if (page) {
						resolve(nextPage(base + page));
					}
					else {
						resolve(result);
					}
				} else {
					resolve(result)
				}
			});
		});
	}
11617
def parse_json(self, page):
        ''''''
        if not isinstance(page, basestring):
            page = util.decode_page(page)
            
        self.doc = json.loads(page)
        results = self.doc.get(self.result_name, [])
    
        if not results:
            self.check_status(self.doc.get('status'))
            return None
            
            
        return results
1246036
function fetchResults(searchText) {
      var items = $scope.$parent.$eval(self.remoteMethod),
        isList = angular.isArray(items),
        isPromise = items ? !!items.then : false; // Every promise should contain a `then` property

      if (isList) onResultsRetrieved(items);
      else if (isPromise) handleAsyncResults(items);

      function handleAsyncResults(items) {
        if (!items) return;

        items = $q.when(items);
        setLoading(true);
        items.then(onResultsRetrieved);
      }

      function onResultsRetrieved(matches) {

        // Just cache the results if the request is now outdated.
        // The request becomes outdated, when the new searchText has changed during the result fetching.
        if ((searchText || '') !== (self.searchText || '')) {
          return;
        }

        handleResults(matches);
      }
    }
453636
function (req, res, parseObj) {
    var data = {};
    var body = req.body;
    var reqID = body.id || req.query.id;
    var reqSections = body.sections || req.query.sections;
    var sections = reqSections ? reqSections.split(',') : undefined;
    var parsedData = parseObj;

    var msgDataNotFound = 'API: HTML data not found, please sync API or run PhantomJS parser.';

    if (reqID) {
        var responseData = '';

        if (reqSections) {
            responseData = parsedData.getBySection(reqID, sections);
        } else {
            responseData = parsedData.getByID(reqID);
        }

        if (responseData && typeof responseData === 'object') {
            res.status(config.statusCodes.OK).json(responseData);
        } else {
            if (typeof responseData === 'undefined') console.warn(msgDataNotFound);

            res.status(config.statusCodes.notFound).json({
                message: "id and requested sections not found"
            });
        }
    } else {
        data = parsedData.getAll();

        if (data) {
            res.status(config.statusCodes.OK).json(data);
        } else {
            console.warn(msgDataNotFound);

            res.status(config.statusCodes.notFound).json({
                message: "data not found"
            });
        }
    }
}