Fetch the html content from the API
@param url
@param args
@param omitScript
@param maxWidth
@param hideMedia
@param hideThread
@param align
@param lang
@param theme
@param linkColor
@param widgetType
@returns {Promise.<*>}
556311
async function _process(
  args,
  { fetch },
  {
    _omitScript,
    maxWidth,
    hideMedia,
    hideThread,
    align,
    lang,
    theme,
    linkColor,
    widgetType
  }
) {
  const params = {
    url: args[0],
    omitScript: _omitScript,
    maxWidth,
    hideMedia,
    hideThread,
    align,
    lang,
    theme,
    linkColor,
    widgetType
  }
  try {
    const apiUrl = `https://api.twitter.com/1/statuses/oembed.json?${getQuery(
      params
    )}`
    const res = await (isBrowser ? jsonp : fetch)(apiUrl)
    return await res.json()
  } catch (e) {
    return {
      html: ""
    }
  }
}
1270142
function(url) {
		return new Promise(function (resolve) {
			self.roptions.url = url;
			request(self.roptions, function (err, response, body) {
				if (!err && response.statusCode === 200) {
					var $ = cheerio.load(body);

					// no results
					if (!$('.result').length) {
						return resolve(result);
					}
					
					$('.result').each(function() {
						var href = $(this).find('a').attr('href');
						if(href.toString().indexOf("http") !== 0) {
							result.push(base + href); 							
						} else {
							result.push(href); 							
						}
					});

					if (num && result.length > num) {
						return resolve(result.slice(0,num));
					}

					// search for current page and select next one
					var page = $('#nextPage').attr('href');
					if (page) {
						resolve(nextPage(base + page));
					}
					else {
						resolve(result);
					}
				} else {
					resolve(result)
				}
			});
		});
	}
1171160
function (url) {
    var req = new XMLHttpRequest();

    // Response handler. Check the console for errors.
    var respond = function () {
        if (req.status !== 200) {
          console.warn(req.responseText);
          return;
        }
        // Show the raw response text on page
        xmlOutput.textContent = req.responseText;
        // convert XML to javascript objects
        var json = sdmxmllib.mapSDMXMLResponse(req.responseText);
        // Convert to json and show on page
        jsonOutput.textContent = JSON.stringify(json, null, 2);
    };

    req.open('GET', url);
    // This will return latest version (SDMX-ML 2.1)
    req.setRequestHeader('Accept', 'application/xml');

    req.onreadystatechange = function () {
      if (req.readyState > 3) respond();
    };

    req.onerror = function (err) {
      console.warn(err);
    };

    req.send(null);
  }
40852
function analyzeHTML(result, $, callback) {

  // if $ is note defined, this is not a HTML page with an http status 200
  if (! $) {
    return callback();
  }

  log.debug({"url" : result.url, "step" : "analyzeHTML", "message" : "Start check HTML code"});

  async.parallel([

    async.apply(crawlHrefs, result, $),
    async.apply(crawlLinks, result, $),
    async.apply(crawlScripts, result, $),
    async.apply(crawlImages, result, $),

  ], callback);

}
1121439
public boolean renderURL(String urlstring, OutputStream out, Type type) throws IOException, SAXException
    {
        if (!urlstring.startsWith("http:") &&
            !urlstring.startsWith("https:") &&
            !urlstring.startsWith("ftp:") &&
            !urlstring.startsWith("file:"))
                urlstring = "http://" + urlstring;
        
        //Open the network connection 
        DocumentSource docSource = new DefaultDocumentSource(urlstring);
        
        //Parse the input document
        DOMSource parser = new DefaultDOMSource(docSource);
        Document doc = parser.parse();
        
        //create the media specification
        MediaSpec media = new MediaSpec(mediaType);
        media.setDimensions(windowSize.width, windowSize.height);
        media.setDeviceDimensions(windowSize.width, windowSize.height);

        //Create the CSS analyzer
        DOMAnalyzer da = new DOMAnalyzer(doc, docSource.getURL());
        da.setMediaSpec(media);
        da.attributesToStyles(); //convert the HTML presentation attributes to inline styles
        da.addStyleSheet(null, CSSNorm.stdStyleSheet(), DOMAnalyzer.Origin.AGENT); //use the standard style sheet
        da.addStyleSheet(null, CSSNorm.userStyleSheet(), DOMAnalyzer.Origin.AGENT); //use the additional style sheet
        da.addStyleSheet(null, CSSNorm.formsStyleSheet(), DOMAnalyzer.Origin.AGENT); //render form fields using css
        da.getStyleSheets(); //load the author style sheets
        
        BrowserCanvas contentCanvas = new BrowserCanvas(da.getRoot(), da, docSource.getURL());
        contentCanvas.setAutoMediaUpdate(false); //we have a correct media specification, do not update
        contentCanvas.getConfig().setClipViewport(cropWindow);
        contentCanvas.getConfig().setLoadImages(loadImages);
        contentCanvas.getConfig().setLoadBackgroundImages(loadBackgroundImages);

        if (type == Type.PNG)
        {
            contentCanvas.createLayout(windowSize);
            ImageIO.write(contentCanvas.getImage(), "png", out);
        }
        else if (type == Type.SVG)
        {
            setDefaultFonts(contentCanvas.getConfig());
            contentCanvas.createLayout(windowSize);
            Writer w = new OutputStreamWriter(out, "utf-8");
            writeSVG(contentCanvas.getViewport(), w);
            w.close();
        }
        
        docSource.close();

        return true;
    }
587593
function fetchView( data, id ) {

	data.backgroundColor = randomCol();
	data.fontColour		 = data.backgroundImage? '' : getContrastYIQ( data.backgroundColor );

	let wrapper 		= document.createElement('div');
	wrapper.className 	= 'section hidden';
	wrapper.innerHTML 	= SimpleViewTemplate( data );
	wrapper.dataset.id  = id;

	container.appendChild( wrapper );
	return wrapper;
}
1055809
function request( url, args ) {
    return new Promise( function( resolve, reject ) {
        const xhr = new XMLHttpRequest();
        xhr.open( "POST", url, true ); // true is for async.
        xhr.onload = () => {
            const DONE = 4;
            if ( xhr.readyState !== DONE ) return;

            console.info( "xhr.status=", xhr.status );
            console.info( "xhr.responseText=", xhr.responseText );
            if ( xhr.status === 200 ) resolve( xhr.responseText );
            /*
            else reject( {
                id: exports.HTTP_ERROR,
                msg: "(" + xhr.status + ") " + xhr.statusText,
                status: xhr.status
            } );
            */
        };
        xhr.onerror = function() {
            reject( {
                id: exports.HTTP_ERROR,
                err: "HTTP_ERROR (" + xhr.status + ") " + xhr.statusText,
                status: xhr.status
            } );
        };
        const params = Object.keys( args )
            .map( key => `${key}=${encodeURIComponent(args[key])}` )
            .join( "&" );
        xhr.setRequestHeader( "Content-type", "application/x-www-form-urlencoded" );
        xhr.send( params );
    } );
}
184046
@Deprecated
  public static void loadScript(String url, String id) {
    GQuery gs = GQuery.$(DOM.createElement("script"));
    GQuery gp = GQuery.$("#" + id).parent();
    if (gp.size() != 1) {
      gp = GQuery.$(GQuery.document.getBody());
    }
    GQuery.$("#" + id).remove();
    gp.append(gs.attr("src", url).attr("type", "text/javascript").attr("id", id));
  }
784205
function(args) {
    var baseUrl = this.baseUrl + this.basePath;
    var authToken = this.authToken;
    var qs = this._buildQueryString(args.params);

    return new Promise(function(resolve, reject) {
      var req = new XMLHttpRequest();

      req.onreadystatechange = function() {
        if (req.readyState === 4) {
          try {
            var response = JSON.parse(req.responseText);
            var status = req.status;

            if (status >= 200 && status < 400) {
              return resolve(
                addShadowProperty({
                  target: response,
                  property: 'httpRequest',
                  value: req
                })
              );
            }

            return reject(
              createApiError(
                merge(response, {
                  status: status,
                  httpRequest: req
                })
              )
            );
          } catch (err) {
            reject(err);
          }
        }
      };

      req.open(args.method, baseUrl + args.path + qs, true);
      req.setRequestHeader('Authorization', 'Bearer ' + authToken);
      req.setRequestHeader('Accept', 'application/json');
      req.setRequestHeader('Content-type', 'application/json; charset=utf-8');
      req.send(args.payload ? JSON.stringify(args.payload) : null);
    });
  }
803014
function pongIoDivHTML( divId, resourceURL, paramObj ) {
	log( "pong-io",  "divId="+divId+" resourceURL="+resourceURL );
	ioSense[ divId ] = new Array();
	if ( moduleConfig[ divId ] != null ) {
		pongIoRenderHTML( divId, resourceURL, paramObj, moduleConfig[ divId ] );
	} else {
		$.getJSON( 
			resourceURL+"/pong-io", 
			function( pmd ) {
				moduleConfig[ divId ] = pmd;
				pongIoRenderHTML( divId, resourceURL, paramObj, pmd );
			}
		);					
	}	
}
280741
function renderHTML(callback) {
            async.map(codeList, function (locals, n) {
                locals.req = req;
                ViewHelper.render({
                    cache: true,
                    path: SCRIPT_JADE
                }, locals, n)
            }, function (err, buf) {
                if (!err) {
                    buf = buf || [];
                    buf.splice(0, 0, SCRIPT_START_TAG);
                    buf.push(bottomScriptList.join(""));
                    buf.push(SCRIPT_END_TAG);
                }
                callback(err, buf.join(""));
            });
        }
925701
function fetch(gexf_url, callback) {
    var xhr = (function() {
      if (window.XMLHttpRequest)
        return new XMLHttpRequest();

      var names,
          i;

      if (window.ActiveXObject) {
        names = [
          'Msxml2.XMLHTTP.6.0',
          'Msxml2.XMLHTTP.3.0',
          'Msxml2.XMLHTTP',
          'Microsoft.XMLHTTP'
        ];

        for (i in names)
          try {
            return new ActiveXObject(names[i]);
          } catch (e) {}
      }

      return null;
    })();

    if (!xhr)
      throw 'XMLHttpRequest not supported, cannot load the file.';

    // Async?
    var async = (typeof callback === 'function'),
        getResult;

    // If we can't override MIME type, we are on IE 9
    // We'll be parsing the response string then.
    if (xhr.overrideMimeType) {
      xhr.overrideMimeType('text/xml');
      getResult = function(r) {
        return r.responseXML;
      };
    }
    else {
      getResult = function(r) {
        var p = new DOMParser();
        return p.parseFromString(r.responseText, 'application/xml');
      };
    }

    xhr.open('GET', gexf_url, async);

    if (async)
      xhr.onreadystatechange = function() {
        if (xhr.readyState === 4)
          callback(getResult(xhr));
      };

    xhr.send();

    return (async) ? xhr : getResult(xhr);
  }
336309
function load(url, successCallback, errorCallback) {
  const xhr = new XMLHttpRequest();

  // async, however scripts will be executed in the order they are in the
  // DOM to mirror normal script loading.
  xhr.open("GET", url, true);
  if ("overrideMimeType" in xhr) {
    xhr.overrideMimeType("text/plain");
  }
  xhr.onreadystatechange = function() {
    if (xhr.readyState === 4) {
      if (xhr.status === 0 || xhr.status === 200) {
        successCallback(xhr.responseText);
      } else {
        errorCallback();
        throw new Error("Could not load " + url);
      }
    }
  };
  return xhr.send(null);
}
1154886
function (widgetId, lang) {
                    var instance = instances[widgetId];

                    if (instance) {
                        var iframe = instance.querySelector('iframe');
                        if (lang) {
                            // Setter
                            if (iframe && iframe.src) {
                                var s = iframe.src;
                                if (/[?&]hl=/.test(s)) {
                                    s = s.replace(/([?&]hl=)\w+/, '$1' + lang);
                                } else {
                                    s += ((s.indexOf('?') === -1) ? '?' : '&') + 'hl=' + lang;
                                }

                                iframe.src = s;
                            }
                        } else {
                            // Getter
                            if (iframe && iframe.src && /[?&]hl=\w+/.test(iframe.src)) {
                                return iframe.src.replace(/.+[?&]hl=(\w+)([^\w].+)?/, '$1');
                            } else {
                                return null;
                            }
                        }
                    } else {
                        throw new Error('reCaptcha Widget ID not exists', widgetId);
                    }
                }
1105485
function phantomas(url, opts) {
	var events = new EventEmitter(),
		browser, options;

	debug('OS: %s %s', process.platform, process.arch);
	debug('Node.js: %s', process.version);
	debug('phantomas: %s', VERSION);
	debug('Puppeteer: preferred revision r%s', puppeteer._launcher._preferredRevision);
	debug('URL: <%s>', url);

	// options handling
	options = util._extend({}, opts || {}); // use util._extend to avoid #563
	options.url = options.url || url || false;

	debug('Options: %s', JSON.stringify(options));

	events.setMaxListeners(100); // MaxListenersExceededWarning: Possible EventEmitter memory leak detected.

	var results = new Results();
	results.setUrl(url);
	results.setGenerator('phantomas v' + VERSION);

	// set up and run Puppeteer
	browser = new Browser();
	browser.bind(events);

	var promise = new Promise(async (resolve, reject) => {
		try {
			if (typeof options.url !== 'string') {
				return reject(Error('URL must be a string'));
			}

			const page = await browser.init(),
				debugScope = require('debug')('phantomas:scope:log');

			// prepare a small instance object that will be passed to modules and extensions on init
			const scope = {
				getParam: (param, _default) => {
					return options[param] || _default;
				},
				getVersion: () => VERSION,

				emit: events.emit.bind(events),
				on: events.on.bind(events),
				once: events.once.bind(events),

				log: debugScope.bind(debug),

				addOffender: results.addOffender.bind(results),
				incrMetric: results.incrMetric.bind(results),
				setMetric: results.setMetric,
				addToAvgMetric: results.addToAvgMetric,

				getMetric: results.getMetric,

				// @see https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#pageevaluatepagefunction-args
				evaluate: page.evaluate.bind(page),

				// @see https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#pageselector-1
				querySelectorAll: async (selector) => {
					debug('querySelectorAll("%s")', selector)
					return page.$$(selector);
				},

				// @see https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#pageevaluateonnewdocumentpagefunction-args
				injectJs: async (script) => {
					const debug = require('debug')('phantomas:injectJs'),
						preloadFile = require('fs').readFileSync(script, 'utf8');

					await page.evaluateOnNewDocument(preloadFile);

					debug(script + ' JavaScript file has been injected into page scope');
				}
			};

			// pass phantomas options to page scope
			// https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#pageevaluateonnewdocumentpagefunction-args
			await page.evaluateOnNewDocument(options => {
				window.__phantomas_options = options;
			}, options);

			// expose the function that will pass events from page scope code into Node.js layer
			// @see https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#pageexposefunctionname-puppeteerfunction
			await page.exposeFunction('__phantomas_emit', scope.emit);

			// Inject helper code into the browser's scope
			events.on('init', () => {
				scope.injectJs(__dirname + '/../core/scope.js');
			});

			// bind to sendMsg calls from page scope code
			events.on('scopeMessage', (type, args) => {
				const debug = require('debug')('phantomas:core:scopeEvents');
				// debug(type + ' [' + args + ']');

				switch(type) {
					case 'addOffender':
					case 'incrMetric':
					case 'log':
					case 'setMetric':
						scope[type].apply(scope, args);
						break;

					default:
						debug('Unrecognized event type: ' + type);
				}
			});

			// bind to a first response
			// and reject a promise if the first response is 4xx / 5xx HTTP error
			var firstResponseReceived = false;

			events.once('recv', async entry => {
				if (!firstResponseReceived && entry.status >= 400) {
					debug('<%s> response code is HTTP %d %s', entry.url, entry.status, entry.statusText);

					// close the browser before leaving here, otherwise subsequent instances will have problems
					await browser.close();
					reject(new Error('HTTP response code from <' + entry.url + '> is ' + entry.status));
				}

				firstResponseReceived = true;
			});

			// load modules and extensions
			debug('Loading core modules...');
			loader.loadCoreModules(scope);

			debug('Loading extensions...');
			loader.loadExtensions(scope);

			debug('Loading modules...');
			loader.loadModules(scope);

			await events.emit('init', page, browser); // @desc Browser's scope and modules are set up, the page is about to be loaded

			// https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#pagegotourl-options
			const waitUntil = options['wait-for-network-idle'] ? 'networkidle0' : undefined,
				timeout = options.timeout;

			await browser.visit(url, waitUntil, timeout);

			// resolve our run
			// https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#browserclose
			await events.emit('beforeClose', page); // @desc Called before the Chromium (and all of its pages) is closed
			await browser.close();

			// your last chance to add metrics
			await events.emit('report'); // @desc Called just before the phantomas results are returned to the caller

			resolve(results);
		}
		catch(ex) {
			// close the browser before leaving here, otherwise subsequent instances will have problems
			await browser.close();
			reject(ex);
		}
	});

	promise.on = events.on.bind(events);

	return promise;
}
929087
function openBrowser(url) {
  const { action, value } = getBrowserEnv();
  switch (action) {
    case Actions.NONE:
      // Special case: BROWSER="none" will prevent opening completely.
      return false;
    case Actions.SCRIPT:
      return executeNodeScript(value, url);
    case Actions.BROWSER:
      return startBrowserProcess(value, url);
    default:
      throw new Error('Not implemented.');
  }
}
1241595
def get_html(url,
             headers=None,
             timeout=None,
             errors="strict",
             wait_time=None,
             driver=None,
             zillow_only=False,
             cache_only=False,
             zillow_first=False,
             cache_first=False,
             random=False,
             **kwargs):
    
    if wait_time is None:
        wait_time = Config.Crawler.wait_time

    # prepare url
    cache_url1 = prefix + url + "/"
    cache_url2 = prefix + url
    zillow_url = url

    only_flags = [zillow_only, cache_only]
    if sum(only_flags) == 0:
        first_flags = [zillow_first, cache_first]
        if sum(first_flags) == 0:
            if random:
                if randint(0, 1):
                    all_url = [zillow_url, cache_url1, cache_url2]
                else:
                    all_url = [cache_url1, cache_url2, zillow_url]
            else:
                all_url = [zillow_url, cache_url1, cache_url2]
        elif sum(first_flags) == 1:
            if zillow_first:
                all_url = [zillow_url, cache_url1, cache_url2]
            elif cache_first:
                all_url = [cache_url1, cache_url2, zillow_url]
        else:
            raise ValueError(
                "Only zero or one `xxx_first` argument could be `True`!")

    elif sum(only_flags) == 1:
        if zillow_only:
            all_url = [zillow_url, ]
        elif cache_only:
            all_url = [cache_url1, cache_url2]

    else:
        raise ValueError(
            "Only zero or one `xxx_only` argument could be `True`!")

    for url in all_url:
        try:
            html = _get_html(url, headers, timeout, errors,
                             wait_time, driver, **kwargs)
            return html
        except Exception as e:
            pass
    raise e