Return response with HAR, inline scritps and software detected by JS matchers.

    :rtype: dict
182198
def get_response(url, plugins, timeout=SPLASH_TIMEOUT):
    
    lua_script = create_lua_script(plugins)
    lua = urllib.parse.quote_plus(lua_script)
    page_url = f'{SPLASH_URL}/execute?url={url}&timeout={timeout}&lua_source={lua}'

    try:
        with docker_container():
            logger.debug('[+] Sending request to Splash instance')
            res = requests.get(page_url)
    except requests.exceptions.ConnectionError:
        raise SplashError("Could not connect to Splash server {}".format(SPLASH_URL))

    logger.debug('[+] Response received')

    json_data = res.json()

    if res.status_code in ERROR_STATUS_CODES:
        raise SplashError(get_splash_error(json_data))

    softwares = json_data['softwares']
    scripts = json_data['scripts'].values()
    har = get_valid_har(json_data['har'])

    js_error = get_evaljs_error(json_data)
    if js_error:
        logger.debug('[+] WARNING: failed to eval JS matchers: %(n)s', {'n': js_error})
    else:
        logger.debug('[+] Detected %(n)d softwares from the DOM', {'n': len(softwares)})
    logger.debug('[+] Detected %(n)d scripts from the DOM', {'n': len(scripts)})
    logger.debug('[+] Final HAR has %(n)d valid entries', {'n': len(har)})

    return {'har': har, 'scripts': scripts, 'softwares': softwares}
236235
function getScraperResults (SCRAPER, urls, htmls) {
  return Promise.all(
    urls.map((url, i) => {
      const html = htmls[i]
      const name = SCRAPER.name
      const Module = require(name)
      return SCRAPER.scrape(Module, url, html).then(metadata => {
        return SCRAPER.normalize(metadata)
      })
    })
  )
}
842599
def _process_json_response(self, response):
        '''  '''

        json_response = self._get_json_response(response)
        
        if self.response_callback is not None:
            json_response = self.response_callback(json_response)
            response._content = json.dumps(json_response)

        self.http_status_code = response.status_code
        self._check_error(response, json_response)
        self._check_warnings(json_response)

        return json_response
767963
def response_json(self, status, response, content_type='application/json', encoding='utf-8', headers=None, jsonp=None):
        

        encoder = JSONEncoder(
            check_circular=self.app.validate_output,
            allow_nan=False,
            sort_keys=True,
            indent=2 if self.app.pretty_output else None,
            separators=(',', ': ') if self.app.pretty_output else (',', ':')
        )
        content = encoder.encode(response)
        if jsonp:
            content_list = [jsonp.encode(encoding), b'(', content.encode(encoding), b');']
        else:
            content_list = [content.encode(encoding)]
        return self.response(status, content_type, content_list, headers=headers)
605576
def js_map_loader(url):
    

    hdr = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.77 Safari/535.7'}
    req = urllib.request.Request(url, headers=hdr)
    page = urlopen(req)
    result = page.read().decode('utf-8')
    result = result[len(re.search(r'^.* = ', result).group()):]

    return json.loads(result)
583393
def _parse_response(response, post_request=False):
        
        try:
            data = response.json()
        except:
            msg = 'Unhandled HTTP %s response, shown truncated below:\n%s...' % (
                response.status_code, response.text[:50]
            )
            raise ValueError(msg)

        if not response.ok:
            utils.error_message_and_exit(None, data)

        if post_request and not data['success']:
            raise Exception('Asmaster Api Error: [%s]' % data['error'])

        return data
434744
def json(self, **kwargs):
        
        try:
            return super(ScraperResponse, self).json(**kwargs)
        except ValueError as ve:
            raise ParseException(ve)
930103
def getresponse(self):
        
        resp = self.http.getresponse()
        self.log.info("resp is %s", str(resp))
        if resp.status < 400:
            return resp
        else:
            errtext = resp.read()
            content_type = resp.getheader('Content-Type', 'text/plain')
            raise HttpError(code=resp.status, content_type=content_type, content=errtext)
409537
def _get_match(self, prefix):
        
        # (hard coded) If we match a CPR response, return Keys.CPRResponse.
        # (This one doesn't fit in the ANSI_SEQUENCES, because it contains
        # integer variables.)
        if _cpr_response_re.match(prefix):
            return Keys.CPRResponse

        elif _mouse_event_re.match(prefix):
            return Keys.Vt100MouseEvent

        # Otherwise, use the mappings.
        try:
            return ANSI_SEQUENCES[prefix]
        except KeyError:
            return None
1121628
def check_status_code(response, codes=None):
    
    codes = codes or [httplib.OK]
    checker = (
        codes
        if callable(codes)
        else lambda resp: resp.status_code in codes
    )
    if not checker(response):
        raise exceptions.ApiError(response, response.json())
298985
def str_matcher(mode):
    

    def f_raw(inp_str, pos):
        return unique_literal_cache_pool[mode] if inp_str.startswith(mode, pos) else None

    def f_collection(inp_str, pos):
        for each in mode:
            if inp_str.startswith(each, pos):
                return unique_literal_cache_pool[each]
        return None

    if isinstance(mode, str):
        return f_raw

    if len(mode) is 1:
        mode = mode[0]
        return f_raw

    return f_collection
86310
async def parse_result(response, response_type=None, *, encoding="utf-8"):
    
    if response_type is None:
        ct = response.headers.get("content-type")
        if ct is None:
            cl = response.headers.get("content-length")
            if cl is None or cl == "0":
                return ""
            raise TypeError(
                "Cannot auto-detect response type "
                "due to missing Content-Type header."
            )
        main_type, sub_type, extras = parse_content_type(ct)
        if sub_type == "json":
            response_type = "json"
        elif sub_type == "x-tar":
            response_type = "tar"
        elif (main_type, sub_type) == ("text", "plain"):
            response_type = "text"
            encoding = extras.get("charset", encoding)
        else:
            raise TypeError("Unrecognized response type: {ct}".format(ct=ct))
    if "tar" == response_type:
        what = await response.read()
        return tarfile.open(mode="r", fileobj=BytesIO(what))
    if "json" == response_type:
        data = await response.json(encoding=encoding)
    elif "text" == response_type:
        data = await response.text(encoding=encoding)
    else:
        data = await response.read()
    return data
943493
def _build_response(data, renderer=None):
    
    if isinstance(data, Response) or isinstance(data, BaseResponse):
        return data
    if not renderer:
        raise AttributeError(" Renderer is required")
    if isinstance(data, dict) or data is None:
        data = {} if data is None else data
        for _ in __view_parsers:
            data = _(data)
        return renderer(data), 200
    elif isinstance(data, tuple):
        data, status, headers = _normalize_response_tuple(data)
        for _ in __view_parsers:
            data = _(data)
        return renderer(data or {}), status, headers
    return data
870872
def parse_one_response(line, ignore_bad_cookies=False,
                       ignore_bad_attributes=True):
    
    cookie_dict = {}
    # Basic validation, extract name/value/attrs-chunk
    match = Definitions.SET_COOKIE_HEADER_RE.match(line)
    if not match:
        if not ignore_bad_cookies:
            raise InvalidCookieError(data=line)
        _report_invalid_cookie(line)
        return None
    cookie_dict.update({
        'name': match.group('name'),
        'value': match.group('value')})
    # Extract individual attrs from the attrs chunk
    for match in Definitions.ATTR_RE.finditer(match.group('attrs')):
        captured = dict((k, v) for (k, v) in match.groupdict().items() if v)
        unrecognized = captured.get('unrecognized', None)
        if unrecognized:
            if not ignore_bad_attributes:
                raise InvalidCookieAttributeError(None, unrecognized,
                                                  "unrecognized")
            _report_unknown_attribute(unrecognized)
            continue
        # for unary flags
        for key in ('secure', 'httponly'):
            if captured.get(key):
                captured[key] = True
        # ignore subcomponents of expires - they're still there to avoid doing
        # two passes
        timekeys = ('weekday', 'month', 'day', 'hour', 'minute', 'second',
                    'year')
        if 'year' in captured:
            for key in timekeys:
                del captured[key]
        elif 'year2' in captured:
            for key in timekeys:
                del captured[key + "2"]
        cookie_dict.update(captured)
    return cookie_dict
207015
def matchers(self):
        """"""
        # To remove python_matches we now have to override it as it's now a property in the superclass.
        return [
            self.file_matches,
            self.magic_matches,
            self.python_func_kw_matches,
            self.dict_key_matches,
        ]
679298
def _parse_header_links(response):
    
    values = response.headers.getRawHeaders(b'link', [b''])
    value = b','.join(values).decode('ascii')
    with LOG_HTTP_PARSE_LINKS(raw_link=value) as action:
        links = {}
        replace_chars = u' \'"'
        for val in re.split(u', *<', value):
            try:
                url, params = val.split(u';', 1)
            except ValueError:
                url, params = val, u''

            link = {}
            link[u'url'] = url.strip(u'<> \'"')
            for param in params.split(u';'):
                try:
                    key, value = param.split(u'=')
                except ValueError:
                    break
                link[key.strip(replace_chars)] = value.strip(replace_chars)
            links[link.get(u'rel') or link.get(u'url')] = link
        action.add_success_fields(parsed_links=links)
        return links
1078954
def from_response(response, method, url):
    

    req_id = response.headers.get("x-openstack-request-id")
    # NOTE(hdd) true for older versions of nova and cinder
    if not req_id:
        req_id = response.headers.get("x-compute-request-id")
    kwargs = {
        "http_status": response.status_code,
        "response": response,
        "method": method,
        "url": url,
        "request_id": req_id,
    }
    if "retry-after" in response.headers:
        kwargs["retry_after"] = response.headers["retry-after"]

    content_type = response.headers.get("Content-Type", "")
    if content_type.startswith("application/json"):
        try:
            body = response.json()
        except ValueError:
            pass
        else:
            if isinstance(body, dict):
                error = body.get(list(body)[0])
                if isinstance(error, dict):
                    kwargs["message"] = (error.get("message") or
                                         error.get("faultstring"))
                    kwargs["details"] = (error.get("details") or
                                         six.text_type(body))
    elif content_type.startswith("text/"):
        kwargs["details"] = response.text

    try:
        cls = _code_map[response.status_code]
    except KeyError:
        if 500 <= response.status_code < 600:
            cls = HttpServerError
        elif 400 <= response.status_code < 500:
            cls = HTTPClientError
        else:
            cls = HttpError
    return cls(**kwargs)