Extract intel from the response body.
332610
def intel_extractor(url, response):
    """"""
    for rintel in rintels:
        res = re.sub(r'<(script).*?</\1>(?s)', '', response)
        res = re.sub(r'<[^<]+?>', '', res)
        matches = rintel[0].findall(res)
        if matches:
            for match in matches:
                verb('Intel', match)
                bad_intel.add((match, rintel[1], url))
890385
def decode_response(response):
    
    def is_compressed(headers):
        encoding = headers.get('content-encoding', [])
        return encoding and encoding[0] in ('gzip', 'deflate')

    def decompress_body(body, encoding):
        """Returns decompressed body according to encoding using zlib.
        to (de-)compress gzip format, use wbits = zlib.MAX_WBITS | 16
        """
        if encoding == 'gzip':
            return zlib.decompress(body, zlib.MAX_WBITS | 16)
        else:  # encoding == 'deflate'
            return zlib.decompress(body)

    # Deepcopy here in case `headers` contain objects that could
    # be mutated by a shallow copy and corrupt the real response.
    response = copy.deepcopy(response)
    headers = CaseInsensitiveDict(response['headers'])
    if is_compressed(headers):
        encoding = headers['content-encoding'][0]
        headers['content-encoding'].remove(encoding)
        if not headers['content-encoding']:
            del headers['content-encoding']

        new_body = decompress_body(response['body']['string'], encoding)
        response['body']['string'] = new_body
        headers['content-length'] = [str(len(new_body))]
        response['headers'] = dict(headers)
    return response
909727
def from_response(cls, response):
        
        hash_key = None
        range_key = None
        # KeySchema may not be in the response if the TableStatus is DELETING.
        if 'KeySchema' in response:
            attrs = dict(((d['AttributeName'],
                           DynamoKey(d['AttributeName'], d['AttributeType']))
                          for d in response['AttributeDefinitions']))
            hash_key = attrs[response['KeySchema'][0]['AttributeName']]
            if len(response['KeySchema']) > 1:
                range_key = attrs[response['KeySchema'][1]['AttributeName']]

        indexes = []
        for idx in response.get('LocalSecondaryIndexes', []):
            indexes.append(LocalIndex.from_response(idx, attrs))
        global_indexes = []
        for idx in response.get('GlobalSecondaryIndexes', []):
            global_indexes.append(GlobalIndex.from_response(idx, attrs))

        table = cls(
            name=response['TableName'],
            hash_key=hash_key,
            range_key=range_key,
            indexes=indexes,
            global_indexes=global_indexes,
            throughput=Throughput.from_response(
                response['ProvisionedThroughput']),
            status=response['TableStatus'],
            size=response['TableSizeBytes'],
        )
        table.response = response
        return table
441950
def _get_cpu_info_from_sysctl():
	'''
	
	'''
	try:
		# Just return {} if there is no sysctl
		if not DataSource.has_sysctl():
			return {}

		# If sysctl fails return {}
		returncode, output = DataSource.sysctl_machdep_cpu_hw_cpufrequency()
		if output == None or returncode != 0:
			return {}

		# Various fields
		vendor_id = _get_field(False, output, None, None, 'machdep.cpu.vendor')
		processor_brand = _get_field(True, output, None, None, 'machdep.cpu.brand_string')
		cache_size = _get_field(False, output, None, None, 'machdep.cpu.cache.size')
		stepping = _get_field(False, output, int, 0, 'machdep.cpu.stepping')
		model = _get_field(False, output, int, 0, 'machdep.cpu.model')
		family = _get_field(False, output, int, 0, 'machdep.cpu.family')

		# Flags
		flags = _get_field(False, output, None, '', 'machdep.cpu.features').lower().split()
		flags.extend(_get_field(False, output, None, '', 'machdep.cpu.leaf7_features').lower().split())
		flags.extend(_get_field(False, output, None, '', 'machdep.cpu.extfeatures').lower().split())
		flags.sort()

		# Convert from GHz/MHz string to Hz
		hz_advertised, scale = _parse_cpu_brand_string(processor_brand)
		hz_actual = _get_field(False, output, None, None, 'hw.cpufrequency')
		hz_actual = _to_decimal_string(hz_actual)

		info = {
		'vendor_id_raw' : vendor_id,
		'brand_raw' : processor_brand,

		'hz_advertised_friendly' : _hz_short_to_friendly(hz_advertised, scale),
		'hz_actual_friendly' : _hz_short_to_friendly(hz_actual, 0),
		'hz_advertised' : _hz_short_to_full(hz_advertised, scale),
		'hz_actual' : _hz_short_to_full(hz_actual, 0),

		'l2_cache_size' : _to_friendly_bytes(cache_size),

		'stepping' : stepping,
		'model' : model,
		'family' : family,
		'flags' : flags
		}

		info = {k: v for k, v in info.items() if v}
		return info
	except:
		return {}
623845
private double extractScore(String clutoOutput) throws IOException {
        double score = 0;
        BufferedReader reader =
            new BufferedReader(new StringReader(clutoOutput));
        String line = null;
        while ((line = reader.readLine()) != null) {
            if (line.contains("[I2=")) {
                String[] split = line.split("=");
                int endOfIndex = split[1].indexOf("]");
                return Double.parseDouble(split[1].substring(0, endOfIndex));
            }
        }
        return 0;
    }
39512
def extract_payload(self):
        """"""
        if not self.check_signature():
            raise InvalidSignature('Invalid Signature')
        if request.is_json:
            # Request.get_json() could be first called with silent=True.
            delete_cached_json_for(request)
            return request.get_json(silent=False, cache=False)
        elif request.content_type == 'application/x-www-form-urlencoded':
            return dict(request.form)
        raise InvalidPayload(request.content_type)
184256
def _get_cached_response(self):
        """"""

        if not self._is_cached():
            response = self._download_response()
            self.cache.set_xml(self._get_cache_key(), response)

        return self.cache.get_xml(self._get_cache_key())
821541
def _parse_response(response, clazz, is_list=False, resource_name=None):
        """"""
        target = response.json()[
            resource_name] if resource_name else response.json()
        if is_list:
            return [clazz.from_json(resource) for resource in target]
        else:
            return clazz.from_json(target)
992994
def _get_cpu_info_from_sysinfo_v2():
	'''
	
	'''
	try:
		# Just return {} if there is no sysinfo
		if not DataSource.has_sysinfo():
			return {}

		# If sysinfo fails return {}
		returncode, output = DataSource.sysinfo_cpu()
		if output == None or returncode != 0:
			return {}

		# Various fields
		vendor_id = '' #_get_field(False, output, None, None, 'CPU #0: ')
		processor_brand = output.split('CPU #0: "')[1].split('"\n')[0].strip()
		cache_size = '' #_get_field(False, output, None, None, 'machdep.cpu.cache.size')
		signature = output.split('Signature:')[1].split('\n')[0].strip()
		#
		stepping = int(signature.split('stepping ')[1].split(',')[0].strip())
		model = int(signature.split('model ')[1].split(',')[0].strip())
		family = int(signature.split('family ')[1].split(',')[0].strip())

		# Flags
		def get_subsection_flags(output):
			retval = []
			for line in output.split('\n')[1:]:
				if not line.startswith('                ') and not line.startswith('		'): break
				for entry in line.strip().lower().split(' '):
					retval.append(entry)
			return retval

		flags = get_subsection_flags(output.split('Features: ')[1]) + \
				get_subsection_flags(output.split('Extended Features (0x00000001): ')[1]) + \
				get_subsection_flags(output.split('Extended Features (0x80000001): ')[1])
		flags.sort()

		# Convert from GHz/MHz string to Hz
		lines = [n for n in output.split('\n') if n]
		raw_hz = lines[0].split('running at ')[1].strip().lower()
		hz_advertised = raw_hz.rstrip('mhz').rstrip('ghz').strip()
		hz_advertised = _to_decimal_string(hz_advertised)
		hz_actual = hz_advertised

		scale = 0
		if raw_hz.endswith('mhz'):
			scale = 6
		elif raw_hz.endswith('ghz'):
			scale = 9

		info = {
		'vendor_id_raw' : vendor_id,
		'brand_raw' : processor_brand,

		'hz_advertised_friendly' : _hz_short_to_friendly(hz_advertised, scale),
		'hz_actual_friendly' : _hz_short_to_friendly(hz_actual, scale),
		'hz_advertised' : _hz_short_to_full(hz_advertised, scale),
		'hz_actual' : _hz_short_to_full(hz_actual, scale),

		'l2_cache_size' : _to_friendly_bytes(cache_size),

		'stepping' : stepping,
		'model' : model,
		'family' : family,
		'flags' : flags
		}

		info = {k: v for k, v in info.items() if v}
		return info
	except:
		#raise # NOTE: To have this throw on error, uncomment this line
		return {}
1324007
def parse_info(response):
    ""
    info = {}
    response = nativestr(response)

    def get_value(value):
        if ',' not in value or '=' not in value:
            try:
                if '.' in value:
                    return float(value)
                else:
                    return int(value)
            except ValueError:
                return value
        else:
            sub_dict = {}
            for item in value.split(','):
                k, v = item.rsplit('=', 1)
                sub_dict[k] = get_value(v)
            return sub_dict

    for line in response.splitlines():
        if line and not line.startswith('#'):
            if line.find(':') != -1:
                key, value = line.split(':', 1)
                info[key] = get_value(value)
            else:
                # if the line isn't splittable, append it to the "__raw__" key
                info.setdefault('__raw__', []).append(line)

    return info
1055773
def _map_response(response, decode=False):
    """"""
    # This causes weird deepcopy errors, so it's commented out for now.
    # item._urllib3_response = response
    item = httplib2.Response(response.getheaders())
    item.status = response.status
    item['status'] = str(item.status)
    item.reason = response.reason
    item.version = response.version

    # httplib2 expects the content-encoding header to be stripped and the
    # content length to be the length of the uncompressed content.
    # This does not occur for 'HEAD' requests.
    if decode and item.get('content-encoding') in ['gzip', 'deflate']:
        item['content-length'] = str(len(response.data))
        item['-content-encoding'] = item.pop('content-encoding')

    return item
476828
def decode_cpu_id(self, cpuid):
        """"""
        ret = ()
        for i in cpuid.split(':'):
            ret += (eval('0x' + i),)

        return ret
1254177
def __get_jp(self, extractor_processor, sub_output=None):
        """"""
        if sub_output is None and extractor_processor.output_field is None:
            raise ValueError(
                "ExtractorProcessors input paths cannot be unioned across fields.  Please specify either a sub_output or use a single scalar output_field")
        if extractor_processor.get_output_jsonpath_with_name(sub_output) is not None:
            return extractor_processor.get_output_jsonpath_with_name(sub_output)
        else:
            return extractor_processor.get_output_jsonpath(sub_output)
472249
def decode_produce_response(cls, data):
        
        ((correlation_id, num_topics), cur) = relative_unpack('>ii', data, 0)

        for _i in range(num_topics):
            topic, cur = read_short_ascii(data, cur)
            ((num_partitions,), cur) = relative_unpack('>i', data, cur)
            for _i in range(num_partitions):
                ((partition, error, offset), cur) = relative_unpack('>ihq', data, cur)

                yield ProduceResponse(topic, partition, error, offset)
1004317
def _multiplexed_response_stream_helper(self, response):
        """"""

        # Disable timeout on the underlying socket to prevent
        # Read timed out(s) for long running processes
        socket = self._get_raw_response_socket(response)
        self._disable_socket_timeout(socket)

        while True:
            header = response.raw.read(STREAM_HEADER_SIZE_BYTES)
            if not header:
                break
            _, length = struct.unpack('>BxxxL', header)
            if not length:
                continue
            data = response.raw.read(length)
            if not data:
                break
            yield data
937292
def parse(self, response, metadata_type):
        
        xml = xmltodict.parse(response.text)
        self.analyze_reply_code(xml_response_dict=xml)
        base = xml.get('RETS', {}).get('METADATA', {}).get(metadata_type, {})

        if metadata_type == 'METADATA-SYSTEM':
            syst = base.get('System', base.get('SYSTEM'))
            if not syst:
                raise ParseError("Could not get the System key from a METADATA-SYSTEM request.")

            system_obj = {}
            if syst.get('SystemID'):
                system_obj['system_id'] = str(syst['SystemID'])
            if syst.get('SystemDescription'):
                system_obj['system_description'] = str(syst['SystemDescription'])
            if syst.get('Comments'):
                system_obj['comments'] = syst['Comments']
            if base.get('@Version'):
                system_obj['version'] = base['@Version']
            return [system_obj]

        elif metadata_type == 'METADATA-CLASS':
            key = 'class'
        elif metadata_type == 'METADATA-RESOURCE':
            key = 'resource'
        elif metadata_type == 'METADATA-LOOKUP_TYPE':
            key = 'lookuptype'
        elif metadata_type == 'METADATA-OBJECT':
            key = 'object'
        elif metadata_type == 'METADATA-TABLE':
            key = 'field'
        else:
            msg = "Got an unknown metadata type of {0!s}".format(metadata_type)
            raise ParseError(msg)

        # Get the version with the right capitalization from the dictionary
        key_cap = None
        for k in base.keys():
            if k.lower() == key:
                key_cap = k

        if not key_cap:
            msg = 'Could not find {0!s} in the response XML'.format(key)
            raise ParseError(msg)

        if isinstance(base[key_cap], list):
            return base[key_cap]
        else:
            return [base[key_cap]]
849203
def unpack_rpc_response(status, response=None, rpc_id=0, address=0):
    """"""

    status_code = status & ((1 << 6) - 1)

    if address == 8:
        status_code &= ~(1 << 7)

    if status == 0:
        raise BusyRPCResponse()
    elif status == 2:
        raise RPCNotFoundError("rpc %d:%04X not found" % (address, rpc_id))
    elif status == 3:
        raise RPCErrorCode(status_code)
    elif status == 0xFF:
        raise TileNotFoundError("tile %d not found" % address)
    elif status_code != 0:
        raise RPCErrorCode(status_code)

    if response is None:
        response = b''

    return response