Extract HTML from response with the proper encoding: - If content type header indicates charset use that - Otherwise, look for <meta> tag with charset in body - Otherwise, browsers default to windows-1252 encoding
337813
function getHTMLFromResponseBody(buffer, contentType) {
  const [mimeType, ...typeOptions]  = contentType.split(/;\s*/);

  // Pick charset from content type
  if (mimeType)
    for (let typeOption of typeOptions)
      if (/^charset=/i.test(typeOption)) {
        const charset = typeOption.split('=')[1];
        return iconv.decode(buffer, charset);
      }

  // Otherwise, HTML documents only, pick charset from meta tag
  // Otherwise, HTML documents only, default charset in US is windows-1252
  const charsetInMetaTag = buffer.toString().match(MATCH_CHARSET);
  if (charsetInMetaTag)
    return iconv.decode(buffer, charsetInMetaTag[1]);
  else
    return iconv.decode(buffer, 'windows-1252');
}
882875
function (response) {
        var contentType = response.headers.get(HEADERS.CONTENT_TYPE),
            contentDisposition = response.headers.get(HEADERS.CONTENT_DISPOSITION),
            mimeInfo = getMimeInfo(contentType, response.stream || response.body),
            fileName = getFileNameFromDispositionHeader(contentDisposition),
            fileExtension = mimeInfo.extension,

            /**
             * @typedef Response~ResponseContentInfo
             *
             * @property {String} mimeType sanitized mime type
             * @property {String} mimeFormat format for the identified mime type
             * @property {String} charset the normalized character set
             * @property {String} fileExtension extension identified from the mime type
             * @property {String} fileName file name extracted from disposition header
             */
            contentInfo = {};


        // if file name is not present in the content disposition headers, use a default file name
        if (!fileName) {
            fileName = DEFAULT_RESPONSE_FILENAME;
            // add extension to default if present
            fileExtension && (fileName += (DOT + fileExtension));
        }

        // create a compacted list of content info from mime info and file name
        mimeInfo.mimeType && (contentInfo.mimeType = mimeInfo.mimeType);
        mimeInfo.mimeFormat && (contentInfo.mimeFormat = mimeInfo.mimeFormat);
        mimeInfo.charset && (contentInfo.charset = mimeInfo.charset);
        fileExtension && (contentInfo.fileExtension = fileExtension);
        fileName && (contentInfo.fileName = fileName);

        return contentInfo;
    }
1112705
def parse_html(html):
    
    body = html.strip().replace('\x00', '').encode('utf8') or b'<html/>'
    parser = lxml.html.HTMLParser(recover=True, encoding='utf8')
    root = lxml.etree.fromstring(body, parser=parser)
    if root is None:
        root = lxml.etree.fromstring(b'<html/>', parser=parser)
    return root
999813
String getResponseEncoding(URLConnection connection) {

        String charset = null;

        String contentType = connection.getHeaderField("Content-Type");
        if (contentType != null) {
            for (String param : contentType.replace(" ", "").split(";")) {
                if (param.startsWith("charset=")) {
                    charset = param.split("=", 2)[1];
                    break;
                }
            }
        }
        return charset;
    }
839223
def parse_response(self, resp):
        
        p, u = self.getparser()

        if hasattr(resp,'text'):
            # modern requests will do this for us
            text = resp.text # this is unicode(py2)/str(py3)
        else:

            encoding = requests.utils.get_encoding_from_headers(resp.headers)
            if encoding is None:
                encoding='utf-8' # FIXME: what to do here?

            if sys.version_info[0]==2:
                text = unicode(resp.content, encoding, errors='replace')
            else:
                assert sys.version_info[0]==3
                text = str(resp.content, encoding, errors='replace')
        p.feed(text)
        p.close()
        return u.close()
860501
private String getEncoding(final String contentType) {
        if (contentType == null || !contentType.contains(CHARSET)) {
            return DEFAULT_ENCODING;
        }
        String encoding = null;
        try {
            encoding = contentType.substring(contentType.indexOf(CHARSET) + CHARSET.length());
            encoding = encoding.toLowerCase().replaceAll("\"", "");
        } catch (Exception e) {
            //Extraction from the string failed.
        }
        if (!validEncodings.contains(encoding)) {
            LOGGER.warn("Invalid Encoding '{}' - using default - '{}'", encoding, DEFAULT_ENCODING);
            encoding = DEFAULT_ENCODING;
        }
        return encoding;
    }
1358314
@Override
    public void doGet(HttpServletRequest request, HttpServletResponse response)
            throws IOException, ServletException {

        String api = request.getParameter("api");

        if (api == null || api.length() == 0) {
            response.setContentType("text/html; charset=UTF-8");
            getIndex(response.getWriter());
        } else {
            response.setContentType("text/xml; charset=UTF-8");
            getWSDL(api.toUpperCase(), request.getRequestURL()
                            .toString(), response.getWriter());
        }

        response.flushBuffer();
    }
1362614
protected function parse_charset()
    {
        global $debugObject;

        $charset = null;

        if (function_exists('get_last_retrieve_url_contents_content_type')) {
            $contentTypeHeader = get_last_retrieve_url_contents_content_type();
            $success = preg_match('/charset=(.+)/', $contentTypeHeader, $matches);
            if ($success) {
                $charset = $matches[1];
                if (is_object($debugObject)) {
                    $debugObject->debugLog(2, 'header content-type found charset of: ' . $charset);
                }
            }
        }

        if (empty($charset)) {
            $el = $this->root->find('meta[http-equiv=Content-Type]', 0);
            if (!empty($el)) {
                $fullvalue = $el->content;
                if (is_object($debugObject)) {
                    $debugObject->debugLog(2, 'meta content-type tag found' . $fullvalue);
                }

                if (!empty($fullvalue)) {
                    $success = preg_match('/charset=(.+)/', $fullvalue, $matches);
                    if ($success) {
                        $charset = $matches[1];
                    } else {
                        // If there is a meta tag, and they don't specify the character set, research says that it's typically ISO-8859-1
                        if (is_object($debugObject)) {
                            $debugObject->debugLog(2, 'meta content-type tag couldn\'t be parsed. using iso-8859 default.');
                        }
                        $charset = 'ISO-8859-1';
                    }
                }
            }
        }

        // If we couldn't find a charset above, then lets try to detect one based on the text we got...
        if (empty($charset)) {
            // Have php try to detect the encoding from the text given to us.
            $charset = mb_detect_encoding($this->root->plaintext . 'ascii', $encoding_list = ['UTF-8', 'CP1252']);
            if (is_object($debugObject)) {
                $debugObject->debugLog(2, 'mb_detect found: ' . $charset);
            }

            // and if this doesn't work...  then we need to just wrongheadedly assume it's UTF-8 so that we can move on - cause this will usually give us most of what we need...
            if ($charset === false) {
                if (is_object($debugObject)) {
                    $debugObject->debugLog(2, 'since mb_detect failed - using default of utf-8');
                }
                $charset = 'UTF-8';
            }
        }

        // Since CP1252 is a superset, if we get one of it's subsets, we want it instead.
        if ((strtolower($charset) == strtolower('ISO-8859-1')) || (strtolower($charset) == strtolower('Latin1')) || (strtolower($charset) == strtolower('Latin-1'))) {
            if (is_object($debugObject)) {
                $debugObject->debugLog(2, 'replacing ' . $charset . ' with CP1252 as its a superset');
            }
            $charset = 'CP1252';
        }

        if (is_object($debugObject)) {
            $debugObject->debugLog(1, 'EXIT - ' . $charset);
        }

        return $this->_charset = $charset;
    }
526667
def _parse_content_type(self, value):
        
        content_type, params = parse_header(value)
        if 'charset' in params:
            charset = params['charset']
        else:
            charset = None
        return content_type, charset
1166354
protected function parse_charset()
	{
		global $debug_object;

		$charset = null;

		if (function_exists('get_last_retrieve_url_contents_content_type'))
		{
			$contentTypeHeader = get_last_retrieve_url_contents_content_type();
			$success = preg_match('/charset=(.+)/', $contentTypeHeader, $matches);
			if ($success)
			{
				$charset = $matches[1];
				if (is_object($debug_object)) {$debug_object->debug_log(2, 'header content-type found charset of: ' . $charset);}
			}

		}

		if (empty($charset))
		{
			$el = $this->root->find('meta[http-equiv=Content-Type]',0,true);
			if (!empty($el))
			{
				$fullvalue = $el->content;
				if (is_object($debug_object)) {$debug_object->debug_log(2, 'meta content-type tag found' . $fullvalue);}

				if (!empty($fullvalue))
				{
					$success = preg_match('/charset=(.+)/i', $fullvalue, $matches);
					if ($success)
					{
						$charset = $matches[1];
					}
					else
					{
						// If there is a meta tag, and they don't specify the character set, research says that it's typically ISO-8859-1
						if (is_object($debug_object)) {$debug_object->debug_log(2, 'meta content-type tag couldn\'t be parsed. using iso-8859 default.');}
						$charset = 'ISO-8859-1';
					}
				}
			}
		}

		// If we couldn't find a charset above, then lets try to detect one based on the text we got...
		if (empty($charset))
		{
			// Use this in case mb_detect_charset isn't installed/loaded on this machine.
			$charset = false;
			if (function_exists('mb_detect_encoding'))
			{
				// Have php try to detect the encoding from the text given to us.
				$charset = mb_detect_encoding($this->root->plaintext . "ascii", $encoding_list = array( "UTF-8", "CP1252" ) );
				if (is_object($debug_object)) {$debug_object->debug_log(2, 'mb_detect found: ' . $charset);}
			}

			// and if this doesn't work...  then we need to just wrongheadedly assume it's UTF-8 so that we can move on - cause this will usually give us most of what we need...
			if ($charset === false)
			{
				if (is_object($debug_object)) {$debug_object->debug_log(2, 'since mb_detect failed - using default of utf-8');}
				$charset = 'UTF-8';
			}
		}

		// Since CP1252 is a superset, if we get one of it's subsets, we want it instead.
		if ((strtolower($charset) == strtolower('ISO-8859-1')) || (strtolower($charset) == strtolower('Latin1')) || (strtolower($charset) == strtolower('Latin-1')))
		{
			if (is_object($debug_object)) {$debug_object->debug_log(2, 'replacing ' . $charset . ' with CP1252 as its a superset');}
			$charset = 'CP1252';
		}

		if (is_object($debug_object)) {$debug_object->debug_log(1, 'EXIT - ' . $charset);}

		return $this->_charset = $charset;
	}
491151
def html_to_dom(html, default_encoding=DEFAULT_ENCODING, encoding=None, errors=DEFAULT_ENC_ERRORS):
    """"""
    if isinstance(html, unicode):
        decoded_html = html
        # encode HTML for case it's XML with encoding declaration
        forced_encoding = encoding if encoding else default_encoding
        html = html.encode(forced_encoding, errors)
    else:
        decoded_html = decode_html(html, default_encoding, encoding, errors)

    try:
        dom = lxml.html.fromstring(decoded_html, parser=lxml.html.HTMLParser())
    except ValueError:
        # Unicode strings with encoding declaration are not supported.
        # for XHTML files with encoding declaration, use the declared encoding
        dom = lxml.html.fromstring(html, parser=lxml.html.HTMLParser())

    return dom
1261432
def get_heading_encoding(response):
    '''
    '''
    encoding = wpull.protocol.http.util.parse_charset(
        response.fields.get('content-type', ''))

    if encoding:
        return wpull.string.normalize_codec_name(encoding)
    else:
        return None
1053553
def _get_tag(self, response, tag_name="html", encoding="utf-8"):
        
        def find_tag(tag_name):
            tag_start = tag_end = None
            found = lambda: \
                tag_start is not None and tag_end is not None
            html = self._html.lower()
            start = html.find("<%s" % tag_name)
            if start >= 0:
                tag_start = start
            else:
                return None # no tag
            end = html.find("</%s>" % tag_name)
            if end > tag_start:
                tag_end = end+len(tag_name)+3
            elif consumed:
                tag_end = -1 # till the end
            if found():
                return self._html[tag_start:tag_end]
            return None
        consumed = getattr(response, 'consumed', False)
        if not consumed:
            stream = getattr(response, 'stream', None)
            if stream is None:
                stream = response.iter_content(config.CHUNK_SIZE) # , decode_unicode=True
                response.stream = stream
            while True:
                try:
                    chunk = next(stream)
                    self._html += chunk
                    tag = find_tag(tag_name)
                    if tag:
                        return tag
                    if len(self._html) > config.HTML_MAX_BYTESIZE:
                        raise HTMLParseError('Maximum response size reached.')
                except StopIteration:
                    response.consumed = True
        tag = find_tag(tag_name)
        return decode(tag, encoding)
179864
function charset (type) {
  if (!type || typeof type !== 'string') {
    return false
  }

  // TODO: use media-typer
  var match = EXTRACT_TYPE_REGEXP.exec(type)
  var mime = match && db[match[1].toLowerCase()]

  if (mime && mime.charset) {
    return mime.charset
  }

  // default text/* to utf-8
  if (match && TEXT_TYPE_REGEXP.test(match[1])) {
    return 'UTF-8'
  }

  return false
}
797320
private static String getResponseMimeType(final HttpResponse httpHeadResponse) {
        Header contentTypeHeader = httpHeadResponse.getFirstHeader("Content-Type");
        if (contentTypeHeader != null) {
            String contentType = contentTypeHeader.getValue();
            if (contentType != null) {
                try {
                    return ContentType.parse(contentType).getMimeType();
                } catch (ParseException | UnsupportedCharsetException exception) {
                    return contentType.split(";")[0].trim();
                }
            }
        }

        return ContentType.DEFAULT_TEXT.getMimeType();
    }
1339246
def parse_content_type(headers: MutableMapping) -> Tuple[Optional[str], str]:
    
    content_type = headers.get("content-type")
    if not content_type:
        return None, "utf-8"
    else:
        type_, parameters = cgi.parse_header(content_type)
        encoding = parameters.get("charset", "utf-8")
        return type_, encoding
849814
def parse_html(fileobj, encoding):
    
    parser = HTMLParser(encoding=encoding, remove_blank_text=True)
    return parse(fileobj, parser)