Check a bytes string for a BOM to correctly detect the encoding

    Fallback to locale.getpreferredencoding(False) like open() on Python3
182583
def auto_decode(data):
    # type: (bytes) -> Text
    """"""
    for bom, encoding in BOMS:
        if data.startswith(bom):
            return data[len(bom):].decode(encoding)
    # Lets check the first two lines as in PEP263
    for line in data.split(b'\n')[:2]:
        if line[0:1] == b'#' and ENCODING_RE.search(line):
            encoding = ENCODING_RE.search(line).groups()[0].decode('ascii')
            return data.decode(encoding)
    return data.decode(
        locale.getpreferredencoding(False) or sys.getdefaultencoding(),
    )
262074
def native(s, encoding='utf-8', fallback='iso-8859-1'):
    """"""

    if isinstance(s, str):
        return s

    if str is unicode:  # Python 3.x ->
        return unicodestr(s, encoding, fallback)

    return bytestring(s, encoding, fallback)
258511
def _verify_python3_env():
    """"""
    if PY2:
        return
    try:
        import locale
        fs_enc = codecs.lookup(locale.getpreferredencoding()).name
    except Exception:
        fs_enc = 'ascii'
    if fs_enc != 'ascii':
        return

    extra = ''
    if os.name == 'posix':
        import subprocess
        try:
            rv = subprocess.Popen(['locale', '-a'], stdout=subprocess.PIPE,
                                  stderr=subprocess.PIPE).communicate()[0]
        except OSError:
            rv = b''
        good_locales = set()
        has_c_utf8 = False

        # Make sure we're operating on text here.
        if isinstance(rv, bytes):
            rv = rv.decode('ascii', 'replace')

        for line in rv.splitlines():
            locale = line.strip()
            if locale.lower().endswith(('.utf-8', '.utf8')):
                good_locales.add(locale)
                if locale.lower() in ('c.utf8', 'c.utf-8'):
                    has_c_utf8 = True

        extra += '\n\n'
        if not good_locales:
            extra += (
                'Additional information: on this system no suitable UTF-8\n'
                'locales were discovered.  This most likely requires resolving\n'
                'by reconfiguring the locale system.'
            )
        elif has_c_utf8:
            extra += (
                'This system supports the C.UTF-8 locale which is recommended.\n'
                'You might be able to resolve your issue by exporting the\n'
                'following environment variables:\n\n'
                '    export LC_ALL=C.UTF-8\n'
                '    export LANG=C.UTF-8'
            )
        else:
            extra += (
                'This system lists a couple of UTF-8 supporting locales that\n'
                'you can pick from.  The following suitable locales were\n'
                'discovered: %s'
            ) % ', '.join(sorted(good_locales))

        bad_locale = None
        for locale in os.environ.get('LC_ALL'), os.environ.get('LANG'):
            if locale and locale.lower().endswith(('.utf-8', '.utf8')):
                bad_locale = locale
            if locale is not None:
                break
        if bad_locale is not None:
            extra += (
                '\n\nClick discovered that you exported a UTF-8 locale\n'
                'but the locale system could not pick up from it because\n'
                'it does not exist.  The exported locale is "%s" but it\n'
                'is not supported'
            ) % bad_locale

    raise RuntimeError(
        'Click will abort further execution because Python 3 was'
        ' configured to use ASCII as encoding for the environment.'
        ' Consult https://click.palletsprojects.com/en/7.x/python3/ for'
        ' mitigation steps.' + extra
    )
801647
def decode_if_bytes(obj, mode=True):
    """"""
    if mode is True:
        mode = unicode_errors_default
    if isinstance(obj, bytes):
        return obj.decode("utf-8", errors=mode)
    return obj
350119
def open_with_encoding(filename,
                       encoding=None, mode='r', limit_byte_check=-1):
    """"""
    if not encoding:
        encoding = detect_encoding(filename, limit_byte_check=limit_byte_check)

    return io.open(filename, mode=mode, encoding=encoding,
                   newline='')
1195921
def guess_stream_needs_encoding(fileobj, default=True):
    
    # XXX: Unicode
    # On Python 2, stdout is bytes. However, we can't wrap it in a
    # TextIOWrapper, as it's not from IOBase, so it doesn't have .seekable.
    # It does, however, have a mode, and we can cheese it base on that.
    # On Python 3, stdout is a TextIOWrapper, and so we can safely write
    # str to it, and it will encode it correctly for the target terminal or
    # whatever.
    # If it's a io.BytesIO or StringIO, then it won't have a mode, but it
    # is a read/write stream, so we can get its type by reading 0 bytes and
    # checking the type.
    try:
        # If it's a r/w stream, this will give us the type of it
        t = type(fileobj.read(0))

        if t is bytes:
            return True
        elif t is unicode:
            return False

    except Exception:
        pass

    try:
        mode = fileobj.mode

        if PY2 and mode == "w":
            mode = "wb"

        if "b" in mode:
            return True
        else:
            return False
    except Exception:
        pass

    return default
2794
function stripBOM (content) {
    content = content.toString()
    // Remove byte order marker. This catches EF BB BF (the UTF-8 BOM)
    // because the buffer-to-string conversion in `fs.readFileSync()`
    // translates it to FEFF, the UTF-16 BOM.
    if (content.charCodeAt(0) === 0xFEFF) return content.slice(1)
    return content
  }
649798
def read_bom(data):
    
    # common case is no BOM, so this is fast
    if data and data[0] in _FIRST_CHARS:
        for bom, encoding in _BOM_TABLE:
            if data.startswith(bom):
                return encoding, bom
    return None, None
290018
def decode(data):
    

    if isinstance(data, unicode):
        return data

    # Detect standard unicode BOMs.
    for bom, encoding in UNICODE_BOMS:
        if data.startswith(bom):
            return data[len(bom):].decode(encoding, errors='ignore')

    # Try straight UTF-8.
    try:
        return data.decode('utf-8')
    except UnicodeDecodeError:
        pass

    # Test for various common encodings.
    for encoding in COMMON_ENCODINGS:
        try:
            return data.decode(encoding)
        except UnicodeDecodeError:
            pass

    # Anything else gets filtered.
    return NON_ASCII_FILTER.sub('', data).decode('ascii', errors='replace')
1286754
def determine_encoding(buf):
    
    # The ultimate default is utf8; bravo, W3C
    bom_encoding = 'UTF-8'

    if not buf:
        # What
        return bom_encoding

    if isinstance(buf, six.text_type):
        # We got a file that, for whatever reason, produces already-decoded
        # text.  Check for the BOM (which is useless now) and believe
        # whatever's in the @charset.
        if buf[0] == '\ufeff':
            buf = buf[0:]

        # This is pretty similar to the code below, but without any encoding
        # double-checking.
        charset_start = '@charset "'
        charset_end = '";'
        if buf.startswith(charset_start):
            start = len(charset_start)
            end = buf.index(charset_end, start)
            return buf[start:end]
        else:
            return bom_encoding

    # BOMs
    if buf[:3] == b'\xef\xbb\xbf':
        bom_encoding = 'UTF-8'
        buf = buf[3:]
    if buf[:4] == b'\x00\x00\xfe\xff':
        bom_encoding = 'UTF-32BE'
        buf = buf[4:]
    elif buf[:4] == b'\xff\xfe\x00\x00':
        bom_encoding = 'UTF-32LE'
        buf = buf[4:]
    if buf[:4] == b'\x00\x00\xff\xfe':
        raise UnicodeError("UTF-32-2143 is not supported")
    elif buf[:4] == b'\xfe\xff\x00\x00':
        raise UnicodeError("UTF-32-2143 is not supported")
    elif buf[:2] == b'\xfe\xff':
        bom_encoding = 'UTF-16BE'
        buf = buf[2:]
    elif buf[:2] == b'\xff\xfe':
        bom_encoding = 'UTF-16LE'
        buf = buf[2:]

    # The spec requires exactly this syntax; no escapes or extra spaces or
    # other shenanigans, thank goodness.
    charset_start = '@charset "'.encode(bom_encoding)
    charset_end = '";'.encode(bom_encoding)
    if buf.startswith(charset_start):
        start = len(charset_start)
        end = buf.index(charset_end, start)
        encoded_encoding = buf[start:end]
        encoding = encoded_encoding.decode(bom_encoding)

        # Ensure that decoding with the specified encoding actually produces
        # the same @charset rule
        encoded_charset = buf[:end + len(charset_end)]
        if (encoded_charset.decode(encoding) !=
                encoded_charset.decode(bom_encoding)):
            raise UnicodeError(
                "@charset {0} is incompatible with detected encoding {1}"
                .format(bom_encoding, encoding))
    else:
        # With no @charset, believe the BOM
        encoding = bom_encoding

    return encoding
692492
def guess_file_encoding(fh, default=DEFAULT_ENCODING):
    """"""
    start = fh.tell()
    detector = chardet.UniversalDetector()
    while True:
        data = fh.read(1024 * 10)
        if not data:
            detector.close()
            break
        detector.feed(data)
        if detector.done:
            break

    fh.seek(start)
    return normalize_result(detector.result, default=default)
1056709
def _check_utf8_encoding(self, bstring):
        
        if not gf.is_bytes(bstring):
            self._failed(u"The given string is not a sequence of bytes")
            return
        if not gf.is_utf8_encoded(bstring):
            self._failed(u"The given string is not encoded in UTF-8.")
275482
def _utf_strip_bom(self, encoding):
        """"""

        if encoding is None:
            pass
        elif encoding.lower() == 'utf-8':
            encoding = 'utf-8-sig'
        elif encoding.lower().startswith('utf-16'):
            encoding = 'utf-16'
        elif encoding.lower().startswith('utf-32'):
            encoding = 'utf-32'
        return encoding
89986
function( /*String*/ path, /*String?*/ encoding) {
					path = frontSlash(path);
					if (encoding === 'utf-8') {
						encoding = 'utf8';
					}
					if (!encoding) {
						encoding = 'utf8';
					}

					var text = cache[path].contents + "";
					//Hmm, would not expect to get A BOM, but it seems to happen,
					//remove it just in case.
					if (text.indexOf('\uFEFF') === 0) {
						text = text.substring(1, text.length);
					}


					return text;
				}
393818
def detect_encoding(fp, default=None):
    
    init_pos = fp.tell()
    try:
        sample = fp.read(
            current_app.config.get('PREVIEWER_CHARDET_BYTES', 1024))
        # Result contains 'confidence' and 'encoding'
        result = cchardet.detect(sample)
        threshold = current_app.config.get('PREVIEWER_CHARDET_CONFIDENCE', 0.9)
        if result.get('confidence', 0) > threshold:
            return result.get('encoding', default)
        else:
            return default
    except Exception:
        current_app.logger.warning('Encoding detection failed.', exc_info=True)
        return default
    finally:
        fp.seek(init_pos)
199760
def decode(string, encoding=None, errors=None):
    
    if encoding is None:
        encoding = getpreferredencoding()
    if errors is None:
        errors = getpreferrederrors()
    return string.decode(encoding, errors)
386004
def has_bom(self, filestream):
        """"""

        content = filestream.read(4)
        if content == b'PK\x03\x04':
            # Zip file found.
            # Return `BINARY_ENCODE` as content is binary type,
            # but don't return None which means we don't know what we have.
            return filters.BINARY_ENCODE
        # We only handle zip files, so if we are checking this, we've already failed.
        return None