Detect encoding of a byte string sample.
491730
def detect_encoding(sample, encoding=None):
    
    # To reduce tabulator import time
    from cchardet import detect
    if encoding is not None:
        return normalize_encoding(sample, encoding)
    result = detect(sample)
    confidence = result['confidence'] or 0
    encoding = result['encoding'] or 'ascii'
    encoding = normalize_encoding(sample, encoding)
    if confidence < config.ENCODING_CONFIDENCE:
        encoding = config.DEFAULT_ENCODING
    if encoding == 'ascii':
        encoding = config.DEFAULT_ENCODING
    return encoding
604537
private static byte[] checkIllegalCharacters(final byte[] bytes) {
        final ByteArrayOutputStream out = new ByteArrayOutputStream();
        for (final byte byt: bytes) {
            if (BASE64CHARS.indexOf(byt) < 0) {
                out.write(byt);
            }
        }
        return out.toByteArray();
    }
244872
function (bytes) {

    var str = "";
    var code, b1, b2, b3, b4, upper, lower;
    var i = 0;

    while (i < bytes.length) {

        b1 = bytes[i++];

        if (b1 < 0x80) {
            // 1 byte
            code = b1;
        } else if ((b1 >> 5) === 0x06) {
            // 2 bytes
            b2 = bytes[i++];
            code = ((b1 & 0x1f) << 6) | (b2 & 0x3f);
        } else if ((b1 >> 4) === 0x0e) {
            // 3 bytes
            b2 = bytes[i++];
            b3 = bytes[i++];
            code = ((b1 & 0x0f) << 12) | ((b2 & 0x3f) << 6) | (b3 & 0x3f);
        } else {
            // 4 bytes
            b2 = bytes[i++];
            b3 = bytes[i++];
            b4 = bytes[i++];
            code = ((b1 & 0x07) << 18) | ((b2 & 0x3f) << 12) | ((b3 & 0x3f) << 6) | (b4 & 0x3f);
        }

        if (code < 0x10000) {
            str += String.fromCharCode(code);
        } else {
            // surrogate pair
            code -= 0x10000;
            upper = (0xD800 | (code >> 10));
            lower = (0xDC00 | (code & 0x3FF));
            str += String.fromCharCode(upper, lower);
        }
    }

    return str;
}
584144
def bytes_to_str(s, encoding='utf-8'):
    """"""
    if six.PY3 and isinstance(s, bytes):
        return s.decode(encoding)
    return s
167355
private String decode(byte[] data, String encoding) {

        try {
            return new String(data, encoding);
        } catch (UnsupportedEncodingException e) {
            LOG.warn(e.getLocalizedMessage(), e);
            return new String(data);
        }
    }
437552
def win_encode(s):
    """"""
    if isinstance(s, unicode):
        return s.encode(locale.getpreferredencoding(False))
    elif isinstance(s, bytes):
        return s
    elif s is not None:
        raise TypeError('Expected bytes or text, but got %r' % (s,))
955203
function (bytes) {
                var s = '';
                for (var i = 0, l = bytes.length; i < l; i++) {
                    s += String.fromCharCode(bytes[i]);
                }
                return s;
            }
279265
def text(value, encoding="utf-8", errors="strict"):
    """"""
    if isinstance(value, text_type):
        return value
    elif isinstance(value, bytes):
        return text_type(value, encoding, errors)
    else:
        return text_type(value)
650324
def b(s):
	

	return s if isinstance(s, bytes) else s.encode(locale.getpreferredencoding())
237447
def guess_encoding(request):
    
    ctype = request.headers.get('content-type')
    if not ctype:
        # we don't have a content-type, somehow, so...
        LOGGER.warning("%s: no content-type; headers are %s",
                       request.url, request.headers)
        return 'utf-8'

    # explicit declaration
    match = re.search(r'charset=([^ ;]*)(;| |$)', ctype)
    if match:
        return match[1]

    # html default
    if ctype.startswith('text/html'):
        return 'iso-8859-1'

    # everything else's default
    return 'utf-8'
369106
def byte_str(s="", encoding="utf-8", input_encoding="utf-8", errors="strict"):
    
    assert isinstance(s, basestring)
    if isinstance(s, unicode):
        return s.encode(encoding, errors)
    if s and encoding != input_encoding:
        return s.decode(input_encoding, errors).encode(encoding, errors)
    return s
1056709
def _check_utf8_encoding(self, bstring):
        
        if not gf.is_bytes(bstring):
            self._failed(u"The given string is not a sequence of bytes")
            return
        if not gf.is_utf8_encoded(bstring):
            self._failed(u"The given string is not encoded in UTF-8.")
372687
def autodecode(self, a_bytes):
        
        try:  # 如果装了chardet
            analysis = chardet.detect(a_bytes)
            if analysis["confidence"] >= 0.75:  # 如果可信
                return (self.decode(a_bytes, analysis["encoding"])[0],
                        analysis["encoding"])
            else:  # 如果不可信, 打印异常
                raise Exception("Failed to detect encoding. (%s, %s)" % (
                    analysis["confidence"],
                    analysis["encoding"]))
        except NameError:  # 如果没有装chardet
            print(
                "Warning! chardet not found. Use utf-8 as default encoding instead.")
            return (a_bytes.decode("utf-8")[0],
                    "utf-8")
391992
function read_raw_string(buf, encoding) {
  var len = buf.readUInt16BE();
  return buf.toString(encoding || 'utf8', len);
}
801647
def decode_if_bytes(obj, mode=True):
    """"""
    if mode is True:
        mode = unicode_errors_default
    if isinstance(obj, bytes):
        return obj.decode("utf-8", errors=mode)
    return obj
226040
def _detect_encoding(self, source_file):
        """"""

        encoding = self._guess(source_file)
        # If we didn't explicitly detect an encoding, assume default.
        if encoding is None:
            encoding = self.default_encoding

        return encoding
528307
function detectEncoding(label, input_stream) {
      if (input_stream.match([0xFF, 0xFE])) {
        input_stream.offset(2);
        return 'utf-16';
      }
      if (input_stream.match([0xFE, 0xFF])) {
        input_stream.offset(2);
        return 'utf-16be';
      }
      if (input_stream.match([0xEF, 0xBB, 0xBF])) {
        input_stream.offset(3);
        return 'utf-8';
      }
      return label;
    }