Get encoding value from splitted output of file --mime --uncompress.
481948
def get_file_mime_encoding (parts):
    """"""
    for part in parts:
        for subpart in part.split(" "):
            if subpart.startswith("compressed-encoding="):
                mime = subpart.split("=")[1].strip()
                return Mime2Encoding.get(mime)
    return None
1012539
def readme(filename, encoding='utf8'):
    

    with io.open(filename, encoding=encoding) as source:
        return source.read()
615312
def get_text(self):
        """"""
        self._load_raw_content()
        if self._text is None:
            assert self._raw_content is not None
            ret_cont = self._raw_content
            if self.compressed:
                ret_cont = zlib.decompress(ret_cont, zlib.MAX_WBITS+16)
            if self.encoded:
                ret_cont = ret_cont.decode('utf-8')
            self._text = ret_cont
        assert self._text is not None
        return self._text
734893
def unzip_unicode(output, version):
    """"""

    unzipper = zipfile.ZipFile(os.path.join(output, 'unicodedata', '%s.zip' % version))
    target = os.path.join(output, 'unicodedata', version)

    print('Unzipping %s.zip...' % version)

    os.makedirs(target)

    for f in unzipper.namelist():
        # Do I need backslash on windows? Or is it forward as well?
        unzipper.extract(f, target)
552613
def get_mime_representation(self):
        """"""
        part = deepcopy(self.part)
        part.set_param('maxlinelen', '78', header='Content-Disposition')
        return part
896006
def fsencoding(s, encoding=sys.getfilesystemencoding()):
    
    if isinstance(s, unicode):
        s = s.encode(encoding)
    return s
766672
private static int[] readUncompressedLength(byte[] compressed, int compressedOffset)
        throws CorruptionException {
        int result;
        int bytesRead = 0;
        {
            int b = compressed[compressedOffset + bytesRead++] & 0xFF;
            result = b & 0x7f;
            if ((b & 0x80) != 0) {
                b = compressed[compressedOffset + bytesRead++] & 0xFF;
                result |= (b & 0x7f) << 7;
                if ((b & 0x80) != 0) {
                    b = compressed[compressedOffset + bytesRead++] & 0xFF;
                    result |= (b & 0x7f) << 14;
                    if ((b & 0x80) != 0) {
                        b = compressed[compressedOffset + bytesRead++] & 0xFF;
                        result |= (b & 0x7f) << 21;
                        if ((b & 0x80) != 0) {
                            b = compressed[compressedOffset + bytesRead++] & 0xFF;
                            result |= (b & 0x7f) << 28;
                            if ((b & 0x80) != 0) {
                                throw new CorruptionException("last byte of compressed length int has high bit set");
                            }
                        }
                    }
                }
            }
        }
        return new int[] { result, bytesRead };
    }
159559
def toUnicode(s):
    
    if isinstance(s, six.binary_type):
        s = s.decode('utf-8')
    return s
854863
def get_process_output(process, encoding=None):
    """"""

    output = process.communicate()
    returncode = process.returncode

    if not encoding:
        try:
            encoding = sys.stdout.encoding
        except Exception:
            encoding = locale.getpreferredencoding()

    if returncode != 0:
        raise RuntimeError("Runtime Error: %s" % (output[0].rstrip().decode(encoding, errors='replace')))

    return output[0].decode(encoding, errors='replace')
603299
def mime_to_bytes(msg, header_len):
    
    fp = BytesIO()
    g = BytesGenerator(fp, maxheaderlen=header_len)
    g.flatten(msg)
    return fp.getvalue()
226040
def _detect_encoding(self, source_file):
        """"""

        encoding = self._guess(source_file)
        # If we didn't explicitly detect an encoding, assume default.
        if encoding is None:
            encoding = self.default_encoding

        return encoding
745243
def _detect_encoding(stdout):
        
        if hasattr(stdout, 'encoding') and stdout.encoding:
            return stdout.encoding

        if os.environ.get('LANG'):
            encoding = os.environ.get('LANG').split('.')[-1]

            # validate the encoding string
            ret = None
            try:
                ret = codecs.lookup(encoding)
            except LookupError:
                pass
            if ret:
                return encoding

        return locale.getpreferredencoding()
1220945
def EncodeUnicode(text):
  
  if type(text) == type(u''):
    return text.encode(OUTPUT_ENCODING)
  else:
    return text
940549
def _encode_multipart_formdata(self):
        
        def get_content_type(filename):
            "Helper to get MIME type."
            return mimetypes.guess_type(filename)[0] or 'application/octet-stream'

        boundary = '----------ThIs_Is_tHe_b0uNdaRY_%d$' % (time.time())
        logical_lines = []
        for name, value in self.fields:
            if value is None:
                continue
            logical_lines.append('--' + boundary)
            if hasattr(value, "read"):
                filename = getattr(value, "name", str(id(value))+".dat")
                logical_lines.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (
                    name,
                    os.path.basename(filename).replace("'", '_').replace('"', '_')
                ))
                logical_lines.append('Content-Type: %s' % get_content_type(filename))
                logical_lines.append('Content-Transfer-Encoding: binary')
                value = value.read()
            else:
                logical_lines.append('Content-Disposition: form-data; name="%s"' % name)
                logical_lines.append('Content-Type: text/plain; charset="UTF-8"')
                value = fmt.to_utf8(value)
            #logical_lines.append('Content-Length: %d' % len(value))
            logical_lines.append('')
            logical_lines.append(value)
        logical_lines.append('--' + boundary + '--')
        logical_lines.append('')

        body = '\r\n'.join(logical_lines)
        content_type = 'multipart/form-data; boundary=%s' % boundary
        return content_type, body
392636
def detectEncodingMeta(self):
        
        buffer = self.rawStream.read(self.numBytesMeta)
        assert isinstance(buffer, bytes)
        parser = EncodingParser(buffer)
        self.rawStream.seek(0)
        encoding = parser.getEncoding()

        if encoding is not None and encoding.name in ("utf-16be", "utf-16le"):
            encoding = lookupEncoding("utf-8")

        return encoding
343018
def decode_raw_stream(self, text, decode_raw, known_encoding, filename):
        
        if isinstance(text, compat.text_type):
            m = self._coding_re.match(text)
            encoding = m and m.group(1) or known_encoding or 'ascii'
            return encoding, text

        if text.startswith(codecs.BOM_UTF8):
            text = text[len(codecs.BOM_UTF8):]
            parsed_encoding = 'utf-8'
            m = self._coding_re.match(text.decode('utf-8', 'ignore'))
            if m is not None and m.group(1) != 'utf-8':
                raise exceptions.CompileException(
                                "Found utf-8 BOM in file, with conflicting "
                                "magic encoding comment of '%s'" % m.group(1),
                                text.decode('utf-8', 'ignore'),
                                0, 0, filename)
        else:
            m = self._coding_re.match(text.decode('utf-8', 'ignore'))
            if m:
                parsed_encoding = m.group(1)
            else:
                parsed_encoding = known_encoding or 'ascii'

        if decode_raw:
            try:
                text = text.decode(parsed_encoding)
            except UnicodeDecodeError:
                raise exceptions.CompileException(
                        "Unicode decode operation of encoding '%s' failed" %
                        parsed_encoding,
                        text.decode('utf-8', 'ignore'),
                        0, 0, filename)

        return parsed_encoding, text
890514
def read_file(file_name, encoding='utf-8'):
    
    with open(file_name, 'rb') as f:
        data = f.read()

    if encoding is not None:
        data = data.decode(encoding)

    return data