Fetch the decoded text contents of a Unicode encoded Entry.

        Since this should return the text contents from the file
        system, we check to see into what sort of subclass we should
        morph this Entry.
558324
def get_text_contents(self):
        """"""
        try:
            self = self.disambiguate(must_exist=1)
        except SCons.Errors.UserError:
            # There was nothing on disk with which to disambiguate
            # this entry.  Leave it as an Entry, but return a null
            # string so calls to get_text_contents() in emitters and
            # the like (e.g. in qt.py) don't have to disambiguate by
            # hand or catch the exception.
            return ''
        else:
            return self.get_text_contents()
1053691
def _try_decode_list(content):
        
        result = list()
        for value in content:
            result.append(try_utf8_decode(value))
        return result
799804
def str_or_unicode(text):
    
    encoding = sys.stdout.encoding
    if sys.version_info > (3, 0):
        return text.encode(encoding).decode(encoding)
    return text.encode(encoding)
1067917
def unicode_decode(data, encoding_list):
    
    assert encoding_list, 'encodings must not be empty.'

    xs = distinct(encoding_list if isinstance(encoding_list, list) else [encoding_list])
    first_exp = None
    for i, encoding in enumerate(xs):
        try:
            return data.decode(encoding)
        except UnicodeDecodeError as e:
            if i == 0:
                first_exp = e
    raise first_exp
984109
public static Reader getReadEncoding(InputStream is, String encoding)
    throws UnsupportedEncodingException
  {
    return getReadFactory(encoding).create(is);
  }
1212864
def decode_source(source_bytes):
    
    # source_bytes_readline = io.BytesIO(source_bytes).readline
    # encoding, _ = detect_encoding(source_bytes_readline)
    newline_decoder = io.IncrementalNewlineDecoder(None, True)
    return newline_decoder.decode(source_to_unicode(source_bytes))
1225998
def content(self):
        
        content = self._get_content()

        if bool(content and '\0' in content):
            return content
        return safe_unicode(content)
844062
def decode_to_unicode(content):
    
    if content and not isinstance(content, str):
        try:
            # Try to decode ISO-8859-1 to unicode
            return content.decode("ISO-8859-1")
        except UnicodeEncodeError:
            # Assume content is unicode already
            return content
    return content
892886
def decode_input(text_in):
    
    
    if type(text_in) == list:
        text_out = u' '.join([t.decode('utf-8') for t in text_in])
    else:
        text_out = text_in.decode('utf-8')
    return text_out
1318544
def fetch(self, url, timeout=None):
        

        # ISO-8859-1 is the default encoding for text files per the specs for
        # HTTP 1.0 (RFC 1945 sec 3.6.1) and HTTP 1.1 (RFC 2616 sec 3.7.1).
        # ref: http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.7.1
        encoding = "iso-8859-1"
        content = ""
        expires_header = None
        content_type_header = None
        self._response_code = 0
        self._source_url = url

        if self.user_agent:
            req = urllib_request.Request(url, None, 
                                         { 'User-Agent' : self.user_agent })
        else:
            req = urllib_request.Request(url)

        try:
            if timeout:
                f = urllib_request.urlopen(req, timeout=timeout)
            else:
                f = urllib_request.urlopen(req)

            content = f.read(MAX_FILESIZE)
            # As of Python 2.5, f.info() looks like it returns the HTTPMessage
            # object created during the connection. 
            expires_header = f.info().get("expires")
            content_type_header = f.info().get("Content-Type")
            # As of Python 2.4, this file-like object reports the response 
            # code, too. 
            if hasattr(f, "code"):
                self._response_code = f.code
            else:
                self._response_code = 200
            f.close()
        except urllib_error.URLError:
            # This is a slightly convoluted way to get the error instance,
            # but it works under Python 2 & 3. 
            error_instance = sys.exc_info()
            if len(error_instance) > 1:
                error_instance = error_instance[1]
            if hasattr(error_instance, "code"):
                self._response_code = error_instance.code
                
        # MK1996 section 3.4 says, "...robots should take note of Expires 
        # header set by the origin server. If no cache-control directives 
        # are present robots should default to an expiry of 7 days".
        
        # This code is lazy and looks at the Expires header but not 
        # Cache-Control directives.
        self.expiration_date = None
        if self._response_code >= 200 and self._response_code < 300:
            # All's well.
            if expires_header:
                self.expiration_date = email_utils.parsedate_tz(expires_header)
                
                if self.expiration_date:
                    # About time zones -- the call to parsedate_tz() returns a
                    # 10-tuple with the time zone offset in the 10th element. 
                    # There are 3 valid formats for HTTP dates, and one of 
                    # them doesn't contain time zone information. (UTC is 
                    # implied since all HTTP header dates are UTC.) When given
                    # a date that lacks time zone information, parsedate_tz() 
                    # returns None in the 10th element. mktime_tz() interprets
                    # None in the 10th (time zone) element to mean that the 
                    # date is *local* time, not UTC. 
                    # Therefore, if the HTTP timestamp lacks time zone info 
                    # and I run that timestamp through parsedate_tz() and pass
                    # it directly to mktime_tz(), I'll get back a local 
                    # timestamp which isn't what I want. To fix this, I simply
                    # convert a time zone of None to zero. It's much more 
                    # difficult to explain than to fix. =)
                    # ref: http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.3.1
                    if self.expiration_date[9] == None: 
                        self.expiration_date = self.expiration_date[:9] + (0,)
                
                    self.expiration_date = email_utils.mktime_tz(self.expiration_date)
                    if self.use_local_time: 
                        # I have to do a little more converting to get this 
                        # UTC timestamp into localtime.
                        self.expiration_date = time.mktime(time.gmtime(self.expiration_date)) 
                #else:
                    # The expires header was garbage.

        if not self.expiration_date: self.expiration_date = self._now() + SEVEN_DAYS

        if (self._response_code >= 200) and (self._response_code < 300):
            # All's well.
            media_type, encoding = _parse_content_type_header(content_type_header)
            # RFC 2616 sec 3.7.1 -- 
            # When no explicit charset parameter is provided by the sender, 
            # media subtypes  of the "text" type are defined to have a default
            # charset value of "ISO-8859-1" when received via HTTP.
            # http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.7.1
            if not encoding: 
                encoding = "iso-8859-1"
        elif self._response_code in (401, 403):
            # 401 or 403 ==> Go away or I will taunt you a second time! 
            # (according to MK1996)
            content = "User-agent: *\nDisallow: /\n"
        elif self._response_code == 404:
            # No robots.txt ==> everyone's welcome
            content = ""
        else:        
            # Uh-oh. I punt this up to the caller. 
            _raise_error(urllib_error.URLError, self._response_code)

        if ((PY_MAJOR_VERSION == 2) and isinstance(content, str)) or \
           ((PY_MAJOR_VERSION > 2)  and (not isinstance(content, str))):
            # This ain't Unicode yet! It needs to be.
            
            # Unicode decoding errors are another point of failure that I punt 
            # up to the caller.
            try:
                content = content.decode(encoding)
            except UnicodeError:
                _raise_error(UnicodeError,
                "Robots.txt contents are not in the encoding expected (%s)." % encoding)
            except (LookupError, ValueError):
                # LookupError ==> Python doesn't have a decoder for that encoding.
                # One can also get a ValueError here if the encoding starts with 
                # a dot (ASCII 0x2e). See Python bug 1446043 for details. This 
                # bug was supposedly fixed in Python 2.5.
                _raise_error(UnicodeError,
                        "I don't understand the encoding \"%s\"." % encoding)
        
        # Now that I've fetched the content and turned it into Unicode, I 
        # can parse it.
        self.parse(content)
826234
def decode(data, encoding='utf-8', errors='strict'):
    
    data_type = type(data)

    if data_type == bytes:
        return bytes2unicode(data, encoding, errors)
    if data_type in (dict, list, tuple):
        if data_type == dict:
            data = data.items()
        return data_type(map(decode, data))
    return data
728891
def _GetUtf8Contents(self, file_name):
    """"""
    contents = self._FileContents(file_name)
    if not contents:  # Missing file
      return

    # Check for errors that will prevent csv.reader from working
    if len(contents) >= 2 and contents[0:2] in (codecs.BOM_UTF16_BE,
        codecs.BOM_UTF16_LE):
      self._problems.FileFormat("appears to be encoded in utf-16", (file_name, ))
      # Convert and continue, so we can find more errors
      contents = codecs.getdecoder('utf-16')(contents)[0].encode('utf-8')

    null_index = contents.find('\0')
    if null_index != -1:
      # It is easier to get some surrounding text than calculate the exact
      # row_num
      m = re.search(r'.{,20}\0.{,20}', contents, re.DOTALL)
      self._problems.FileFormat(
          "contains a null in text \"%s\" at byte %d" %
          (codecs.getencoder('string_escape')(m.group()), null_index + 1),
          (file_name, ))
      return

    # strip out any UTF-8 Byte Order Marker (otherwise it'll be
    # treated as part of the first column name, causing a mis-parse)
    contents = contents.lstrip(codecs.BOM_UTF8)
    return contents
1113722
def _ensure_unicode(text):
    
    if isinstance(text, six.binary_type):
        return text.decode(sys.getfilesystemencoding(), 'replace')
    else:
        return text
983267
def _parse(self):
        
        cur_ver = None
        cur_line = None
        for line in self.content:
            m = re.match('[^ ]+ \(([0-9]+\.[0-9]+\.[0-9]+)-[0-9]+\) [^ ]+; urgency=[^ ]+', line)
            if m:
                cur_ver = m.group(1)
                self.versions.append(cur_ver)
                self.entries[cur_ver] = []
                cur_entry = self.entries[cur_ver]
                if self.latest_version is None or StrictVersion(cur_ver) > StrictVersion(self.latest_version):
                    self.latest_version = m.group(1)
            elif cur_ver:
                m = re.match('  \* (.*)', line)
                if m:
                    cur_entry.append(m.group(1).strip())
                elif not re.match('$', line) and re.match(' *[^$]+', line):
                    cur_entry[-1] += " " + line.strip()
94558
def decode(self, data: bytes) -> bytes:
        
        if CONTENT_TRANSFER_ENCODING in self.headers:
            data = self._decode_content_transfer(data)
        if CONTENT_ENCODING in self.headers:
            return self._decode_content(data)
        return data
984551
def decode(text):
    
    try:
        if text.startswith(BOM_UTF8):
            # UTF-8 with BOM
            return to_text_string(text[len(BOM_UTF8):], 'utf-8'), 'utf-8-bom'
        elif text.startswith(BOM_UTF16):
            # UTF-16 with BOM
            return to_text_string(text[len(BOM_UTF16):], 'utf-16'), 'utf-16'
        elif text.startswith(BOM_UTF32):
            # UTF-32 with BOM
            return to_text_string(text[len(BOM_UTF32):], 'utf-32'), 'utf-32'
        coding = get_coding(text)
        if coding:
            return to_text_string(text, coding), coding
    except (UnicodeError, LookupError):
        pass
    # Assume UTF-8
    try:
        return to_text_string(text, 'utf-8'), 'utf-8-guessed'
    except (UnicodeError, LookupError):
        pass
    # Assume Latin-1 (behaviour before 3.7.1)
    return to_text_string(text, "latin-1"), 'latin-1-guessed'
812395
def Read(self):
    
    encoded_string = self._file_object.readline()

    if isinstance(encoded_string, py2to3.UNICODE_TYPE):
      return encoded_string

    try:
      string = codecs.decode(encoded_string, self._encoding, self._errors)
    except UnicodeDecodeError:
      if self._errors == 'strict':
        logger.error(
            'Unable to properly read input due to encoding error. '
            'Switching to error tolerant encoding which can result in '
            'non Basic Latin (C0) characters to be replaced with "?" or '
            '"\\ufffd".')
        self._errors = 'replace'

      string = codecs.decode(encoded_string, self._encoding, self._errors)

    return string