Convert a file to plain text.

        Useful for full-text indexing. Returns a Unicode string.
423965
def to_text(self, digest, blob, mime_type):
        
        # Special case, for now (XXX).
        if mime_type.startswith("image/"):
            return ""

        cache_key = "txt:" + digest

        text = self.cache.get(cache_key)
        if text:
            return text

        # Direct conversion possible
        for handler in self.handlers:
            if handler.accept(mime_type, "text/plain"):
                text = handler.convert(blob)
                self.cache[cache_key] = text
                return text

        # Use PDF as a pivot format
        pdf = self.to_pdf(digest, blob, mime_type)
        for handler in self.handlers:
            if handler.accept("application/pdf", "text/plain"):
                text = handler.convert(pdf)
                self.cache[cache_key] = text
                return text

        raise HandlerNotFound(f"No handler found to convert from {mime_type} to text")
854979
def infer_delimiter(filepath):
    
    ext = os.path.splitext(filepath)[1][1:].lower()
    return {"txt": "\t", "tsv": "\t", "csv": ","}.get(ext)
152617
def html_to_plain_text(html):
    """"""
    # Use BeautifulSoup to normalize the html
    soup = BeautifulSoup(html, "html.parser")
    # Init the parser
    parser = HTML2PlainParser()
    parser.feed(str(soup.encode('utf-8')))
    # Strip the end of the plain text
    result = parser.text.rstrip()
    # Add footnotes
    if parser.links:
        result += '\n\n'
        for link in parser.links:
            result += '[{}]: {}\n'.format(link[0], link[1])
    return result
11985
def file_to_string(path):
    
    if not os.path.exists(path):
        ui.error(c.MESSAGES["path_missing"], path)
        sys.exit(1)

    with codecs.open(path, "r", "UTF-8") as contents:
        return contents.read()
956520
def _read_from_file(self):
        
        # test if we can read the given file
        if not gf.file_can_be_read(self.file_path):
            self.log_exc(u"File '%s' cannot be read" % (self.file_path), None, True, OSError)

        if self.file_format not in TextFileFormat.ALLOWED_VALUES:
            self.log_exc(u"Text file format '%s' is not supported." % (self.file_format), None, True, ValueError)

        # read the contents of the file
        self.log([u"Reading contents of file '%s'", self.file_path])
        with io.open(self.file_path, "r", encoding="utf-8") as text_file:
            lines = text_file.readlines()

        # clear text fragments
        self.clear()

        # parse the contents
        map_read_function = {
            TextFileFormat.MPLAIN: self._read_mplain,
            TextFileFormat.MUNPARSED: self._read_munparsed,
            TextFileFormat.PARSED: self._read_parsed,
            TextFileFormat.PLAIN: self._read_plain,
            TextFileFormat.SUBTITLES: self._read_subtitles,
            TextFileFormat.UNPARSED: self._read_unparsed
        }
        map_read_function[self.file_format](lines)

        # log the number of fragments
        self.log([u"Parsed %d fragments", len(self.fragments)])
21635
def set_plain_text(self, text, is_code):
        """"""

        # text is coming from utils.dochelpers.getdoc
        if type(text) is dict:
            name = text['name']
            if name:
                rst_title = ''.join(['='*len(name), '\n', name, '\n',
                                    '='*len(name), '\n\n'])
            else:
                rst_title = ''

            if text['argspec']:
                definition = ''.join(['Definition: ', name, text['argspec'],
                                      '\n'])
            else:
                definition = ''

            if text['note']:
                note = ''.join(['Type: ', text['note'], '\n\n----\n\n'])
            else:
                note = ''

            full_text = ''.join([rst_title, definition, note,
                                 text['docstring']])
        else:
            full_text = text

        self.plain_text.set_text(full_text, is_code)
        self.save_text([self.plain_text.set_text, full_text, is_code])
1173858
def text(self) -> str:
        
        return self.export(output=Mimetypes.PLAINTEXT, exclude=self.default_exclude)
900100
def to_text(value, encoding='utf-8'):
    
    if not value:
        return ''
    if isinstance(value, six.text_type):
        return value
    if isinstance(value, six.binary_type):
        return value.decode(encoding)
    return six.text_type(value)
59905
def unicode2ascii(text, expand=True):
    
    """
    translate = UTF8_TO_ASCII if not expand else UTF8_TO_MULTIASCII
    output = ''
    for c in text:
        if not c or ord(c) < 128:
            output += c
        else:
            output += translate[c] if c in translate else ' '
    return output.strip()
728891
def _GetUtf8Contents(self, file_name):
    """"""
    contents = self._FileContents(file_name)
    if not contents:  # Missing file
      return

    # Check for errors that will prevent csv.reader from working
    if len(contents) >= 2 and contents[0:2] in (codecs.BOM_UTF16_BE,
        codecs.BOM_UTF16_LE):
      self._problems.FileFormat("appears to be encoded in utf-16", (file_name, ))
      # Convert and continue, so we can find more errors
      contents = codecs.getdecoder('utf-16')(contents)[0].encode('utf-8')

    null_index = contents.find('\0')
    if null_index != -1:
      # It is easier to get some surrounding text than calculate the exact
      # row_num
      m = re.search(r'.{,20}\0.{,20}', contents, re.DOTALL)
      self._problems.FileFormat(
          "contains a null in text \"%s\" at byte %d" %
          (codecs.getencoder('string_escape')(m.group()), null_index + 1),
          (file_name, ))
      return

    # strip out any UTF-8 Byte Order Marker (otherwise it'll be
    # treated as part of the first column name, causing a mis-parse)
    contents = contents.lstrip(codecs.BOM_UTF8)
    return contents
760996
def to_unicode(text):
    
    # both str and unicode inherit from basestring
    if not isinstance(text, basestring):
        tmpl = 'expected UTF-8 encoded string or unicode, got %s value %s'
        raise TypeError(tmpl % (type(text), text))
    # return unicode strings unchanged
    if isinstance(text, unicode):
        return text
    # otherwise assume UTF-8 encoding, which also works for ASCII
    return unicode(text, 'utf-8')
826871
def read_plain(file_obj, type_, count):
    """"""
    if count == 0:
        return []
    conv = DECODE_PLAIN[type_]
    return conv(file_obj, count)
477137
def ustr(text):
    
    if text is not None:
        if sys.version_info >= (3, 0):
            return str(text)
        else:
            return unicode(text)
    else:
        return text
1058237
def to_unicode(value):
    
    if isinstance(value, ffi.CData):
        return ffi.string(value).decode('utf-8')
    elif isinstance(value, binary_type):
        return value.decode('utf-8')
    elif isinstance(value, text_type):
        return value
    else:
        raise ValueError('Value must be text, bytes, or char[]')
7622
def _to_unicode(self, data, encoding, errors='strict'):
    ''''''

    # strip Byte Order Mark (if present)
    if (len(data) >= 4) and (data[:2] == '\xfe\xff') and (data[2:4] != '\x00\x00'):
        encoding = 'utf-16be'
        data = data[2:]
    elif (len(data) >= 4) and (data[:2] == '\xff\xfe') and (data[2:4] != '\x00\x00'):
        encoding = 'utf-16le'
        data = data[2:]
    elif data[:3] == '\xef\xbb\xbf':
        encoding = 'utf-8'
        data = data[3:]
    elif data[:4] == '\x00\x00\xfe\xff':
        encoding = 'utf-32be'
        data = data[4:]
    elif data[:4] == '\xff\xfe\x00\x00':
        encoding = 'utf-32le'
        data = data[4:]
    newdata = unicode(data, encoding, errors)
    return newdata
111720
def _read_text(self, filename):
        
        with io.open(filename, 'rt', encoding='utf-8') as f:
            return f.read()
1113722
def _ensure_unicode(text):
    
    if isinstance(text, six.binary_type):
        return text.decode(sys.getfilesystemencoding(), 'replace')
    else:
        return text