Replace certain unicode characters to avoid errors when trying
    to read various strings.
    Returns
    ----------
    str
965102
def replace_chars(astring):
    
    for k, v in CHAR_REPLACE.items():
        astring = astring.replace(k, v)
    return astring
1192445
def safely_decode(unicode_or_str, encoding='utf-8'):
  '''  '''
  if isinstance(unicode_or_str, unicode):
    ustr = unicode_or_str
  elif isinstance(unicode_or_str, str):
    ustr = unicode_or_str.decode(encoding, 'ignore')
  else:
    raise Exception(u'Not of type unicode or str')

  return ustr
760996
def to_unicode(text):
    
    # both str and unicode inherit from basestring
    if not isinstance(text, basestring):
        tmpl = 'expected UTF-8 encoded string or unicode, got %s value %s'
        raise TypeError(tmpl % (type(text), text))
    # return unicode strings unchanged
    if isinstance(text, unicode):
        return text
    # otherwise assume UTF-8 encoding, which also works for ASCII
    return unicode(text, 'utf-8')
221530
def to_utf8(str_or_unicode):
    
    if not isinstance(str_or_unicode, six.text_type):
        return str_or_unicode.encode("utf-8", "ignore")
    return str(str_or_unicode)
993880
def unicode_escape_sequence_fix(self, value):
        

        def fix_fn(match):
            # we don't escape an escaped backslash
            if match.group(0) == r"\\":
                return r"\\"
            return match.group(0).encode("utf-8").decode("unicode-escape")

        return re.sub(r"\\\\|\\u\w{4}|\\U\w{8}|\\N\{([^}\\]|\\.)+\}", fix_fn, value)
12167
def convert_unicode(text):
	
	if isPython2:
		h = HTMLParser()
		s = h.unescape(text)
	else:
		try:
			s = unescape(text)
		except Exception:
			# Python 3.3 and below
			# https://stackoverflow.com/a/2360639/2295672
			s = HTMLParser().unescape(text)
	return s
495562
def ensureUtf(s, encoding='utf8'):
    
    # In Python2, str == bytes.
    # In Python3, bytes remains unchanged, but str means unicode
    # while unicode is not defined anymore
    if type(s) == bytes:
        return s.decode(encoding, 'ignore')
    else:
        return s
448294
def unicode_safe(text, encoding=UrlEncoding, errors='ignore'):
    """"""
    try:
        text_type = unicode
    except NameError:
        text_type = str
    if isinstance(text, text_type):
        return text
    return text.decode(encoding, errors)
1291654
def sanitize_string(string_or_unicode):
    
    if isinstance(string_or_unicode, unicode):
        return string_or_unicode.strip()
    elif isinstance(string_or_unicode, str):
        return string_or_unicode.decode('utf-8').strip()
    else:  # e.g. if input is None
        return u''
414458
def remove_line_breaks(text):
    
    return unicode(text, 'utf-8').replace('\f', '').replace('\n', '') \
        .replace('\r', '').replace(u'\xe2\x80\xa8', '') \
        .replace(u'\xe2\x80\xa9', '').replace(u'\xc2\x85', '') \
        .encode('utf-8')
812395
def Read(self):
    
    encoded_string = self._file_object.readline()

    if isinstance(encoded_string, py2to3.UNICODE_TYPE):
      return encoded_string

    try:
      string = codecs.decode(encoded_string, self._encoding, self._errors)
    except UnicodeDecodeError:
      if self._errors == 'strict':
        logger.error(
            'Unable to properly read input due to encoding error. '
            'Switching to error tolerant encoding which can result in '
            'non Basic Latin (C0) characters to be replaced with "?" or '
            '"\\ufffd".')
        self._errors = 'replace'

      string = codecs.decode(encoded_string, self._encoding, self._errors)

    return string
634882
def text(self):
        
        t = self.t
        if t is None:
            return u''
        text = t.text
        return to_unicode(text) if text is not None else u''
686937
def _clean_str(self, s):
        
        return s.translate(str.maketrans('', '', punctuation)).replace('\u200b', " ").strip().lower()
598135
def to_unicode(text):
    
    if not isinstance(text, str):
        tmpl = 'expected unicode string, got %s value %s'
        raise TypeError(tmpl % (type(text), text))
    return text
642843
async def load_unicode(reader):
    
    ivalue = await load_uvarint(reader)
    fvalue = bytearray(ivalue)
    await reader.areadinto(fvalue)
    return str(fvalue, 'utf8')
799804
def str_or_unicode(text):
    
    encoding = sys.stdout.encoding
    if sys.version_info > (3, 0):
        return text.encode(encoding).decode(encoding)
    return text.encode(encoding)
667373
def entity_to_unicode(string):
    
    # Selected character replacements that have been seen
    replacements = []
    replacements.append((r"&alpha;", u"\u03b1"))
    replacements.append((r"&beta;", u"\u03b2"))
    replacements.append((r"&gamma;", u"\u03b3"))
    replacements.append((r"&delta;", u"\u03b4"))
    replacements.append((r"&epsilon;", u"\u03b5"))
    replacements.append((r"&ordm;", u"\u00ba"))
    replacements.append((r"&iuml;", u"\u00cf"))
    replacements.append((r"&ldquo;", '"'))
    replacements.append((r"&rdquo;", '"'))

    # First, replace numeric entities with unicode
    string = re.sub(r"&#x(....);", repl, string)
    # Second, replace some specific entities specified in the list
    for entity, replacement in replacements:
        string = re.sub(entity, replacement, string)
    return string