Utility to properly transliterate text.
547497
def transliterate(text):
    
    text = unidecode(six.text_type(text))
    text = text.replace('@', 'a')
    return text
1105121
def slugify(text, sep='-'):
    """"""
    text = stringify(text)
    if text is None:
        return None
    text = text.replace(sep, WS)
    text = normalize(text, ascii=True)
    if text is None:
        return None
    return text.replace(WS, sep)
904623
def _transliterate (self, text, outFormat):
        
        result = []
        text = self._preprocess(text)
        i = 0
        while i < len(text):
            if text[i].isspace(): 
                result.append(text[i])
                i = i+1
            else: 
                chr = self._getNextChar(text, i)
                try:
                    result.append(self[chr].unichr)
                except KeyError:
                    result.append(_unrecognised(chr))
                i = i + len(chr)
        return result
455976
def _strify(s):
    """"""
    if s is None:
        return None
    if isinstance(s, unicode):
        return s.encode("utf-8")
    return str(s)
108014
def transcribe(self, text, as_phonemes = False):
		'''
		
		'''
		phoneme_words = [self.transcribe_word(word) for word in self._tokenize(text)]
		if not as_phonemes:
			words = [''.join([phoneme.ipa for phoneme in word]) for word in phoneme_words]
			return ' '.join(words)
		else:
			return phoneme_words
803568
public static function transliterate($txt)
    {
        if ($txt === null) {
            return null;
        }
        return strtolower(\Patchwork\Utf8::toAscii((string) $txt));
    }
539511
def iter_text_fixups(data, encoding):
    """"""

    yield data
    if encoding == Encoding.UTF16BE:
        # wrong termination
        yield data + b"\x00"
    elif encoding == Encoding.UTF16:
        # wrong termination
        yield data + b"\x00"
        # utf-16 is missing BOM, content is usually utf-16-le
        yield codecs.BOM_UTF16_LE + data
        # both cases combined
        yield codecs.BOM_UTF16_LE + data + b"\x00"
1085129
def normalize_text(text: str) -> str:
    
    return ' '.join([token
                     for token in text.lower().strip(STRIPPED_CHARACTERS).split()
                     if token not in IGNORED_TOKENS])
1071861
static function translit($string)
    {
        if (!isset(self::$translit_table)){
            self::$translit_table = array(
                'А' => 'A',		'Б' => 'B',		'В' => 'V',		'Г' => 'G',		'Д' => 'D',
                'Е' => 'E',		'Ё' => 'YO',	'Ж' => 'ZH',	'З' => 'Z',		'И' => 'I',
                'Й' => 'J',		'К' => 'K',		'Л' => 'L',		'М' => 'M',		'Н' => 'N',
                'О' => 'O',		'П' => 'P',		'Р' => 'R',		'С' => 'S',		'Т' => 'T',
                'У' => 'U',		'Ф' => 'F',		'Х' => 'H',		'Ц' => 'C',		'Ч' => 'CH',
                'Ш' => 'SH',	'Щ' => 'CSH',	'Ь' => '',		'Ы' => 'Y',		'Ъ' => '',
                'Э' => 'E',		'Ю' => 'YU',	'Я' => 'YA',
                'а' => 'a',		'б' => 'b',		'в' => 'v',		'г' => 'g',		'д' => 'd',
                'е' => 'e',		'ё' => 'yo',	'ж' => 'zh',	'з' => 'z',		'и' => 'i',
                'й' => 'j',		'к' => 'k',		'л' => 'l',		'м' => 'm',		'н' => 'n',
                'о' => 'o',		'п' => 'p',		'р' => 'r',		'с' => 's',		'т' => 't',
                'у' => 'u',		'ф' => 'f',		'х' => 'h',		'ц' => 'c',		'ч' => 'ch',
                'ш' => 'sh',	'щ' => 'csh',	'ь' => '',		'ы' => 'y',		'ъ' => '',
                'э' => 'e',		'ю' => 'yu',	'я' => 'ya',
            );
        }
        return @iconv('UTF-8', 'ASCII//TRANSLIT//IGNORE', str_replace(array_keys(self::$translit_table), array_values(self::$translit_table), $string));
    }
498639
def transliterate(self, target_language="en"):
    """"""
    return WordList([w.transliterate(target_language) for w in self.words],
                     language=target_language, parent=self)
540050
def ensure_text(s, encoding='utf-8', errors='strict'):
    
    if isinstance(s, binary_type):
        return s.decode(encoding, errors)
    elif isinstance(s, text_type):
        return s
    else:
        raise TypeError("not expecting type '%s'" % type(s))
944854
def raw(text):
    """"""
    new_string = ''
    for char in text:
        try:
            new_string += escape_dict[char]
        except KeyError:
            new_string += char
    return new_string
19849
def ensure_str(text):
    """
    if isinstance(text, unicode):
        try:
            return text.encode(pyreadline_codepage, u"replace")
        except (LookupError, TypeError):
            return text.encode(u"ascii", u"replace")
    return text
32232
def normalizeUnicode(text, encoding='humanascii'):
    
    if text == "":
	return ""

    unicodeinput = True
    if not isinstance(text, unicode):
        text = unicode(text, 'utf-8')
        unicodeinput = False

    res = ''
    global allowed, allowedid
    if encoding == 'humanascii' or encoding == 'identifier':
        enc = 'ascii'
    else:
        enc = encoding
    for ch in text:
        if (encoding == 'humanascii') and (ch in allowed):
            # ASCII chars, digits etc. stay untouched
            res += ch
            continue
        if (encoding == 'identifier') and (ch in allowedid):
            # ASCII chars, digits etc. stay untouched
            res += ch
            continue
        else:
            try:
                ch.encode(enc,'strict')
                if encoding == 'identifier':
                    res += '_'
                else:
                    res += ch
            except UnicodeEncodeError:
                ordinal = ord(ch)
                if mapping.has_key(ordinal):
                    # try to apply custom mappings
                    res += mapping.get(ordinal)
                elif decomposition(ch) or len(normalize('NFKD',ch)) > 1:
                    normalized = filter(lambda i: not combining(i), normalize('NFKD', ch)).strip()
                    # normalized string may contain non-letter chars too. Remove them
                    # normalized string may result to  more than one char
                    if encoding == 'identifier':
                        res += ''.join([c for c in normalized if c in allowedid])
                    else:
                        res += ''.join([c for c in normalized if c in allowed])
                else:
                    # hex string instead of unknown char
                    res += "%x" % ordinal
    if encoding == 'identifier':
        res = res.strip('_').replace('_____','_').replace('____','_').replace('___','_').replace('__','_')
        if not res.strip('_')[0] in string.ascii_letters:
            res = '_' + res
    if unicodeinput:
        return res
    else:
        return res.encode('utf-8')
1096340
def translate_text(estimator, subtokenizer, txt):
  """"""
  encoded_txt = _encode_and_add_eos(txt, subtokenizer)

  def input_fn():
    ds = tf.data.Dataset.from_tensors(encoded_txt)
    ds = ds.batch(_DECODE_BATCH_SIZE)
    return ds

  predictions = estimator.predict(input_fn)
  translation = next(predictions)["outputs"]
  translation = _trim_and_decode(translation, subtokenizer)
  print("Translation of \"%s\": \"%s\"" % (txt, translation))
641005
def to_unicode(text):
    """"""
    if isinstance(text, unicode):
        return text
    if isinstance(text, six.string_types):
        return decode_to_unicode(text)
    return unicode(text)
59905
def unicode2ascii(text, expand=True):
    
    """
    translate = UTF8_TO_ASCII if not expand else UTF8_TO_MULTIASCII
    output = ''
    for c in text:
        if not c or ord(c) < 128:
            output += c
        else:
            output += translate[c] if c in translate else ' '
    return output.strip()