Translate non-ASCII characters to spaces or equivalent ASCII characters
488070
def clean_csvs(dialogpath=None):
    
    dialogdir = os.dirname(dialogpath) if os.path.isfile(dialogpath) else dialogpath
    filenames = [dialogpath.split(os.path.sep)[-1]] if os.path.isfile(dialogpath) else os.listdir(dialogpath)
    for filename in filenames:
        filepath = os.path.join(dialogdir, filename)
        df = clean_df(filepath)
        df.to_csv(filepath, header=None)
    return filenames
171859
def _to_ascii(s):
    
    # TODO: Always use unicode within ambry.
    from six import text_type, binary_type
    if isinstance(s, text_type):
        ascii_ = s.encode('ascii', 'ignore')
    elif isinstance(s, binary_type):
        ascii_ = s.decode('utf-8').encode('ascii', 'ignore')
    else:
        raise Exception('Unknown text type - {}'.format(type(s)))
    return ascii_
857802
def xtob(data, sep=''):
    """"""
    # remove the non-hex characters
    data = re.sub("[^0-9a-fA-F]", '', data)

    # interpret the hex
    return binascii.unhexlify(data)
1005337
def _get_ascii_field(self, values):
        ''''''

        empty_space = ' '

        # This formats as field[x][y]
        field = [[empty_space for y in range(max(values) + 1)] for x in range(len(values))]

        # Draw graph into field
        for x in range(len(values)):
            y = values[x]
            y_prev = values[x - 1] if x - 1 in range(len(values)) else y
            y_next = values[x + 1] if x + 1 in range(len(values)) else y
            # Fill empty space
            if abs(y_prev - y) > 1:
                # Fill space between y and y_prev
                step = 1 if y_prev - y > 0 else -1

                # We don't want the first item to be inclusive, so we use step instead of y+1 since step can be negative
                for h in range(y + step, y_prev, step):
                    if field[x][h] is empty_space:
                        field[x][h] = '|'

            # Assign the character to be placed into the graph
            char = self._assign_ascii_character(y_prev, y, y_next)

            field[x][y] = char
        return field
573762
def p_chars(self, p):
        """"""
        if len(p) == 1:
            p[0] = unicode()
        else:
            p[0] = p[1] + p[2]
578698
def _remove_invalid_char(s):
    """"""

    s = ''.join([i if ord(i) >= 32 and ord(i) < 127 else '' for i in s])
    s = s.translate(dict.fromkeys(map(ord, "_%~#\\{}\":")))
    return s
303747
function toAscii(hex) {
  var str = ''; // eslint-disable-line
  var i = 0, l = hex.length; // eslint-disable-line

  if (hex.substring(0, 2) === '0x') {
    i = 2;
  }

  for (; i < l; i += 2) {
    const code = parseInt(hex.substr(i, 2), 16);
    str += String.fromCharCode(code);
  }

  return str;
}
509912
def strip_non_ascii(string):
    ''' '''
    stripped = (c for c in string if 0 < ord(c) < 127)
    return ''.join(stripped)
400195
def strip_spaces(value, sep=None, join=True):
    """"""
    value = value.strip()
    value = [v.strip() for v in value.split(sep)]
    join_sep = sep or ' '
    return join_sep.join(value) if join else value
565681
def _translate(unistr, table):
    ''''''
    if type(unistr) is str:
        try:
            unistr = unistr.decode('utf-8')
        # Python 3 returns AttributeError when .decode() is called on a str
        # This means it is already unicode.
        except AttributeError:
            pass
    try:
        if type(unistr) is not unicode:
            return unistr
    # Python 3 returns NameError because unicode is not a type.
    except NameError:
        pass

    chars = []
    for c in unistr:
        replacement = table.get(c)
        chars.append(replacement if replacement else c)
    return u''.join(chars)
1289752
public static String toASCII(String s, int flags) {
    String[] parts = SEPARATORS_REGEX.split(s);
    for (int i = 0; i < parts.length; i++) {
      if (nonASCII(parts[i])) {
        parts[i] = PUNYCODE_PREFIX + encode(parts[i]);
      }
    }
    return String.join(".", parts);
  }
953050
def convert(word):
    """"""
    if six.PY2:
        if isinstance(word, unicode):
            return word.encode('utf-8')
        else:
            return word.decode('utf-8').encode('utf-8') # make sure it is real utf8, otherwise complain
    else: # ==> Py3
        if isinstance(word, bytes):
            return word.decode('utf-8') # bytes must be in utf8
        return word
374597
def _remove_nonascii(self, df):
    """"""

    df_copy = df.copy(deep=True)
    for col in df_copy.columns:
      if (df_copy[col].dtype == np.dtype('O')):
        df_copy[col] = df[col].apply(
          lambda x: re.sub(r'[^\x00-\x7f]', r'', x) if isinstance(x, six.string_types) else x)

    return df_copy
1116292
def full_process(s, force_ascii=False):
    """"""

    if force_ascii:
        s = asciidammit(s)
    # Keep only Letters and Numbers (see Unicode docs).
    string_out = StringProcessor.replace_non_letters_non_numbers_with_whitespace(s)
    # Force into lowercase.
    string_out = StringProcessor.to_lower_case(string_out)
    # Remove leading and trailing whitespaces.
    string_out = StringProcessor.strip(string_out)
    return string_out
807559
def to_safe_str(s):
    
    # TODO: This is insufficient as it doesn't do anything for other non-ascii chars
    return re.sub(r'[^0-9a-zA-Z]+', '_', s.strip().replace(u'ğ', 'g').replace(u'ö', 'o').replace(
        u'ç', 'c').replace(u'Ç','c').replace(u'Ö', u'O').replace(u'Ş', 's').replace(
        u'Ü', 'u').replace(u'ı', 'i').replace(u'İ','i').replace(u'Ğ', 'g').replace(
        u'ö', 'o').replace(u'ş', 's').replace(u'ü', 'u').lower(), re.UNICODE)
341704
def latinize_text(text, ascii=False):
    
    if text is None or not isinstance(text, six.string_types) or not len(text):
        return text

    if ascii:
        if not hasattr(latinize_text, '_ascii'):
            # Transform to latin, separate accents, decompose, remove
            # symbols, compose, push to ASCII
            latinize_text._ascii = Transliterator.createInstance('Any-Latin; NFKD; [:Symbol:] Remove; [:Nonspacing Mark:] Remove; NFKC; Accents-Any; Latin-ASCII')  # noqa
        return latinize_text._ascii.transliterate(text)

    if not hasattr(latinize_text, '_tr'):
        latinize_text._tr = Transliterator.createInstance('Any-Latin')
    return latinize_text._tr.transliterate(text)
134624
def tmpl_asciify(text):
        
        ger_umlaute = {'ae': u'ä',
                       'oe': u'ö',
                       'ue': u'ü',
                       'Ae': u'Ä',
                       'Oe': u'Ö',
                       'Ue': u'Ü'}
        for replace, search in ger_umlaute.items():
            text = text.replace(search, replace)
        return str(unidecode(text).replace('[?]', ''))