Handle unrecognised characters.
183152
def _unrecognised(achr):
    
    if options['handleUnrecognised'] == UNRECOGNISED_ECHO:
        return achr
    elif options['handleUnrecognised'] == UNRECOGNISED_SUBSTITUTE:
        return options['substituteChar']
    else:
        raise KeyError(achr)
283590
def fix_one_step_and_explain(text):
    
    if isinstance(text, bytes):
        raise UnicodeError(BYTES_ERROR_TEXT)
    if len(text) == 0:
        return text, []

    # The first plan is to return ASCII text unchanged.
    if possible_encoding(text, 'ascii'):
        return text, []

    # As we go through the next step, remember the possible encodings
    # that we encounter but don't successfully fix yet. We may need them
    # later.
    possible_1byte_encodings = []

    # Suppose the text was supposed to be UTF-8, but it was decoded using
    # a single-byte encoding instead. When these cases can be fixed, they
    # are usually the correct thing to do, so try them next.
    for encoding in CHARMAP_ENCODINGS:
        if possible_encoding(text, encoding):
            encoded_bytes = text.encode(encoding)
            encode_step = ('encode', encoding, ENCODING_COSTS.get(encoding, 0))
            transcode_steps = []

            # Now, find out if it's UTF-8 (or close enough). Otherwise,
            # remember the encoding for later.
            try:
                decoding = 'utf-8'
                # Check encoded_bytes for sequences that would be UTF-8,
                # except they have b' ' where b'\xa0' would belong.
                if ALTERED_UTF8_RE.search(encoded_bytes):
                    encoded_bytes = restore_byte_a0(encoded_bytes)
                    cost = encoded_bytes.count(0xa0) * 2
                    transcode_steps.append(('transcode', 'restore_byte_a0', cost))

                # Check for the byte 0x1a, which indicates where one of our
                # 'sloppy' codecs found a replacement character.
                if encoding.startswith('sloppy') and 0x1a in encoded_bytes:
                    encoded_bytes = replace_lossy_sequences(encoded_bytes)
                    transcode_steps.append(('transcode', 'replace_lossy_sequences', 0))

                if 0xed in encoded_bytes or 0xc0 in encoded_bytes:
                    decoding = 'utf-8-variants'

                decode_step = ('decode', decoding, 0)
                steps = [encode_step] + transcode_steps + [decode_step]
                fixed = encoded_bytes.decode(decoding)
                return fixed, steps

            except UnicodeDecodeError:
                possible_1byte_encodings.append(encoding)

    # Look for a-hat-euro sequences that remain, and fix them in isolation.
    if PARTIAL_UTF8_PUNCT_RE.search(text):
        steps = [('transcode', 'fix_partial_utf8_punct_in_1252', 1)]
        fixed = fix_partial_utf8_punct_in_1252(text)
        return fixed, steps

    # The next most likely case is that this is Latin-1 that was intended to
    # be read as Windows-1252, because those two encodings in particular are
    # easily confused.
    if 'latin-1' in possible_1byte_encodings:
        if 'windows-1252' in possible_1byte_encodings:
            # This text is in the intersection of Latin-1 and
            # Windows-1252, so it's probably legit.
            return text, []
        else:
            # Otherwise, it means we have characters that are in Latin-1 but
            # not in Windows-1252. Those are C1 control characters. Nobody
            # wants those. Assume they were meant to be Windows-1252. Don't
            # use the sloppy codec, because bad Windows-1252 characters are
            # a bad sign.
            encoded = text.encode('latin-1')
            try:
                fixed = encoded.decode('windows-1252')
                steps = []
                if fixed != text:
                    steps = [('encode', 'latin-1', 0),
                             ('decode', 'windows-1252', 1)]
                return fixed, steps
            except UnicodeDecodeError:
                # This text contained characters that don't even make sense
                # if you assume they were supposed to be Windows-1252. In
                # that case, let's not assume anything.
                pass

    # The cases that remain are mixups between two different single-byte
    # encodings, and not the common case of Latin-1 vs. Windows-1252.
    #
    # These cases may be unsolvable without adding false positives, though
    # I have vague ideas about how to optionally address them in the future.

    # Return the text unchanged; the plan is empty.
    return text, []
215743
def unescape_single_character(match):
    
    try:
        return DECODE_MAP[match.group(0)]
    except KeyError:
        value = match.group(0)
        if value == '\\':
            raise ValueError("Unterminated escape sequence encountered")

        raise ValueError(
            "Unrecognized escape sequence encountered: {}".format(value))
1292063
def handle_identifier_qualified(self, line: str, position: int, tokens: ParseResults) -> ParseResults:
        """"""
        namespace, name = tokens[NAMESPACE], tokens[NAME]

        self.raise_for_missing_namespace(line, position, namespace, name)
        self.raise_for_missing_name(line, position, namespace, name)

        return tokens
1220573
def clean_chars(value):
    ""
    value = value.replace('\xb9', ' ')
    value = value.replace('\xf8', ' ')
    value = value.replace('\xab', ' ')
    value = value.replace('\xa7', ' ')
    value = value.replace('\xa8', ' ')
    value = value.replace('\xfb', ' ')
    value = value.replace('\xfc', ' ')
    return value
613490
def sanitize(self, name, valid_fn):
        ""
        invalid = {'\a':'a','\b':'b', '\v':'v','\f':'f','\r':'r'}
        for cc in filter(lambda el: el in name, invalid.keys()):
            raise Exception("Please use a raw string or escape control code '\%s'"
                            % invalid[cc])
        sanitized, chars = [], ''
        for split in name.split():
            for c in split:
                if valid_fn(c): chars += str(c) if c=='_' else c
                else:
                    short = self.shortened_character_name(c, self.eliminations,
                                                         self.substitutions,
                                                         self.transforms)
                    sanitized.extend([chars] if chars else [])
                    if short != '':
                       sanitized.append(short)
                    chars = ''
            if chars:
                sanitized.extend([chars])
                chars=''
        return self._process_underscores(sanitized + ([chars] if chars else []))
448294
def unicode_safe(text, encoding=UrlEncoding, errors='ignore'):
    """"""
    try:
        text_type = unicode
    except NameError:
        text_type = str
    if isinstance(text, text_type):
        return text
    return text.decode(encoding, errors)
650678
def keep_allowed_chars(text):
    
    #     print ','.join(str(ord(c)) for c in text)
    return "".join(" " if c == "\n" else c for c in text.strip() if _allowed_char(c))
1177975
def _raise_on_bad_representation(representation):
        """"""
        if representation not in REPRESENTATIONS:
            repr_desc = ', '.join(map(repr, REPRESENTATIONS))
            raise ValueError("Unknown representation: %r (should be one "
                             "of %s)" % (representation, repr_desc))
51880
def characters(self, numberOfCharacters):
        """"""
        return self.code[self.index:self.index + numberOfCharacters]
379923
def check_unassigned(chars, bad_tables):
    
    bad_tables = (
        stringprep.in_table_a1,)

    violator = check_against_tables(chars, bad_tables)
    if violator is not None:
        raise ValueError("Input contains unassigned code point: "
                         "U+{:04x}".format(ord(violator)))
732696
def _unquote_c_string(s):
     """"""

     # doing a s.encode('utf-8').decode('unicode_escape') can return an
     # incorrect output with unicode string (both in py2 and py3) the safest way
     # is to match the escape sequences and decoding them alone.
     def decode_match(match):
          return utf8_bytes_string(
               codecs.decode(match.group(0), 'unicode-escape')
          )

     if sys.version_info[0] >= 3 and isinstance(s, bytes):
          return ESCAPE_SEQUENCE_BYTES_RE.sub(decode_match, s)
     else:
          return ESCAPE_SEQUENCE_RE.sub(decode_match, s)
855326
private void throwUnrecognizedInputIfAny( StringBuilder builder, int ch ) {
        if( !isWhitespace( builder ) ) {
            throw createException( "Unrecognized input: '" + trim( builder ) + (char)ch + "'" );
        }
        builder.setLength( 0 );
    }
244300
def clean_text(self, text):
        ''''''
        if text is None:
            return ''
        text = re.sub(ILLEGAL_CHARACTERS_RE, '', text)
        if '<' in text or '&lt' in text:
            text = clean(text, tags=self.tags, strip=self.strip)

        return unescape(text)
134294
function invalidChar(name, field = undefined) {
  let msg = `Invalid character in ${name}`;
  if (field !== undefined) {
    msg += ` ["${field}"]`;
  }
  return msg;
}
1015445
def accept_lit(char, buf, pos):
    """"""
    if pos >= len(buf) or buf[pos] != char:
        return None, pos
    return char, pos+1
1134709
function removeUnwantedCharacters(cleanedString, unwantedCharArray) {
    for (var x = 0; x < unwantedCharArray.length; x++) {
        var unwantedCharRegExp = new RegExp(unwantedCharArray[x], 'g');
        cleanedString = cleanedString.replace(unwantedCharRegExp, '');
    }
    return cleanedString;
}