per http://www.w3.org/TR/html40/appendix/notes.html#non-ascii-chars this should be UTF-8 to provide for broadest compatibility
961549
private static String decodeURL(String url) {
    	try {
			return URLDecoder.decode(url, UTF_8);
		} catch (UnsupportedEncodingException e) {
		}
		return null;
    }
461863
public static function ascii($str, $allow_non_ascii = false)
	{
		// Translate unicode characters to their simpler counterparts
		\Config::load('ascii', true);
		$foreign_characters = \Config::get('ascii');

		$str = preg_replace(array_keys($foreign_characters), array_values($foreign_characters), $str);

		if ( ! $allow_non_ascii)
		{
			return preg_replace('/[^\x09\x0A\x0D\x20-\x7E]/', '', $str);
		}

		return $str;
	}
1183414
function __encode_utf8(text) {

        // Fill buffer with UTF-8
        for (var i=0; i<text.length; i++) {
            var codepoint = text.charCodeAt(i);
            __append_utf8(codepoint);
        }

        // Flush buffer
        if (length > 0) {
            var out_buffer = buffer.subarray(0, length);
            length = 0;
            return out_buffer;
        }

    }
331893
function str2rstr_utf8(input) {
    var output = "";
    var i = -1;
    var x, y;

    while (++i < input.length) {
        /* Decode utf-16 surrogate pairs */
        x = input.charCodeAt(i);
        y = i + 1 < input.length ? input.charCodeAt(i + 1) : 0;
        if (0xD800 <= x && x <= 0xDBFF && 0xDC00 <= y && y <= 0xDFFF) {
            x = 0x10000 + ((x & 0x03FF) << 10) + (y & 0x03FF);
            i++;
        }

        /* Encode output as utf-8 */
        if (x <= 0x7F)
            output += String.fromCharCode(x);
        else if (x <= 0x7FF)
            output += String.fromCharCode(0xC0 | ((x >>> 6 ) & 0x1F),
                0x80 | ( x & 0x3F));
        else if (x <= 0xFFFF)
            output += String.fromCharCode(0xE0 | ((x >>> 12) & 0x0F),
                0x80 | ((x >>> 6 ) & 0x3F),
                0x80 | ( x & 0x3F));
        else if (x <= 0x1FFFFF)
            output += String.fromCharCode(0xF0 | ((x >>> 18) & 0x07),
                0x80 | ((x >>> 12) & 0x3F),
                0x80 | ((x >>> 6 ) & 0x3F),
                0x80 | ( x & 0x3F));
    }
    return output;
}
328891
def _safe_unicode(o):
    
    def clean(s):
        return u''.join([c if c in ASCII_PRINTABLE else '?' for c in s])
    if USING_PYTHON2:
        try:
            return unicode(o)
        except:
            try:
                s = str(o)
                try:
                    return s.decode("utf-8")
                except:
                    return clean(s[:2048]) + u" [Raw error message: " + unicode(s.encode("hex"), 'utf-8') + u"]"
            except:
                return u"(Unable to decode Python exception message)"
    else:
        return str(o)
78856
def utf82unicode(value):
    
    try:
        return value.decode("utf8")
    except Exception as e:
        if not _Log:
            _late_import()

        if not is_binary(value):
            _Log.error("Can not convert {{type}} to unicode because it's not bytes",  type= type(value).__name__)

        e = _Except.wrap(e)
        for i, c in enumerate(value):
            try:
                c.decode("utf8")
            except Exception as f:
                _Log.error("Can not convert charcode {{c}} in string index {{i}}", i=i, c=ord(c), cause=[e, _Except.wrap(f)])

        try:
            latin1 = text_type(value.decode("latin1"))
            _Log.error("Can not explain conversion failure, but seems to be latin1", e)
        except Exception:
            pass

        try:
            a = text_type(value.decode("latin1"))
            _Log.error("Can not explain conversion failure, but seems to be latin1", e)
        except Exception:
            pass

        _Log.error("Can not explain conversion failure of " + type(value).__name__ + "!", e)
284533
def _to_unicode(s):
    
    try:
        return s.encode('ascii')
    except (UnicodeError, AttributeError):
        try:
            return s.decode('utf-8')
        except UnicodeError:
            return s
116763
def charset(self, default='UTF-8'):
        
        if 'charset=' in self.content_type:
            return self.content_type.split('charset=')[-1].split(';')[0].strip()
        return default
408533
def _unicode_to_ascii(obj):  # pragma: no cover
    """"""
    # pylint: disable=E0602,R1717
    if isinstance(obj, dict):
        return dict(
            [
                (_unicode_to_ascii(key), _unicode_to_ascii(value))
                for key, value in obj.items()
            ]
        )
    if isinstance(obj, list):
        return [_unicode_to_ascii(element) for element in obj]
    if isinstance(obj, unicode):
        return obj.encode("utf-8")
    return obj
191287
static byte[] asciiCompatibleEncoding(byte[] asciiCompatName) {
        CaseInsensitiveBytesHash<TranscoderDB.Entry> dTable = TranscoderDB.transcoders.get(asciiCompatName);
        if (dTable == null || dTable.size() != 1) return null;

        byte[] asciiCN = null;
        for (Entry e : dTable) {
            if (!EConv.decorator(e.source, e.destination)) {
                Transcoder transcoder = e.getTranscoder();
                if (transcoder != null && transcoder.compatibility.isDecoder()) {
                    asciiCN = transcoder.destination;
                    break;
                }
            }
        }
        return asciiCN;
    }
825624
public static StringBuilder encodeHtml(StringBuilder buffer, CharSequence chars) {
		chars.chars().forEach(c -> {
			switch (c) {
			case '<':
				buffer.append("&lt;");
				break;
			case '>':
				buffer.append("&gt;");
				break;
			case '&':
				buffer.append("&amp;");
				break;
			case '"':
				buffer.append("&quot;");
				break;
			case '\'':
				buffer.append("&#39;");
				break;
			default:
				if (Character.isAlphabetic(c)) {
					buffer.append((char) c);
				} else {
					buffer.append("&#").append(c).append(';');
				}
			}
		});
		return buffer;
	}
556621
def encode_non_ascii_string(string):
    
    encoded_string = string.encode('utf-8', 'replace')
    if six.PY3:
        encoded_string = encoded_string.decode()

    return encoded_string
237447
def guess_encoding(request):
    
    ctype = request.headers.get('content-type')
    if not ctype:
        # we don't have a content-type, somehow, so...
        LOGGER.warning("%s: no content-type; headers are %s",
                       request.url, request.headers)
        return 'utf-8'

    # explicit declaration
    match = re.search(r'charset=([^ ;]*)(;| |$)', ctype)
    if match:
        return match[1]

    # html default
    if ctype.startswith('text/html'):
        return 'iso-8859-1'

    # everything else's default
    return 'utf-8'
496000
function toUtf8String(bytes, ignoreErrors) {
    bytes = bytes_1.arrayify(bytes);
    var result = '';
    var i = 0;
    // Invalid bytes are ignored
    while (i < bytes.length) {
        var c = bytes[i++];
        // 0xxx xxxx
        if (c >> 7 === 0) {
            result += String.fromCharCode(c);
            continue;
        }
        // Multibyte; how many bytes left for this character?
        var extraLength = null;
        var overlongMask = null;
        // 110x xxxx 10xx xxxx
        if ((c & 0xe0) === 0xc0) {
            extraLength = 1;
            overlongMask = 0x7f;
            // 1110 xxxx 10xx xxxx 10xx xxxx
        }
        else if ((c & 0xf0) === 0xe0) {
            extraLength = 2;
            overlongMask = 0x7ff;
            // 1111 0xxx 10xx xxxx 10xx xxxx 10xx xxxx
        }
        else if ((c & 0xf8) === 0xf0) {
            extraLength = 3;
            overlongMask = 0xffff;
        }
        else {
            if (!ignoreErrors) {
                if ((c & 0xc0) === 0x80) {
                    throw new Error('invalid utf8 byte sequence; unexpected continuation byte');
                }
                throw new Error('invalid utf8 byte sequence; invalid prefix');
            }
            continue;
        }
        // Do we have enough bytes in our data?
        if (i + extraLength > bytes.length) {
            if (!ignoreErrors) {
                throw new Error('invalid utf8 byte sequence; too short');
            }
            // If there is an invalid unprocessed byte, skip continuation bytes
            for (; i < bytes.length; i++) {
                if (bytes[i] >> 6 !== 0x02) {
                    break;
                }
            }
            continue;
        }
        // Remove the length prefix from the char
        var res = c & ((1 << (8 - extraLength - 1)) - 1);
        for (var j = 0; j < extraLength; j++) {
            var nextChar = bytes[i];
            // Invalid continuation byte
            if ((nextChar & 0xc0) != 0x80) {
                res = null;
                break;
            }
            ;
            res = (res << 6) | (nextChar & 0x3f);
            i++;
        }
        if (res === null) {
            if (!ignoreErrors) {
                throw new Error('invalid utf8 byte sequence; invalid continuation byte');
            }
            continue;
        }
        // Check for overlong seuences (more bytes than needed)
        if (res <= overlongMask) {
            if (!ignoreErrors) {
                throw new Error('invalid utf8 byte sequence; overlong');
            }
            continue;
        }
        // Maximum code point
        if (res > 0x10ffff) {
            if (!ignoreErrors) {
                throw new Error('invalid utf8 byte sequence; out-of-range');
            }
            continue;
        }
        // Reserved for UTF-16 surrogate halves
        if (res >= 0xd800 && res <= 0xdfff) {
            if (!ignoreErrors) {
                throw new Error('invalid utf8 byte sequence; utf-16 surrogate');
            }
            continue;
        }
        if (res <= 0xffff) {
            result += String.fromCharCode(res);
            continue;
        }
        res -= 0x10000;
        result += String.fromCharCode(((res >> 10) & 0x3ff) + 0xd800, (res & 0x3ff) + 0xdc00);
    }
    return result;
}
1263938
def force_ascii_values(data):
    """"""
    return {
        k: v.encode('utf8').decode('ascii', 'backslashreplace')
        for k, v in data.items()
    }
1186441
public static String unquoteHtmlChars(String item) {
    if (item == null) {
      return null;
    }
    int next = item.indexOf('&');
    // nothing was quoted
    if (next == -1) {
      return item;
    }
    int len = item.length();
    int posn = 0;
    StringBuilder buffer = new StringBuilder();
    while (next != -1) {
      buffer.append(item.substring(posn, next));
      if (item.startsWith("&amp;", next)) {
        buffer.append('&');
        next += 5;
      } else if (item.startsWith("&apos;", next)) {
        buffer.append('\'');
        next += 6;        
      } else if (item.startsWith("&gt;", next)) {
        buffer.append('>');
        next += 4;
      } else if (item.startsWith("&lt;", next)) {
        buffer.append('<');
        next += 4;
      } else if (item.startsWith("&quot;", next)) {
        buffer.append('"');
        next += 6;
      } else {
        int end = item.indexOf(';', next)+1;
        if (end == 0) {
          end = len;
        }
        throw new IllegalArgumentException("Bad HTML quoting for " + 
                                           item.substring(next,end));
      }
      posn = next;
      next = item.indexOf('&', posn);
    }
    buffer.append(item.substring(posn, len));
    return buffer.toString();
  }
483067
function _utf8ToChar(&$chars, &$idx)
	{
		if(($chars[$idx] >= 240) && ($chars[$idx] <= 255))
			/* 4 bytes */
			$utf =    ((int)($chars[$idx]-240)   << 18) +
				((int)($chars[++$idx]-128) << 12) +
				((int)($chars[++$idx]-128) << 6) +
				((int)($chars[++$idx]-128) << 0);
		else if (($chars[$idx] >= 224) && ($chars[$idx] <= 239))
			/* 3 bytes */
			$utf =    ((int)($chars[$idx]-224)   << 12) +
				((int)($chars[++$idx]-128) << 6) +
				((int)($chars[++$idx]-128) << 0);
		else if (($chars[$idx] >= 192) && ($chars[$idx] <= 223))
			/* 2 bytes */
			$utf =    ((int)($chars[$idx]-192)   << 6) +
				((int)($chars[++$idx]-128) << 0);
		else
			/* 1 byte */
			$utf = $chars[$idx];
		if(array_key_exists($utf,$this->utfMap))
			return \chr($this->utfMap[$utf]);
		else
			return "?";
	}