/* UTF-8 sequence to single byte character
483067
function _utf8ToChar(&$chars, &$idx)
	{
		if(($chars[$idx] >= 240) && ($chars[$idx] <= 255))
			/* 4 bytes */
			$utf =    ((int)($chars[$idx]-240)   << 18) +
				((int)($chars[++$idx]-128) << 12) +
				((int)($chars[++$idx]-128) << 6) +
				((int)($chars[++$idx]-128) << 0);
		else if (($chars[$idx] >= 224) && ($chars[$idx] <= 239))
			/* 3 bytes */
			$utf =    ((int)($chars[$idx]-224)   << 12) +
				((int)($chars[++$idx]-128) << 6) +
				((int)($chars[++$idx]-128) << 0);
		else if (($chars[$idx] >= 192) && ($chars[$idx] <= 223))
			/* 2 bytes */
			$utf =    ((int)($chars[$idx]-192)   << 6) +
				((int)($chars[++$idx]-128) << 0);
		else
			/* 1 byte */
			$utf = $chars[$idx];
		if(array_key_exists($utf,$this->utfMap))
			return \chr($this->utfMap[$utf]);
		else
			return "?";
	}
629621
function encodeCharacter (chr) {
        var
            result = '',
            octets = utf8.encode(chr),
            octet,
            index;
        for (index = 0; index < octets.length; index += 1) {
            octet = octets.charCodeAt(index);
            result += '%' + (octet < 0x10 ? '0' : '') + octet.toString(16).toUpperCase();
        }
        return result;
    }
1160823
def utf8str(code)
      if (code & ~(0x7f)) == 0
        # UCS-4 range 0x00000000 - 0x0000007F
        return(code.chr)
      end

      buf = ""
      if (code & ~(0x7ff)) == 0
        # UCS-4 range 0x00000080 - 0x000007FF
        buf << (0b11000000 | (code >> 6)).chr
        buf << (0b10000000 | (code & 0b00111111)).chr
        return(buf)
      end

      if (code & ~(0x000ffff)) == 0
        # UCS-4 range 0x00000800 - 0x0000FFFF
        buf << (0b11100000 | (code >> 12)).chr
        buf << (0b10000000 | ((code >> 6) & 0b00111111)).chr
        buf << (0b10000000 | (code & 0b0011111)).chr
        return(buf)
      end

      # Not used -- JSON only has UCS-2, but for the sake
      # of completeness
      if (code & ~(0x1FFFFF)) == 0
        # UCS-4 range 0x00010000 - 0x001FFFFF
        buf << (0b11110000 | (code >> 18)).chr
        buf << (0b10000000 | ((code >> 12) & 0b00111111)).chr
        buf << (0b10000000 | ((code >> 6) & 0b00111111)).chr
        buf << (0b10000000 | (code & 0b0011111)).chr
        return(buf)
      end

      if (code & ~(0x03FFFFFF)) == 0
        # UCS-4 range 0x00200000 - 0x03FFFFFF
        buf << (0b11110000 | (code >> 24)).chr
        buf << (0b10000000 | ((code >> 18) & 0b00111111)).chr
        buf << (0b10000000 | ((code >> 12) & 0b00111111)).chr
        buf << (0b10000000 | ((code >> 6) & 0b00111111)).chr
        buf << (0b10000000 | (code & 0b0011111)).chr
        return(buf)
      end

      # UCS-4 range 0x04000000 - 0x7FFFFFFF
      buf << (0b11111000 | (code >> 30)).chr
      buf << (0b10000000 | ((code >> 24) & 0b00111111)).chr
      buf << (0b10000000 | ((code >> 18) & 0b00111111)).chr
      buf << (0b10000000 | ((code >> 12) & 0b00111111)).chr
      buf << (0b10000000 | ((code >> 6) & 0b00111111)).chr
      buf << (0b10000000 | (code & 0b0011111)).chr
      return(buf)
    end
1260504
function _ucs4_to_utf8($input)
    {
        $output = '';
        $k = 0;
        foreach ($input as $v) {
            ++$k;
            // $v = ord($v);
            if ($v < 128) { // 7bit are transferred literally
                $output .= chr($v);
            } elseif ($v < (1 << 11)) { // 2 bytes
                $output .= chr(192 + ($v >> 6)) . chr(128 + ($v & 63));
            } elseif ($v < (1 << 16)) { // 3 bytes
                $output .= chr(224 + ($v >> 12)) . chr(128 + (($v >> 6) & 63)) . chr(128 + ($v & 63));
            } elseif ($v < (1 << 21)) { // 4 bytes
                $output .= chr(240 + ($v >> 18)) . chr(128 + (($v >> 12) & 63))
                         . chr(128 + (($v >> 6) & 63)) . chr(128 + ($v & 63));
            } elseif ($v < (1 << 26)) { // 5 bytes
                $output .= chr(248 + ($v >> 24)) . chr(128 + (($v >> 18) & 63))
                         . chr(128 + (($v >> 12) & 63)) . chr(128 + (($v >> 6) & 63))
                         . chr(128 + ($v & 63));
            } elseif ($v < (1 << 31)) { // 6 bytes
                $output .= chr(252 + ($v >> 30)) . chr(128 + (($v >> 24) & 63))
                         . chr(128 + (($v >> 18) & 63)) . chr(128 + (($v >> 12) & 63))
                         . chr(128 + (($v >> 6) & 63)) . chr(128 + ($v & 63));
            } else {
                $this->_error('Conversion from UCS-4 to UTF-8 failed: malformed input at byte '.$k);
                return false;
            }
        }
        return $output;
    }
1144278
public static int utf8Length(String string) {
        CharacterIterator iter = new StringCharacterIterator(string);
        char ch = iter.first();
        int size = 0;
        while (ch != CharacterIterator.DONE) {
            if ((ch >= 0xD800) && (ch < 0xDC00)) {
                // surrogate pair?
                char trail = iter.next();
                if ((trail > 0xDBFF) && (trail < 0xE000)) {
                    // valid pair
                    size += 4;
                } else {
                    // invalid pair
                    size += 3;
                    iter.previous(); // rewind one
                }
            } else if (ch < 0x80) {
                size++;
            } else if (ch < 0x800) {
                size += 2;
            } else {
                // ch < 0x10000, that is, the largest char value
                size += 3;
            }
            ch = iter.next();
        }
        return size;
    }
142204
function (bytes) {
                var str = '';
                for (var i = 0, l = bytes.length; i < l; i += 2) {
                    str += String.fromCharCode((bytes[i] << 8) + bytes[i + 1]);
                }
                return str;
            }
114168
static private void encodeBytes(byte previous, byte current, int byteIndex, StringBuilder output) {
        int chunk;
        int offset = byteIndex % 5;
        switch (offset) {
            case 0:
                chunk = ((current & 0xF8) >>> 3);
                output.append(lookupTable.charAt(chunk));
                break;
            case 1:
                chunk = (((previous & 0x07) << 2) | ((current & 0xC0) >>> 6));
                output.append(lookupTable.charAt(chunk));
                chunk = ((current & 0x3E) >>> 1);
                output.append(lookupTable.charAt(chunk));
                break;
            case 2:
                chunk = (((previous & 0x01) << 4) | ((current & 0xF0) >>> 4));
                output.append(lookupTable.charAt(chunk));
                break;
            case 3:
                chunk = (((previous & 0x0F) << 1) | ((current & 0x80) >>> 7));
                output.append(lookupTable.charAt(chunk));
                chunk = ((current & 0x7C) >>> 2);
                output.append(lookupTable.charAt(chunk));
                break;
            case 4:
                chunk = (((previous & 0x03) << 3) | ((current & 0xE0) >>> 5));
                output.append(lookupTable.charAt(chunk));
                chunk = (current & 0x1F);
                output.append(lookupTable.charAt(chunk));
                break;
        }
    }
479049
protected static function utf8_to_codepoints($input) {
		$codepoints = array();

		// Get number of bytes
		$strlen = strlen($input);

		for ($position = 0; $position < $strlen; $position++) {
			$value = ord($input[$position]);

			// One byte sequence:
			if ((~$value & 0x80) === 0x80) {
				$character = $value;
				$length = 1;
				$remaining = 0;
			}
			// Two byte sequence:
			elseif (($value & 0xE0) === 0xC0) {
				$character = ($value & 0x1F) << 6;
				$length = 2;
				$remaining = 1;
			}
			// Three byte sequence:
			elseif (($value & 0xF0) === 0xE0) {
				$character = ($value & 0x0F) << 12;
				$length = 3;
				$remaining = 2;
			}
			// Four byte sequence:
			elseif (($value & 0xF8) === 0xF0) {
				$character = ($value & 0x07) << 18;
				$length = 4;
				$remaining = 3;
			}
			// Invalid byte:
			else {
				throw new Requests_Exception('Invalid Unicode codepoint', 'idna.invalidcodepoint', $value);
			}

			if ($remaining > 0) {
				if ($position + $length > $strlen) {
					throw new Requests_Exception('Invalid Unicode codepoint', 'idna.invalidcodepoint', $character);
				}
				for ($position++; $remaining > 0; $position++) {
					$value = ord($input[$position]);

					// If it is invalid, count the sequence as invalid and reprocess the current byte:
					if (($value & 0xC0) !== 0x80) {
						throw new Requests_Exception('Invalid Unicode codepoint', 'idna.invalidcodepoint', $character);
					}

					$character |= ($value & 0x3F) << (--$remaining * 6);
				}
				$position--;
			}

			if (
				// Non-shortest form sequences are invalid
				   $length > 1 && $character <= 0x7F
				|| $length > 2 && $character <= 0x7FF
				|| $length > 3 && $character <= 0xFFFF
				// Outside of range of ucschar codepoints
				// Noncharacters
				|| ($character & 0xFFFE) === 0xFFFE
				|| $character >= 0xFDD0 && $character <= 0xFDEF
				|| (
					// Everything else not in ucschar
					   $character > 0xD7FF && $character < 0xF900
					|| $character < 0x20
					|| $character > 0x7E && $character < 0xA0
					|| $character > 0xEFFFD
				)
			) {
				throw new Requests_Exception('Invalid Unicode codepoint', 'idna.invalidcodepoint', $character);
			}

			$codepoints[] = $character;
		}

		return $codepoints;
	}
245646
function (s) {
        var buffer = new ArrayBuffer(s.length);
        var b = new Uint8Array(buffer);
        var i;
        for (i = 0; i < s.length; i++) {
            b[i] = s.charCodeAt(i);
        }
        return b;
    }
1294427
function _Char_toCode(char)
{
	var code = char.charCodeAt(0);
	if (0xD800 <= code && code <= 0xDBFF)
	{
		return (code - 0xD800) * 0x400 + char.charCodeAt(1) - 0xDC00 + 0x10000
	}
	return code;
}
1100585
private int utf8Position(int charPosition) {
    int pos = offset;
    for (int i = 0; i < charPosition; i++) {
      int byteValue = data[pos] & 0xff;
      if (byteValue < 0x80) {
        pos += 1;
      } else if (byteValue < 0xC2) {
        throw new UncheckedIOException("invalid UTF8", new CharacterCodingException());
      } else if (byteValue < 0xE0) {
        pos += 2;
      } else if (byteValue < 0xF0) {
        pos += 3;
      } else if (byteValue < 0xF8) {
        pos += 4;
      } else {
        throw new UncheckedIOException("invalid UTF8", new CharacterCodingException());
      }
    }
    return pos;
  }
1000422
public static UTF8String fromBytes(byte[] bytes) {
    if (bytes != null) {
      return new UTF8String(bytes, BYTE_ARRAY_OFFSET, bytes.length);
    } else {
      return null;
    }
  }
437734
function fromUTF8(input){
    var output = '';
    var len = input.length;
    var i = 0;
    var c1;
    var c2;
    var c3;

    while (i < len)
    {
      c1 = input.charCodeAt(i++);
      if (c1 < 128)
        output += chars[c1];
      else
      {
        c2 = input.charCodeAt(i++);
        if (c1 & 32)
        {
          c3 = input.charCodeAt(i++);
          output += String.fromCharCode(((c1 & 15) << 12) | ((c2 & 63) << 6) | (c3 & 63));
        }
        else
          output += String.fromCharCode(((c1 & 31) << 6)  | (c2 & 63));
      }
    }
    return output;
  }
672221
public static function toUnicode($str)
    {
        $mState = 0; // cached expected number of octets after the current octet
        // until the beginning of the next UTF8 character sequence
        $mUcs4 = 0; // cached Unicode character
        $mBytes = 1; // cached expected number of octets in the current sequence

        $out = [];

        $len = strlen($str);

        for ($i = 0; $i < $len; $i++) {
            $in = ord($str{$i});

            if ($mState == 0) {
                // When mState is zero we expect either a US-ASCII character or a
                // multi-octet sequence.
                if (0 == (0x80 & ($in))) {
                    // US-ASCII, pass straight through.
                    $out[] = $in;
                    $mBytes = 1;
                } elseif (0xC0 == (0xE0 & ($in))) {
                    // First octet of 2 octet sequence
                    $mUcs4 = ($in);
                    $mUcs4 = ($mUcs4 & 0x1F) << 6;
                    $mState = 1;
                    $mBytes = 2;
                } elseif (0xE0 == (0xF0 & ($in))) {
                    // First octet of 3 octet sequence
                    $mUcs4 = ($in);
                    $mUcs4 = ($mUcs4 & 0x0F) << 12;
                    $mState = 2;
                    $mBytes = 3;
                } elseif (0xF0 == (0xF8 & ($in))) {
                    // First octet of 4 octet sequence
                    $mUcs4 = ($in);
                    $mUcs4 = ($mUcs4 & 0x07) << 18;
                    $mState = 3;
                    $mBytes = 4;
                } elseif (0xF8 == (0xFC & ($in))) {
                    /* First octet of 5 octet sequence.
                    *
                    * This is illegal because the encoded codepoint must be either
                    * (a) not the shortest form or
                    * (b) outside the Unicode range of 0-0x10FFFF.
                    * Rather than trying to resynchronize, we will carry on until the end
                    * of the sequence and let the later error handling code catch it.
                    */
                    $mUcs4 = ($in);
                    $mUcs4 = ($mUcs4 & 0x03) << 24;
                    $mState = 4;
                    $mBytes = 5;
                } elseif (0xFC == (0xFE & ($in))) {
                    // First octet of 6 octet sequence, see comments for 5 octet sequence.
                    $mUcs4 = ($in);
                    $mUcs4 = ($mUcs4 & 1) << 30;
                    $mState = 5;
                    $mBytes = 6;
                } else {
                    /* Current octet is neither in the US-ASCII range nor a legal first
                     * octet of a multi-octet sequence.
                     */
                    trigger_error(
                        'utf8_to_unicode: Illegal sequence identifier ' .
                        'in UTF-8 at byte ' . $i,
                        E_USER_WARNING
                    );

                    return false;
                }
            } else {
                // When mState is non-zero, we expect a continuation of the multi-octet
                // sequence
                if (0x80 == (0xC0 & ($in))) {
                    // Legal continuation.
                    $shift = ($mState - 1) * 6;
                    $tmp = $in;
                    $tmp = ($tmp & 0x0000003F) << $shift;
                    $mUcs4 |= $tmp;

                    /**
                     * End of the multi-octet sequence. mUcs4 now contains the final
                     * Unicode codepoint to be output
                     */
                    if (0 == --$mState) {
                        /*
                        * Check for illegal sequences and codepoints.
                        */
                        // From Unicode 3.1, non-shortest form is illegal
                        if (((2 == $mBytes) && ($mUcs4 < 0x0080)) ||
                            ((3 == $mBytes) && ($mUcs4 < 0x0800)) ||
                            ((4 == $mBytes) && ($mUcs4 < 0x10000)) ||
                            (4 < $mBytes) ||
                            // From Unicode 3.2, surrogate characters are illegal
                            (($mUcs4 & 0xFFFFF800) == 0xD800) ||
                            // Codepoints outside the Unicode range are illegal
                            ($mUcs4 > 0x10FFFF)
                        ) {
                            trigger_error(
                                'utf8_to_unicode: Illegal sequence or codepoint ' .
                                'in UTF-8 at byte ' . $i,
                                E_USER_WARNING
                            );

                            return false;
                        }

                        if (0xFEFF != $mUcs4) {
                            // BOM is legal but we don't want to output it
                            $out[] = $mUcs4;
                        }

                        //initialize UTF8 cache
                        $mState = 0;
                        $mUcs4 = 0;
                        $mBytes = 1;
                    }
                } else {
                    /**
                     *((0xC0 & (*in) != 0x80) && (mState != 0))
                     * Incomplete multi-octet sequence.
                     */
                    trigger_error(
                        'utf8_to_unicode: Incomplete multi-octet ' .
                        '   sequence in UTF-8 at byte ' . $i,
                        E_USER_WARNING
                    );

                    return false;
                }
            }
        }
    }
406771
def utf8(value):
    
    """
    try:
        if isinstance(value, bytes):
            return value.decode('utf-8')
        else:
            return value
    except Exception:
        raise ValueError('Not UTF-8: %r' % value)
1165821
function utf8_strtrunc($str, $len) {
		$i = $len - 1;
		if (ord($str{$i}) & 0x80) { // part of a multibyte sequence
			for (; $i > 0 && !(ord($str{$i}) & 0x40); $i--) {
				// find the first byte
				;
			}
			if ($i <= 0) {
				return '';
			} // sanity check
			for ($bc = 0, $mbs = ord($str{$i}); $mbs & 0x80; $mbs = $mbs << 1) {
				// calculate number of bytes
				$bc++;
			}
			if ($bc + $i > $len) {
				return substr($str, 0, $i);
			}
			// fallthru: multibyte char fits into length
		}
		return substr($str, 0, $len);
	}
605541
private static function utf8ToHexSequence( $str ) {
		$buf = '';
		foreach ( preg_split( '//u', $str, -1, PREG_SPLIT_NO_EMPTY ) as $cp ) {
			$buf .= sprintf( '%04x ', self::utf8ToCodepoint( $cp ) );
		}

		return rtrim( $buf );
	}