Convert a surrogate pair to the single codepoint it represents.

    This implements the formula described at:
    http://en.wikipedia.org/wiki/Universal_Character_Set_characters#Surrogates
334274
def convert_surrogate_pair(match):
    
    pair = match.group(0)
    codept = 0x10000 + (ord(pair[0]) - 0xd800) * 0x400 + (ord(pair[1]) - 0xdc00)
    return chr(codept)
1172365
def sec_to_public_pair(sec, generator=None, strict=True):
    """"""
    byte_count = (generator.p().bit_length() + 7) >> 3 if generator else (len(sec) - 1)
    x = from_bytes_32(sec[1:1 + byte_count])
    sec0 = sec[:1]
    if len(sec) == 1 + byte_count * 2:
        isok = sec0 == b'\4'
        if not strict:
            isok = isok or (sec0 in [b'\6', b'\7'])
        if isok:
            y = from_bytes_32(sec[1+byte_count:1+2*byte_count])
            return (x, y)
    elif len(sec) == 1 + byte_count:
        if not strict or (sec0 in (b'\2', b'\3')):
            is_y_odd = (sec0 != b'\2')
            return generator.points_for_x(x)[is_y_odd]
    raise EncodingError("bad sec encoding for public key")
422860
function toCodePoints(input) {
  const codepoints = [];
  const size = input.length;

  for (let i = 0; i < size; i += 1) {
    const before = input.charCodeAt(i);

    if (before >= 0xd800 && before <= 0xdbff && size > i + 1) {
      const next = input.charCodeAt(i + 1);

      if (next >= 0xdc00 && next <= 0xdfff) {
        codepoints.push((before - 0xd800) * 0x400 + next - 0xdc00 + 0x10000);
        i += 1;
        continue;
      }
    }

    codepoints.push(before);
  }

  return codepoints;
}
628004
public function fromCodePoint($c)
    {
        if ($c < 0x7F) // U+0000-U+007F - 1 byte
            return chr($c);
        if ($c < 0x7FF) // U+0080-U+07FF - 2 bytes
            return chr(0xC0 | ($c >> 6)) . chr(0x80 | ($c & 0x3F));
        if ($c < 0xFFFF) // U+0800-U+FFFF - 3 bytes
            return chr(0xE0 | ($c >> 12)) . chr(0x80 | (($c >> 6) & 0x3F)) . chr(0x80 | ($c & 0x3F));
        // U+010000-U+10FFFF - 4 bytes
        return chr(0xF0 | ($c >> 18)) . chr(0x80 | ($c >> 12) & 0x3F) .chr(0x80 | (($c >> 6) & 0x3F)) . chr(0x80 | ($c & 0x3F));
    }
463407
private void setMaxDelimCodePoint() {
        if (delimiters == null) {
            maxDelimCodePoint = 0;
            return;
        }

        int m = 0;
        int c;
        int count = 0;
        for (int i = 0; i < delimiters.length(); i += Character.charCount(c)) {
            c = delimiters.charAt(i);
            if (c >= Character.MIN_HIGH_SURROGATE && c <= Character.MAX_LOW_SURROGATE) {
                c = delimiters.codePointAt(i);
                hasSurrogates = true;
            }
            if (m < c)
                m = c;
            count++;
        }
        maxDelimCodePoint = m;

        if (hasSurrogates) {
            delimiterCodePoints = new int[count];
            for (int i = 0, j = 0; i < count; i++, j += Character.charCount(c)) {
                c = delimiters.codePointAt(j);
                delimiterCodePoints[i] = c;
            }
        }
    }
1198343
def normalize_cjk_fullwidth_ascii(seq: str) -> str:
    

    def convert(char: str) -> str:
        code_point = ord(char)
        if not 0xFF01 <= code_point <= 0xFF5E:
            return char
        return chr(code_point - 0xFEE0)

    return ''.join(map(convert, seq))
86406
function encodeEntities(value) {
  return value.
    replace(/&/g, '&amp;').
    replace(SURROGATE_PAIR_REGEXP, function (value) {
      var hi = value.charCodeAt(0);
      var low = value.charCodeAt(1);
      return '&#' + (((hi - 0xD800) * 0x400) + (low - 0xDC00) + 0x10000) + ';';
    }).
    replace(NON_ALPHANUMERIC_REGEXP, function (value) {
      return '&#' + value.charCodeAt(0) + ';';
    }).
    replace(/</g, '&lt;').
    replace(/>/g, '&gt;');
}
194654
function getUTF8Length(s) {
  let len = 0;
  let i;
  let code;
  for (i = 0; i < s.length; i++) {
    code = s.charCodeAt(i);
    if (code <= 0x7f) {
      len += 1;
    } else if (code <= 0x7ff) {
      len += 2;
    } else if (code >= 0xd800 && code <= 0xdfff) {
      // Surrogate pair: These take 4 bytes in UTF-8 and 2 chars in UCS-2
      // (Assume next char is the other [valid] half and just skip it)
      len += 4; i++;
    } else if (code < 0xffff) {
      len += 3;
    } else {
      len += 4;
    }
  }
  return len;
}
1342842
function fromCodePoint(codepoint) {
			var code = typeof codepoint === 'string' ?
				parseInt(codepoint, 16) : codepoint;
			if (code < 0x10000) {
				return fromCharCode(code);
			}
			code -= 0x10000;
			return fromCharCode(
				0xD800 + (code >> 10),
				0xDC00 + (code & 0x3FF)
			);
		}
156529
def encode(input)
      input_encoding = input.encoding
      input = input.encode(Encoding::UTF_8).codepoints.to_a
      output = []

      # Initialize the state:
      n = INITIAL_N
      delta = 0
      bias = INITIAL_BIAS

      # Handle the basic code points:
      output = input.select { |char| char <= ASCII_MAX }

      h = b = output.length

      # h is the number of code points that have been handled, b is the
      # number of basic code points

      output << DELIMITER if b > 0

      # Main encoding loop:
      while h < input.length
        # All non-basic code points < n have been
        # handled already. Find the next larger one:

        m = MAXINT

        input.each do |char|
          m = char if char >= n && char < m
        end

        # Increase delta enough to advance the decoder's
        # <n,i> state to <m,0>, but guard against overflow:

        raise(ConversionError, "punycode_overflow (1)") if m - n > ((MAXINT - delta) / (h + 1)).floor

        delta += (m - n) * (h + 1)
        n = m

        input.each_with_index do |char, _|
          if char < n
            delta += 1
            raise(ConversionError, "punycode_overflow(2)") if delta > MAXINT
          end

          next unless char == n

          # Represent delta as a generalized variable-length integer:
          q = delta
          k = BASE
          loop do
            t = k <= bias ? TMIN : k >= bias + TMAX ? TMAX : k - bias
            break if q < t
            output << encode_digit(t + (q - t) % (BASE - t))
            q = ((q - t) / (BASE - t)).floor
            k += BASE
          end
          output << encode_digit(q)
          bias = adapt(delta, h + 1, h == b)
          delta = 0
          h += 1
        end

        delta += 1
        n += 1
      end
      output.collect {|c| c.chr(Encoding::UTF_8)}.join(EMPTY).encode(input_encoding)
    end
718055
function curveFpDecodePointHex(s) {
    switch(parseInt(s.substr(0,2), 16)) { // first byte
    case 0:
	return this.infinity;
    case 2:
    case 3:
	// point compression not supported yet
	return null;
    case 4:
    case 6:
    case 7:
	var len = (s.length - 2) / 2;
	var xHex = s.substr(2, len);
	var yHex = s.substr(len+2, len);

	return new ECPointFp(this,
			     this.fromBigInteger(new BigInteger(xHex, 16)),
			     this.fromBigInteger(new BigInteger(yHex, 16)));

    default: // unsupported
	return null;
    }
}
543619
public static int setCodePointAt(int codePoint, Slice utf8, int position)
    {
        if (codePoint < 0) {
            throw new InvalidCodePointException(codePoint);
        }
        if (codePoint < 0x80) {
            // normal ASCII
            // 0xxx_xxxx
            utf8.setByte(position, codePoint);
            return 1;
        }
        if (codePoint < 0x800) {
            // 110x_xxxx 10xx_xxxx
            utf8.setByte(position, 0b1100_0000 | (codePoint >>> 6));
            utf8.setByte(position + 1, 0b1000_0000 | (codePoint & 0b0011_1111));
            return 2;
        }
        if (MIN_SURROGATE <= codePoint && codePoint <= MAX_SURROGATE) {
            throw new InvalidCodePointException(codePoint);
        }
        if (codePoint < 0x1_0000) {
            // 1110_xxxx 10xx_xxxx 10xx_xxxx
            utf8.setByte(position, 0b1110_0000 | ((codePoint >>> 12) & 0b0000_1111));
            utf8.setByte(position + 1, 0b1000_0000 | ((codePoint >>> 6) & 0b0011_1111));
            utf8.setByte(position + 2, 0b1000_0000 | (codePoint & 0b0011_1111));
            return 3;
        }
        if (codePoint < 0x11_0000) {
            // 1111_0xxx 10xx_xxxx 10xx_xxxx 10xx_xxxx
            utf8.setByte(position, 0b1111_0000 | ((codePoint >>> 18) & 0b0000_0111));
            utf8.setByte(position + 1, 0b1000_0000 | ((codePoint >>> 12) & 0b0011_1111));
            utf8.setByte(position + 2, 0b1000_0000 | ((codePoint >>> 6) & 0b0011_1111));
            utf8.setByte(position + 3, 0b1000_0000 | (codePoint & 0b0011_1111));
            return 4;
        }
        // Per RFC3629, UTF-8 is limited to 4 bytes, so more bytes are illegal
        throw new InvalidCodePointException(codePoint);
    }
569815
function(str) {
    if(typeof(str)=='undefined') return undefined;
    if(typeof(str)=='undefined') return undefined;

    var surrogate_1st = 0;
    var unicode_codes = [];
    for (var i = 0; i < str.length; ++i) {
	var utf16_code = str.charCodeAt(i);
	if (surrogate_1st != 0) {
	    if (utf16_code >= 0xdc00 && utf16_code <= 0xdfff) {
		var surrogate_2nd = utf16_code;
		var unicode_code = (surrogate_1st - 0xd800) * (1 << 10) + (1 << 16) +
                    (surrogate_2nd - 0xdc00);
		unicode_codes.push(unicode_code);
	    }
	    surrogate_1st = 0;
	}
	else if (utf16_code >= 0xd800 && utf16_code <= 0xdbff) {
	    surrogate_1st = utf16_code;
	}
	else {
	    unicode_codes.push(utf16_code);
	}
    }

    var utf8_bytes = "";
    var i, unicode_code;
    for(i=0; i<unicode_codes.length; ++i) {
	unicode_code = unicode_codes[i];
	if (unicode_code < 0x80) {  // 1-byte
	    utf8_bytes += String.fromCharCode(unicode_code);
	}
	else if (unicode_code < (1 << 11)) {  // 2-byte
	    utf8_bytes += String.fromCharCode((unicode_code >>> 6) | 0xC0);
	    utf8_bytes += String.fromCharCode((unicode_code & 0x3F) | 0x80);
	}
	else if (unicode_code < (1 << 16)) {  // 3-byte
	    utf8_bytes += String.fromCharCode((unicode_code >>> 12) | 0xE0);
	    utf8_bytes += String.fromCharCode(((unicode_code >> 6) & 0x3f) | 0x80);
	    utf8_bytes += String.fromCharCode((unicode_code & 0x3F) | 0x80);
	}
	else if (unicode_code < (1 << 21)) {  // 4-byte
	    utf8_bytes += String.fromCharCode((unicode_code >>> 18) | 0xF0);
	    utf8_bytes += String.fromCharCode(((unicode_code >> 12) & 0x3F) | 0x80);
	    utf8_bytes += String.fromCharCode(((unicode_code >> 6) & 0x3F) | 0x80);
	    utf8_bytes += String.fromCharCode((unicode_code & 0x3F) | 0x80);
	}
    }
    return utf8_bytes;
}
1112655
def replace_surrogate_encode(mystring, exc):
    
    decoded = []
    for ch in mystring:
        # if PY3:
        #     code = ch
        # else:
        code = ord(ch)

        # The following magic comes from Py3.3's Python/codecs.c file:
        if not 0xD800 <= code <= 0xDCFF:
            # Not a surrogate. Fail with the original exception.
            raise exc
        # mybytes = [0xe0 | (code >> 12),
        #            0x80 | ((code >> 6) & 0x3f),
        #            0x80 | (code & 0x3f)]
        # Is this a good idea?
        if 0xDC00 <= code <= 0xDC7F:
            decoded.append(_unichr(code - 0xDC00))
        elif code <= 0xDCFF:
            decoded.append(_unichr(code - 0xDC00))
        else:
            raise NotASurrogateError
    return str().join(decoded)
1307887
private static String filter(String s) {
		int len = s.length();
		StringBuilder filtered = new StringBuilder(len);
		int pos = 0;
		while(pos < len) {
			char ch1 = s.charAt(pos++);
			if(Character.isHighSurrogate(ch1)) {
				// Handle surrogates
				if(pos < len) {
					char ch2 = s.charAt(pos++);
					if(Character.isLowSurrogate(ch2)) {
						if(isValidCharacter(Character.toCodePoint(ch1, ch2))) {
							filtered.append(ch1).append(ch2);
						}
					} else {
						// High surrogate not followed by low surrogate, invalid
					}
				} else {
					// High surrogate at end of string, invalid
				}
			} else {
				// Not surrogates
				if(isValidCharacter(ch1)) {
					filtered.append(ch1);
				}
			}
		}
		assert filtered.length() <= len;
		return filtered.length() != len ? filtered.toString() : s;
	}
329377
def encode_unicode_codepoints(str):
    
    """
    codepoint_format = '\\u{0:04x}'.format

    def codepoint(m):
        return codepoint_format(ord(m.group(0)))

    return re.sub('[^\x00-\x7f]', codepoint, str)
1234058
private final int _convertSurrogate(int secondPart)
        throws IOException
    {
        int firstPart = mSurrogate;
        mSurrogate = 0;

        // Ok, then, is the second part valid?
        if (secondPart < SURR2_FIRST || secondPart > SURR2_LAST) {
            throw new IOException("Broken surrogate pair: first char 0x"+Integer.toHexString(firstPart)+", second 0x"+Integer.toHexString(secondPart)+"; illegal combination");
        }
        return 0x10000 + ((firstPart - SURR1_FIRST) << 10) + (secondPart - SURR2_FIRST);
    }