Convert provided string into an array of Unicode Code Points.
Based on https://stackoverflow.com/a/21409165/1556249
and https://www.npmjs.com/package/code-point-at.
@param {string} input
@returns {number[]}
422860
function toCodePoints(input) {
  const codepoints = [];
  const size = input.length;

  for (let i = 0; i < size; i += 1) {
    const before = input.charCodeAt(i);

    if (before >= 0xd800 && before <= 0xdbff && size > i + 1) {
      const next = input.charCodeAt(i + 1);

      if (next >= 0xdc00 && next <= 0xdfff) {
        codepoints.push((before - 0xd800) * 0x400 + next - 0xdc00 + 0x10000);
        i += 1;
        continue;
      }
    }

    codepoints.push(before);
  }

  return codepoints;
}
536232
public Token[] tokenize(String input) 
    {
        Token[] initialTokens = innerTokenizer.tokenize(input);
        List tokens = new ArrayList();
        for (int i=0; i<initialTokens.length; i++) {
            Token tok = initialTokens[i];
            String str = "^"+tok.getValue()+"$";
            if (keepOldTokens) tokens.add( intern(str) );
            for (int lo=0; lo<str.length(); lo++) {
                for (int len=minNGramSize; len<=maxNGramSize; len++) {
                    if (lo+len<str.length()) {
                        tokens.add( innerTokenizer.intern( str.substring(lo,lo+len) )); 
                    }
                }
            }
        }
        return (Token[]) tokens.toArray(new BasicToken[tokens.size()]);
    }
722125
public String normalize(String input) {
		char[] c = input.toCharArray();
		normalise(c);
		return new String(c);
	}
77107
function (input) {
			var bi = BigInteger.fromByteArrayUnsigned(input);
			var chars = [];

			while (bi.compareTo(B58.base) >= 0) {
				var mod = bi.mod(B58.base);
				chars.unshift(B58.alphabet[mod.intValue()]);
				bi = bi.subtract(mod).divide(B58.base);
			}
			chars.unshift(B58.alphabet[bi.intValue()]);

			// Convert leading zeros too.
			for (var i = 0; i < input.length; i++) {
				if (input[i] == 0x00) {
					chars.unshift(B58.alphabet[0]);
				} else break;
			}

			return chars.join('');
		}
331893
function str2rstr_utf8(input) {
    var output = "";
    var i = -1;
    var x, y;

    while (++i < input.length) {
        /* Decode utf-16 surrogate pairs */
        x = input.charCodeAt(i);
        y = i + 1 < input.length ? input.charCodeAt(i + 1) : 0;
        if (0xD800 <= x && x <= 0xDBFF && 0xDC00 <= y && y <= 0xDFFF) {
            x = 0x10000 + ((x & 0x03FF) << 10) + (y & 0x03FF);
            i++;
        }

        /* Encode output as utf-8 */
        if (x <= 0x7F)
            output += String.fromCharCode(x);
        else if (x <= 0x7FF)
            output += String.fromCharCode(0xC0 | ((x >>> 6 ) & 0x1F),
                0x80 | ( x & 0x3F));
        else if (x <= 0xFFFF)
            output += String.fromCharCode(0xE0 | ((x >>> 12) & 0x0F),
                0x80 | ((x >>> 6 ) & 0x3F),
                0x80 | ( x & 0x3F));
        else if (x <= 0x1FFFFF)
            output += String.fromCharCode(0xF0 | ((x >>> 18) & 0x07),
                0x80 | ((x >>> 12) & 0x3F),
                0x80 | ((x >>> 6 ) & 0x3F),
                0x80 | ( x & 0x3F));
    }
    return output;
}
389553
function buildInputsArray(rawInputsString) {
  let returnArray = []; // eslint-disable-line
  const rawMethodInputs = rawInputsString.split(',');

  // no inputs
  if (typeof rawMethodInputs === 'undefined' || rawMethodInputs.length === 0) {
    return [];
  }

  rawMethodInputs.forEach((rawMethodInput) => {
    const inputData = rawMethodInput.trim().split(' ');
    const type = inputData[0];
    const name = inputData[1] || '';

    // if type exists
    if (type !== '' && typeof type !== 'undefined') {
      returnArray.push({
        type,
        name,
      });
    }
  });

  return returnArray;
}
183012
function( input ) {

    var self = this

    return input.replace(
      /[=][?]([^?]+)[?]([a-z])[?]([^?]*)[?][=]/gi,
      function( match, charset, type, data ) {

        type = type.toUpperCase()

        return type === 'B' ?
          self.decodeBase64( data, charset ) :
          self.decodeQP( data, charset, true )

      }
    )

  }
863154
function toCodePoint(input, separator = "-") {
  const codePoints = [];
  for (let codePoint of input) {
    codePoints.push(codePoint.codePointAt(0).toString(16));
  }
  return codePoints.join(separator);
}
400241
function purify(input) {
    var result = [], indent = null, codeBlock = false;
    
    // We can have two types of lines:
    // - those beginning with *
    // - and those without it
    input.split("\n").forEach(function(line) {
        // chomp !
        line = line.replace(/(\n|\r)+$/, '');

        if (!codeBlock && /^\s*\*\s*```/.test(line))
          codeBlock = true;
        else if (codeBlock && /^\s*\*\s*```/.test(line))
          codeBlock = false;

        var m;
        if ( ( m = line.match(/^\s*\*\s?(.*)$/)) ) {
            // When comment contains *-lines, switch indent-trimming off
            indent = 0;
            
            if (codeBlock)
              result.push(m[1]);
            else
              result.push(m[1].replace(/^\s+/,""));
        }
        else if (/^\s*$/.test(line)) {
            // pass-through empty lines
            result.push(line);
        }
        else if (indent === undefined && (m = line.match(/^(\s*)(.*?$)/) ) ) {
            // When indent not measured, measure it and remember
            indent = m[1];
            result.push(m[2]);
        }
        else {
            // Trim away indent if available
            //result.push(line.replace(new RegExp("^\s{0," + indent + "}"), ""));
            result.push(line.replace(/^\s+/,""));
        }
    });

    return result.join("\n");
}
519926
function UTF8Encoder(options) {
    var fatal = options.fatal;
    /**
     * @param {Stream} stream Input stream.
     * @param {number} code_point Next code point read from the stream.
     * @return {(number|!Array.<number>)} Byte(s) to emit.
     */
    this.handler = function(stream, code_point) {
      // 1. If code point is end-of-stream, return finished.
      if (code_point === end_of_stream)
        return finished;

      // 2. If code point is in the range U+0000 to U+007F, return a
      // byte whose value is code point.
      if (inRange(code_point, 0x0000, 0x007f))
        return code_point;

      // 3. Set count and offset based on the range code point is in:
      var count, offset;
      // U+0080 to U+07FF, inclusive:
      if (inRange(code_point, 0x0080, 0x07FF)) {
        // 1 and 0xC0
        count = 1;
        offset = 0xC0;
      }
      // U+0800 to U+FFFF, inclusive:
      else if (inRange(code_point, 0x0800, 0xFFFF)) {
        // 2 and 0xE0
        count = 2;
        offset = 0xE0;
      }
      // U+10000 to U+10FFFF, inclusive:
      else if (inRange(code_point, 0x10000, 0x10FFFF)) {
        // 3 and 0xF0
        count = 3;
        offset = 0xF0;
      }

      // 4.Let bytes be a byte sequence whose first byte is (code
      // point >> (6 × count)) + offset.
      var bytes = [(code_point >> (6 * count)) + offset];

      // 5. Run these substeps while count is greater than 0:
      while (count > 0) {

        // 1. Set temp to code point >> (6 × (count − 1)).
        var temp = code_point >> (6 * (count - 1));

        // 2. Append to bytes 0x80 | (temp & 0x3F).
        bytes.push(0x80 | (temp & 0x3F));

        // 3. Decrease count by one.
        count -= 1;
      }

      // 6. Return bytes bytes, in order.
      return bytes;
    };
  }
968532
public static String toStringUTF8(byte[] input)
    {
        if (input == null)
        {
            return null;
        }

        try
        {
            return new String(input, "UTF-8");
        }
        catch (UnsupportedEncodingException e)
        {
            // This won't happen.
            return null;
        }
    }
1108434
public static String encode(byte[] input) {
      if (input.length == 0) {
         return "";
      }
      input = copyOfRange(input, 0, input.length);
      // Count leading zeroes.
      int zeroCount = 0;
      while (zeroCount < input.length && input[zeroCount] == 0) {
         ++zeroCount;
      }
      // The actual encoding.
      byte[] temp = new byte[input.length * 2];
      int j = temp.length;

      int startAt = zeroCount;
      while (startAt < input.length) {
         byte mod = divmod58(input, startAt);
         if (input[startAt] == 0) {
            ++startAt;
         }
         temp[--j] = (byte) ALPHABET[mod];
      }

      // Strip extra '1' if there are some after decoding.
      while (j < temp.length && temp[j] == ALPHABET[0]) {
         ++j;
      }
      // Add as many leading '1' as there were leading zeros.
      while (--zeroCount >= 0) {
         temp[--j] = (byte) ALPHABET[0];
      }

      byte[] output = copyOfRange(temp, j, temp.length);
      try {
         return new String(output, "US-ASCII");
      } catch (UnsupportedEncodingException e) {
         throw new RuntimeException(e); // Cannot happen.
      }
   }
684507
function canConvert(input) {
    return input != null && (typeof input === 'string' || input.nodeType && (input.nodeType === 1 || input.nodeType === 9 || input.nodeType === 11));
}
334656
public static List<Token> tokenize(CharSequence input, boolean textTrim) {
        if (input == null) {
            throw new IllegalArgumentException("input must not be null");
        }

        int inputLen = input.length();
        if (inputLen == 0) {
            List<Token> tokens = new ArrayList<>(1);
            tokens.add(new Token(""));
            return tokens;
        }

        int status = AT_TEXT;
        int tokenStartOffset = 0; // start position of token in the stringBuffer
        char symbol = Token.PARAMETER_SYMBOL; // PARAMETER_SYMBOL or ATTRIBUTE_SYMBOL
        char c;

        StringBuilder nameBuf = new StringBuilder();
        StringBuilder valueBuf = new StringBuilder();
        StringBuilder textBuf = new StringBuilder();

        List<Token> tokens = new ArrayList<>();

        for (int i = 0; i < inputLen; i++) {
            c = input.charAt(i);

            switch (status) {
            case AT_TEXT:
                textBuf.append(c);
                if (Token.isTokenSymbol(c)) {
                    symbol = c;
                    status = AT_TOKEN_SYMBOL;
                    // abc$ --> tokenStartOffset: 3
                    tokenStartOffset = textBuf.length() - 1;
                }
                break;
            case AT_TOKEN_SYMBOL:
                textBuf.append(c);
                if (c == Token.START_BRACKET) {
                    status = AT_TOKEN_NAME;
                } else {
                    if (Token.isTokenSymbol(c)) {
                        symbol = c;
                        status = AT_TOKEN_SYMBOL;
                        // abc$ --> tokenStartOffset: 3
                        tokenStartOffset = textBuf.length() - 1;
                    } else {
                        status = AT_TEXT;
                    }
                }
                break;
            case AT_TOKEN_NAME:
            case AT_TOKEN_VALUE:
                textBuf.append(c);
                if (status == AT_TOKEN_NAME) {
                    if (c == Token.VALUE_SEPARATOR) {
                        status = AT_TOKEN_VALUE;
                        break;
                    }
                }
                if (c == Token.END_BRACKET) {
                    if (nameBuf.length() > 0 || valueBuf.length() > 0) {
                        // save previous non-token string
                        if (tokenStartOffset > 0) {
                            String text = trimBuffer(textBuf, tokenStartOffset, textTrim);
                            Token token = new Token(text);
                            tokens.add(token);
                        }

                        // save token name and default value
                        Token token = createToken(symbol, nameBuf, valueBuf);
                        tokens.add(token);

                        status = AT_TEXT;
                        textBuf.setLength(0);
                        break;
                    }

                    status = AT_TEXT;
                    break;
                }
                if (status == AT_TOKEN_NAME) {
                    if (nameBuf.length() > MAX_TOKEN_NAME_LENGTH) {
                        status = AT_TEXT;
                        nameBuf.setLength(0);
                    }
                    nameBuf.append(c);
                } else {
                    valueBuf.append(c);
                }
                break;
            }
        }

        if (textBuf.length() > 0) {
            String text = trimBuffer(textBuf, textBuf.length(), textTrim);
            Token token = new Token(text);
            tokens.add(token);
        }

        return tokens;
    }
555563
public static String stripNonAlphaNumericCharacters(String input) {
    StringBuilder output = new StringBuilder();
    for (int i = 0; i < input.length(); i++) {
      char current = input.charAt(i);
      if ("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 ".indexOf(current) >= 0)
        output.append(current);
    }
    return output.toString();
  }
156529
def encode(input)
      input_encoding = input.encoding
      input = input.encode(Encoding::UTF_8).codepoints.to_a
      output = []

      # Initialize the state:
      n = INITIAL_N
      delta = 0
      bias = INITIAL_BIAS

      # Handle the basic code points:
      output = input.select { |char| char <= ASCII_MAX }

      h = b = output.length

      # h is the number of code points that have been handled, b is the
      # number of basic code points

      output << DELIMITER if b > 0

      # Main encoding loop:
      while h < input.length
        # All non-basic code points < n have been
        # handled already. Find the next larger one:

        m = MAXINT

        input.each do |char|
          m = char if char >= n && char < m
        end

        # Increase delta enough to advance the decoder's
        # <n,i> state to <m,0>, but guard against overflow:

        raise(ConversionError, "punycode_overflow (1)") if m - n > ((MAXINT - delta) / (h + 1)).floor

        delta += (m - n) * (h + 1)
        n = m

        input.each_with_index do |char, _|
          if char < n
            delta += 1
            raise(ConversionError, "punycode_overflow(2)") if delta > MAXINT
          end

          next unless char == n

          # Represent delta as a generalized variable-length integer:
          q = delta
          k = BASE
          loop do
            t = k <= bias ? TMIN : k >= bias + TMAX ? TMAX : k - bias
            break if q < t
            output << encode_digit(t + (q - t) % (BASE - t))
            q = ((q - t) / (BASE - t)).floor
            k += BASE
          end
          output << encode_digit(q)
          bias = adapt(delta, h + 1, h == b)
          delta = 0
          h += 1
        end

        delta += 1
        n += 1
      end
      output.collect {|c| c.chr(Encoding::UTF_8)}.join(EMPTY).encode(input_encoding)
    end
699751
function codePointAt(str, index) {
  var code = str.charCodeAt(index);
  var low;
  if (0xd800 <= code && code <= 0xdbff) { // High surrogate
    low = str.charCodeAt(index + 1);
    if (!isNaN(low)) {
      code = 0x10000 + (code - 0xd800) * 0x400 + (low - 0xdc00);
    }
  }
  return code;
}