See: https://help.datadoghq.com/hc/en-us/articles/203764705-What-are-valid-metric-names-
<p>
Datadog's publish API will automatically strip Unicode without replacement. It will also replace
all non-alphanumeric characters with '_'.
492493
@Override
    public String name(String name, Meter.Type type, @Nullable String baseUnit) {
        String sanitized = StringEscapeUtils.escapeJson(delegate.name(name, type, baseUnit)
                .replace('/', '_')); // forward slashes, even URL encoded, blow up the POST metadata API

        // Metrics that don't start with a letter get dropped on the floor by the Datadog publish API,
        // so we will prepend them with 'm.'.
        if (!Character.isLetter(sanitized.charAt(0))) {
            sanitized = "m." + sanitized;
        }
        return StringUtils.truncate(sanitized, MAX_NAME_LENGTH);
    }
1253314
private void postMetricMetadata(String metricName, DatadogMetricMetadata metadata) {
        // already posted the metadata for this metric
        if (verifiedMetadata.contains(metricName))
            return;

        try {
            httpClient
                    .put(config.uri() + "/api/v1/metrics/" + URLEncoder.encode(metricName, "UTF-8")
                            + "?api_key=" + config.apiKey() + "&application_key=" + config.applicationKey())
                    .withJsonContent(metadata.editMetadataBody())
                    .send()
                    .onSuccess(response -> verifiedMetadata.add(metricName))
                    .onError(response -> {
                        if (logger.isErrorEnabled()) {
                            String msg = response.body();

                            // Ignore when the response content contains "metric_name not found".
                            // Metrics that are newly created in Datadog are not immediately available
                            // for metadata modification. We will keep trying this request on subsequent publishes,
                            // where it will eventually succeed.
                            if (!msg.contains("metric_name not found")) {
                                logger.error("failed to send metric metadata to datadog: {}", msg);
                            }
                        }
                    });
        } catch (Throwable e) {
            logger.warn("failed to send metric metadata to datadog", e);
        }
    }
283218
private static CharSequence validName(final CharSequence name) {
        if (!RsWithCookie.CNAME_PTRN.matcher(name).matches()) {
            throw new IllegalArgumentException(
                String.format(
                    "Cookie name \"%s\" contains invalid characters",
                    name
                )
            );
        }
        return name;
    }
520273
def xml_entity_escape(data):
    

    data = data.replace("&", "&amp;")
    data = data.replace(">", "&gt;")
    data = data.replace("<", "&lt;")
    return data
374597
def _remove_nonascii(self, df):
    """"""

    df_copy = df.copy(deep=True)
    for col in df_copy.columns:
      if (df_copy[col].dtype == np.dtype('O')):
        df_copy[col] = df[col].apply(
          lambda x: re.sub(r'[^\x00-\x7f]', r'', x) if isinstance(x, six.string_types) else x)

    return df_copy
1023984
def IsValidBase64String(string_to_validate):
    
    # '-' is not supported char for decoding in Python(same as C# and Java) which has similar logic while parsing ResourceID generated by backend
    try:
        buffer = base64.standard_b64decode(string_to_validate.replace('-', '/'))
        if len(buffer) != 4:
            return False
    except Exception as e:
        if six.PY2:
            e = e.message
        if type(e) == binascii.Error:
            return False
        else:
            raise e
    return True
629130
function encodeEntities(value) {
        return value.
            replace(/&/g, '&amp;').
            replace(surrogatePairRegexp, function(value) {
                var hi = value.charCodeAt(0),
                    low = value.charCodeAt(1);
                return '&#' + (((hi - 0xD800) * 0x400) + (low - 0xDC00) + 0x10000) + ';';
            }).
            replace(nonAlphanumericRegexp, function(value) {
                return '&#' + value.charCodeAt(0) + ';';
            }).
            replace(/</g, '&lt;').
            replace(/>/g, '&gt;');
    }
805608
def replace_unicode(self, m):
        """"""

        groups = m.groupdict()
        esc = m.group(0)
        if groups.get('fesc'):
            value = m.group(0)
        elif groups.get('format'):
            value = ' '
        elif groups.get('special'):
            value = BACK_SLASH_TRANSLATION[esc]
        elif groups.get('char'):
            try:
                value = chr(int(esc[2:], 16))
            except Exception:
                value = esc
        elif groups.get('oct'):
            value = chr(int(esc[1:], 8))
        elif groups.get('name'):
            try:
                value = unicodedata.lookup(esc[3:-1])
            except Exception:
                value = esc
        return value.replace('\x00', '\n')
1037407
def slugify(value, allow_unicode=False):
    
    value
    if allow_unicode:
        value = unicodedata.normalize('NFKC', value)
    else:
        value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore').decode('ascii')
    value = re.sub(r'[^\w\s-]', '', value).strip().lower()

    return re.sub(r'[-\s]+', '-', value)
241557
function UnicodeCharacter(fields) {
	if(Array.isArray(fields)) {
		/**
		 *	The numeric character code. For example, for LATIN CAPITAL LETTER A it is 65.
		 *	This member is present on all instances that represent single characters and is absent
		 *	on all instances that represent character ranges.
		 *
		 *	@type {number}
		 */
		this.codePoint = Number.parseInt(fields[0], 16);
		/**
		 *	The official name of the character as given in the Unicode standard. This string is
		 *	always upper-case. For example, the character U+0041 (A) has name “LATIN CAPITAL LETTER
		 *	A”. This emmber is present on all instances.
		 *
		 *	@type {string}
		 */
		this.name = fields[1];
		if(fields[2] && fields[2] !== UnicodeData.prototype.Category['default']) {
			/**
			 *	The two-letter abbreviation for this character’s general category as given in the
			 *	Unicode standard. For example, the character U+0041 (A) is in general category “Lu”
			 *	which includes all upper-case letters. If this member is absent the default value is
			 *	assumed.
			 *
			 *	@type {string}
			 *	@defaultvalue “Lo” (Letter, Other)
			 */
			this.general = fields[2];
		}
		if(fields[3]) {
			let n = Number.parseInt(fields[3]);
			if(n !== UnicodeData.prototype.Combining['default']) {
				/**
				 *	The numeric value of this character’s combining class as given in the Unicode
				 *	standard. For example, the character U+0041 (A) has combining class 0 which
				 *	includes all spacing, split, enclosing, reordrant, and Tibetan subjoined
				 *	characters. If this member is absent the default value is assumed.
				 *
				 *	@type {number}
				 *	@defaultvalue 0 (Spacing, split, enclosing, reordrant, and Tibetan subjoined)
				 */
				this.combining = n;
			}
		}
		if(fields[4] && fields[4] !== UnicodeData.prototype.Bidirectionality['default']) {
			/**
			 *	The two-or three-letter abbreviation that gives the bi-directionality of this
			 *	character as specified in the Unicode standard. For example, the character U+0041
			 *	(A) has bi-directionality “L” i.e., left-to-right. If this member is absent the
			 *	default value is assumed.
			 *
			 *	@type {string}
			 *	@defaultvalue “L” (Left-to-right)
			 */
			this.bidi = fields[4];
		}
		if(fields[5]) {
			let subfields = fields[5].split(' ').map(
				(x) => /\<\w+\>/.test(x) ? x.substring(1,x.length-1) : Number.parseInt(x, 16)
			);
			let decomp = {};
			let tag = UnicodeData.prototype.Decomposition['default'];
			for(let i = 0; i < subfields.length; ++i) {
				if(typeof(subfields[i]) === 'number') {
					if(tag in decomp) {
						decomp[tag].push(subfields[i]);
					} else {
						decomp[tag] = [subfields[i]];
						/**
						 *	The decomposition class of this character as given in the Unicode
						 *	standard. For example, the character U+0041 (A) has “canonical”
						 *	decomposition. If this member is absent, the default value is assumed.
						 *
						 *	@type {string}
						 *	@defaultvalue “canonical” (Canonical decomposition)
						 */
						this.decomp = decomp;
					}
				} else {
					tag = subfields[i];
				}
			}
		}
		if(fields[6]) {
			/**
			 *	The numeric value of this character if it represents a decimal digit. For example, the
			 *	decimal digit value of the character U+0665 (٥) ARABIC-INDIC DIGIT FIVE is 5. If
			 *	this member is absent the character does not represent a decimal digit: it may still
			 *	represent a non-decimal digit or other numeric value.
			 *
			 *	@type {number}
			 */
			this.decimal = Number.parseInt(fields[6]);
			this.digit = this.decimal;
			this.numeric = new Fraction(this.decimal);
		} else if(fields[7]) {
			/**
			 *	The numeric value of the digit if this character represents a digit. For example,
			 *	digit value of the character U+2075 (⁵) SUPERSCRIPT FIVE is 5. If this member is
			 *	absent the character does not represent a digit, but it may still represent a
			 *	numeric value.
			 *
			 *	@type {number}
			 */
			this.digit = Number.parseInt(fields[7]);
			this.numeric = new Fraction(this.digit);
		} else if(fields[8]) {
			/**
			 *	The fractional value of this character if it represents a number. For example,
			 *	the fractional numeric value of the character U+00BC (¼) VULGAR FRACTION ONE QUARTER
			 *	has numericator 1 and denominator 4. If this member is absent the character does
			 *	not have a numeric value.
			 *
			 *	@type {Fraction}
			 */
			this.numeric = new Fraction(fields[8]);
		}
		if(fields[9] === 'Y') {
			/**
			 *	Indicates the character is identified as a “mirrored” character in bi-directional
			 *	text. For example, character U+00AB («) LEFT-POINTING DOUBLE ANGLE QUOTATION MARK
			 *	is mirrored. If this member is absent the default value is assumed.
			 *
			 *	@type {Boolean}
			 *	@defaultvalue false
			 */
			this.mirrored = true;
		}
		if(fields[10]) {
			/**
			 *	The name of the character in the Unicode 1.0 standard, if different from the name
			 *	in the current standard.
			 *
			 *	@type {string}
			 */
			this.oldname = fields[10];
		}
		if(fields[11]) {
			/**
			 *	The comment given for the character in ISO 10646 — the ISO analogue of Unicode — if
			 *	the character has such a comment.
			 *
			 *	@type {string}
			 */
			this.comment = fields[11];
		}
		if(fields[12]) {
			/**
			 *	The numeric value of the code point of the character that is the uppercase
			 *	equivalent of this lowercase character. For example, the uppercase equivalent
			 *	of U+0061 (a) LATIN SMALL LETTER A is 65 (i.e., the code point for U+0041 (A) LATIN
			 *	CAPITAL LETTER A. If this member is absent, the letter has no uppercase equivalent.
			 *
			 *	@type {number}
			 */
			this.uppercase = Number.parseInt(fields[12], 16);
			if(fields[14]) {
				let tc = Number.parseInt(fields[14], 16);
				if(tc !== this.uppercase) {
					/**
					 *	The numeric value of the code point of the character that is the title case
					 *	equivalent of this lowercase character if different from the uppercase
					 *	equivalent. For example, the character U+01F3 (ǳ) LATIN SMALL LETTER DZ
					 *	has an uppercase equivalent of U+01F1 (Ǳ) LATIN CAPITAL LETTER DZ but its
					 *	titlecase equivalent is U+01F2 (ǲ) LATIN CAPITAL LETTER D WITH SMALL LETTER
					 *	Z. If this member is absent the character doesn’t have a titlecase
					 *	equivalent different from the uppercase equivalent.
					 *
					 *	@type {number}
					 */
					this.titlecase = tc;
				}
			}
		}
		if(fields[13]) {
			/**
			 *	The numeric value of the code point of the character that is the lowercase
			 *	equivalent of this uppercase character. For example, the lowercase equivalent
			 *	of U+0041 (A) LATIN CAPITAL LETTER A is 97 (i.e., the code point for U+0061 (a)
			 *	LATIN CAPITAL LETTER A. If this member is absent, the letter has no uppercase
			 *	equivalent.
			 *
			 *	@type {number}
			 */
			this.lowercase = Number.parseInt(fields[13], 16);
		}
		if(!this.name || this.name === '<control>') {
			if(this['oldname']) {
				this.name = this.oldname;
				delete this.oldname;
			} else if(this['comment']) {
				this.name = this.comment;
				delete this.comment;
			}
		} else if(this.name.endsWith(', First>')) {
			this.name = this.name.substring(1, this.name.length - ', First>'.length);
			/**
			 *	The first code point (inclusive) of a range of characters. This member is present
			 *	on all instances that represent range of characters and absent from instances that
			 *	represent single characters.
			 *
			 *	@type {number}
			 */
			this.first = this.codePoint;
			delete this.codePoint;
		} else if(this.name.endsWith(', Last>')) {
			this.name = this.name.substring(1, this.name.length - ', Last>'.length);
			/**
			 *	The last code point (inclusive) of a range of characters. This member is present on
			 *	all instances that represent range of characters and absent from instances that
			 *	represent single characters.
			 *
			 *	@type {number}
			 */
			this.last = this.codePoint;
			delete this.codePoint;
		}
	} else if(typeof(fields) === "object") {
		if('codePoint' in fields) {
			this.codePoint = fields.codePoint;
		} else if(('first' in fields) && ('last' in fields)) {
			this.first = fields.first;
			this.last = fields.last;
		}
		const subfields = [
			'name', 'general', 'combining', 'bidi', 'decomp', 'decimal', 'digit', 'numeric',
			'mirrored', 'oldname', 'comment', 'uppercase', 'titlecase', 'lowercase'
		];
		for(let i = 0; i < subfields.length; ++i) {
			if(subfields[i] in fields) {
				this[subfields[i]] = fields[subfields[i]];
			}
		}
		if('numeric' in this) {
			this.numeric = new Fraction(this.numeric);
		}
	}
}
1321366
private String checkHeaderCharacters(String data) { //PI45266
        // if the last character is a CR or LF, then this fails
        int index = data.length() - 1;
        if (index < 0) {
            // empty string, quit now with success
            return data;
        }
        String error = null;
        char c = data.charAt(index);
        if (BNFHeaders.LF == c || BNFHeaders.CR == c) {
            error = "Illegal trailing EOL";
        }

        // scan through the data now for invalid CRLF presence. Note that CRLFs
        // may be followed by whitespace for valid multiline headers.
        StringBuilder sb = new StringBuilder(); //PI57228
        for (int i = 0; null == error && i <= index; i++) {
            c = data.charAt(i);
            if (i < index) {
                if (BNFHeaders.CR == c) {
                    // next char must be an LF
                    if (BNFHeaders.LF != data.charAt(i + 1)) {
                        error = "Invalid CR not followed by LF";
                    }
                } else if (BNFHeaders.LF == c) {
                    char x = data.charAt(i + 1);
                    // if it is not followed by whitespace then this value is bad
                    if (BNFHeaders.TAB != x && BNFHeaders.SPACE != x) {
                        error = "Invalid LF not followed by whitespace";
                    }
                }
            }
            if (c >= 32 && c < 127) {
                sb.append(c);
            } else if (c == BNFHeaders.LF || c == BNFHeaders.CR) {
                sb.append(' ');
                if (TraceComponent.isAnyTracingEnabled() && tc.isDebugEnabled()) {
                    Tr.debug(tc, "Found a CR or LF, replacing it with SP");
                }
            } else {
                if (TraceComponent.isAnyTracingEnabled() && tc.isDebugEnabled()) {
                    Tr.debug(tc, "The Character: " + c + " is not printable");
                }
                final int maskedCodePoint = c & 0xFF;
                if (maskedCodePoint == BNFHeaders.LF || maskedCodePoint == BNFHeaders.CR) {
                    sb.append('?');
                    if (TraceComponent.isAnyTracingEnabled() && tc.isDebugEnabled()) {
                        Tr.debug(tc, "Character: " + c + " unicode ends with a 0a or 0d, replacing it with ?");
                        Tr.debug(tc, "The Unicode is: " + (char) maskedCodePoint);
                    }
                } else
                    sb.append(c);
            }
        }

        // if we found an error, throw the exception now
        if (null != error) {
            IllegalArgumentException iae = new IllegalArgumentException(error);
            FFDCFilter.processException(iae, getClass().getName() + ".checkHeaderValue(String)", "1", this);
            throw iae;
        }
        return sb.toString();
    }
456057
def remove_diacritics(self_or_cls, identifier):
        
        chars = ''
        for c in identifier:
            replacement = unicodedata.normalize('NFKD', c).encode('ASCII', 'ignore')
            if replacement != '':
                chars += bytes_to_unicode(replacement)
            else:
                chars += c
        return chars
1171687
def check_name(name, safe_chars):
    '''
    
    '''
    regexp = re.compile('[^{0}]'.format(safe_chars))
    if regexp.search(name):
        raise SaltCloudException(
            '{0} contains characters not supported by this cloud provider. '
            'Valid characters are: {1}'.format(
                name, safe_chars
            )
        )
649212
def escape_unicode_string(u):
    
    def replacer(matchobj):
        if ord(matchobj.group(1)) == 127:
            return "\\x7f"
        if ord(matchobj.group(1)) == 92: # backslash
            return "\\\\"
        return REPLACEMENT_TABLE[ord(matchobj.group(1))]
    return re.sub("([\\000-\\037\\134\\177])", replacer, u)
7622
def _to_unicode(self, data, encoding, errors='strict'):
    ''''''

    # strip Byte Order Mark (if present)
    if (len(data) >= 4) and (data[:2] == '\xfe\xff') and (data[2:4] != '\x00\x00'):
        encoding = 'utf-16be'
        data = data[2:]
    elif (len(data) >= 4) and (data[:2] == '\xff\xfe') and (data[2:4] != '\x00\x00'):
        encoding = 'utf-16le'
        data = data[2:]
    elif data[:3] == '\xef\xbb\xbf':
        encoding = 'utf-8'
        data = data[3:]
    elif data[:4] == '\x00\x00\xfe\xff':
        encoding = 'utf-32be'
        data = data[4:]
    elif data[:4] == '\xff\xfe\x00\x00':
        encoding = 'utf-32le'
        data = data[4:]
    newdata = unicode(data, encoding, errors)
    return newdata
939914
public static UnicodeFriendlyString unicodeFriendly(String utf16CodeUnits) {
    if (utf16CodeUnits.codePointCount(0, utf16CodeUnits.length()) == utf16CodeUnits.length()) {
      return StringWithoutNonBmp.of(utf16CodeUnits);
    } else {
      return StringWithNonBmp.of(utf16CodeUnits);
    }
  }
448294
def unicode_safe(text, encoding=UrlEncoding, errors='ignore'):
    """"""
    try:
        text_type = unicode
    except NameError:
        text_type = str
    if isinstance(text, text_type):
        return text
    return text.decode(encoding, errors)