Swap characters from a script to transliteration and vice versa. 
        Optionally sanitize string by using preprocess function.

        :param preprocess:
        :param string:
        :return:
486475
def convert(self, string, preprocess = None):
        
        string = unicode(preprocess(string) if preprocess else string, encoding="utf-8")
        if self.regex:
            return self.regex.sub(lambda x: self.substitutes[x.group()], string).encode('utf-8')
        else:
            return string
808083
def convert_string(string, chars=None):
    """"""
    if chars is None:
        chars = [',', '.', '-', '/', ':', '  ']

    for ch in chars:
        if ch in string:
            string = string.replace(ch, ' ')
    return string
67045
def tmpl_replchars(text, replace, chars):
        
        for char in chars:
            text = text.replace(char, replace)
        return text
896761
def _sanitize(self, value):
        
        if isinstance(value, six.binary_type):
            value = value.decode('utf-8')
        if isinstance(value, six.text_type):
            new_value = ''.join(ch for ch in value if self._valid_char(ch))
        else:
            return value
        # The new string will be equivalent to the original string if no control characters are present.
        # If equivalent, return the original string - some tests check for object equality instead of string equality.
        return value if value == new_value else new_value
1342531
def _swap_curly(string):
    """"""
    return (
        string.replace('{{ ', '{{').replace('{{', '\x00').replace('{', '{{')
        .replace('\x00', '{').replace(' }}', '}}').replace('}}', '\x00')
        .replace('}', '}}').replace('\x00', '}')
    )
31525
def format_raw_script(raw_script):
    
    if six.PY2:
        script = ' '.join(arg.decode('utf-8') for arg in raw_script)
    else:
        script = ' '.join(raw_script)

    return script.strip()
948560
def preprocess(string):
    

    split = string.split('${')

    rval = [split[0]]

    for candidate in split[1:]:
        subsplit = candidate.split('}')

        if len(subsplit) < 2:
            raise ValueError('Open ${ not followed by } before ' \
                    + 'end of string or next ${ in "' \
                    + string + '"')

        varname = subsplit[0]

        if varname == 'PYLEARN2_TRAIN_FILE_NAME':
            warnings.warn("PYLEARN2_TRAIN_FILE_NAME is deprecated and may be "
                    "removed from the library on or after Oct 22, 2013. Switch"
                    " to PYLEARN2_TRAIN_FILE_FULL_STEM")

        try:
            val = os.environ[varname]
        except KeyError:
            if varname == 'PYLEARN2_DATA_PATH':
                raise NoDataPathError()
            if varname == 'PYLEARN2_VIEWER_COMMAND':
                raise EnvironmentVariableError(environment_variable_essay)

            raise ValueError('Unrecognized environment variable "' + varname
                    + '". Did you mean ' + match(varname, os.environ.keys())
                    + '?')

        rval.append(val)

        rval.append('}'.join(subsplit[1:]))

    rval = ''.join(rval)

    return rval
411112
def convert_reserved_characters(string)
    string.gsub!(/&quot;/, '\"')
    string.gsub!(/&apos;/, '\'')
    string.gsub!(/&amp;/,  '\&')
    string.gsub!(/&lt;/,   '\<')
    string.gsub!(/&lt;/,   '\>')
    string
  end
813732
def preprocess(string):
    
    string = unicode(string, encoding="utf-8")
    return regex.sub('', string).encode('utf-8')
804698
def replace_first(input, string, replacement = ''.freeze)
      input.to_s.sub(string.to_s, replacement.to_s)
    end
451125
protected function preprocess() {
		if ( $this->preprocessed || $this->skipPreprocess ) {
			return;
		}

		// Normalize line endings
		if ( strcspn( $this->text, "\r" ) !== strlen( $this->text ) ) {
			$this->text = preg_replace( '/\r\n?/', "\n", $this->text );
			$this->length = strlen( $this->text );
		}

		// Raise parse errors for any control characters
		static $re;
		if ( $re === null ) {
			// Note that we deliberately do not use the 'u' flag on the
			// regexp below, as that bypasses the PCRE JIT.  Instead we
			// rewrite character classes containing codepoints which
			// require more than one UTF-8 byte as alternations.
			$re = '/[' .
				// "C0 controls" (u0000 - u001F) but not
				// "ASCII whitespace" (u0009, u000A, u000C, u000D, u0020) or
				// NULL (u0000)
				// https://infra.spec.whatwg.org/#c0-control
				// https://infra.spec.whatwg.org/#ascii-whitespace
				'\x{0001}-\x{0008}' .
				'\x{000B}' .
				'\x{000E}-\x{001F}' .
				// "Controls" other than C0 controls (u007F - u009F)
				// https://infra.spec.whatwg.org/#control
				'\x{007F}]|' .
				// (We can't use character classes above u007F)
				"\u{0080}|\u{0081}|\u{0082}|\u{0083}|" .
				"\u{0084}|\u{0085}|\u{0086}|\u{0087}|" .
				"\u{0088}|\u{0089}|\u{008A}|\u{008B}|" .
				"\u{008C}|\u{008D}|\u{008E}|\u{008F}|" .
				"\u{0090}|\u{0091}|\u{0092}|\u{0093}|" .
				"\u{0094}|\u{0095}|\u{0096}|\u{0097}|" .
				"\u{0098}|\u{0099}|\u{009A}|\u{009B}|" .
				"\u{009C}|\u{009D}|\u{009E}|\u{009F}|" .
				// HTML spec calls these "noncharacters"
				// https://infra.spec.whatwg.org/#noncharacter
				"\u{FDD0}|\u{FDD1}|\u{FDD2}|\u{FDD3}|" .
				"\u{FDD4}|\u{FDD5}|\u{FDD6}|\u{FDD7}|" .
				"\u{FDD8}|\u{FDD9}|\u{FDDA}|\u{FDDB}|" .
				"\u{FDDC}|\u{FDDD}|\u{FDDE}|\u{FDDF}|" .
				"\u{FDE0}|\u{FDE1}|\u{FDE2}|\u{FDE3}|" .
				"\u{FDE4}|\u{FDE5}|\u{FDE6}|\u{FDE7}|" .
				"\u{FDE8}|\u{FDE9}|\u{FDEA}|\u{FDEB}|" .
				"\u{FDEC}|\u{FDED}|\u{FDEE}|\u{FDEF}|" .
				"\u{FFFE}|\u{FFFF}|" .
				"\u{1FFFE}|\u{1FFFF}|" .
				"\u{2FFFE}|\u{2FFFF}|" .
				"\u{3FFFE}|\u{3FFFF}|" .
				"\u{4FFFE}|\u{4FFFF}|" .
				"\u{5FFFE}|\u{5FFFF}|" .
				"\u{6FFFE}|\u{6FFFF}|" .
				"\u{7FFFE}|\u{7FFFF}|" .
				"\u{8FFFE}|\u{8FFFF}|" .
				"\u{9FFFE}|\u{9FFFF}|" .
				"\u{AFFFE}|\u{AFFFF}|" .
				"\u{BFFFE}|\u{BFFFF}|" .
				"\u{CFFFE}|\u{CFFFF}|" .
				"\u{DFFFE}|\u{DFFFF}|" .
				"\u{EFFFE}|\u{EFFFF}|" .
				"\u{FFFFE}|\u{FFFFF}|" .
				"\u{10FFFE}|\u{10FFFF}/S";
		}
		if ( !$this->ignoreErrors ) {
			$pos = 0;
			while ( $pos < $this->length ) {
				$count = preg_match( $re, $this->text, $m, PREG_OFFSET_CAPTURE, $pos );
				if ( $count === false ) {
					$this->throwPregError();
				} elseif ( !$count ) {
					break;
				}
				$pos = $m[0][1];
				$this->error( "disallowed control character", $pos );
				$pos += strlen( $m[0][0] );
			}
		}
	}
32232
def normalizeUnicode(text, encoding='humanascii'):
    
    if text == "":
	return ""

    unicodeinput = True
    if not isinstance(text, unicode):
        text = unicode(text, 'utf-8')
        unicodeinput = False

    res = ''
    global allowed, allowedid
    if encoding == 'humanascii' or encoding == 'identifier':
        enc = 'ascii'
    else:
        enc = encoding
    for ch in text:
        if (encoding == 'humanascii') and (ch in allowed):
            # ASCII chars, digits etc. stay untouched
            res += ch
            continue
        if (encoding == 'identifier') and (ch in allowedid):
            # ASCII chars, digits etc. stay untouched
            res += ch
            continue
        else:
            try:
                ch.encode(enc,'strict')
                if encoding == 'identifier':
                    res += '_'
                else:
                    res += ch
            except UnicodeEncodeError:
                ordinal = ord(ch)
                if mapping.has_key(ordinal):
                    # try to apply custom mappings
                    res += mapping.get(ordinal)
                elif decomposition(ch) or len(normalize('NFKD',ch)) > 1:
                    normalized = filter(lambda i: not combining(i), normalize('NFKD', ch)).strip()
                    # normalized string may contain non-letter chars too. Remove them
                    # normalized string may result to  more than one char
                    if encoding == 'identifier':
                        res += ''.join([c for c in normalized if c in allowedid])
                    else:
                        res += ''.join([c for c in normalized if c in allowed])
                else:
                    # hex string instead of unknown char
                    res += "%x" % ordinal
    if encoding == 'identifier':
        res = res.strip('_').replace('_____','_').replace('____','_').replace('___','_').replace('__','_')
        if not res.strip('_')[0] in string.ascii_letters:
            res = '_' + res
    if unicodeinput:
        return res
    else:
        return res.encode('utf-8')
798051
def translify(text):
    """"""
    try:
        res = translit.translify(smart_text(text, encoding))
    except Exception as err:
        # because filter must die silently
        res = default_value % {'error': err, 'value': text}
    return res
1131843
def sanitizer(name, replacements=[(':','_'), ('/','_'), ('\\','_')]):
    
    for old,new in replacements:
        name = name.replace(old,new)
    return name
59293
def _preprocess_input(self, input):
        '''
        
        '''
        if not re.search(preprocess_chars, input):
            # No characters that we need to preprocess, so continue without.
            return input

        input = self._add_punctuation_spacing(input)

        return input
558315
def normalize!(options = nil)
      options = default_normalize_options.merge(options || {})

      if translit_option = options[:transliterate]
        if translit_option != true
          transliterate!(*translit_option)
        else
          transliterate!(*options[:transliterations])
        end
      end
      to_ascii! if options[:to_ascii]
      clean!
      word_chars!
      clean!
      downcase!
      truncate_bytes!(options[:max_length])
      with_separators!(options[:separator])
    end
1140054
function swapi(path, value, format) {
    return sanitizeOptions({ path: path, value: value, format: format });
  }