Preprocesses _input_ to prepare it for the tokenizer.

 3.3. http://dev.w3.org/csswg/css-syntax/#input-preprocessing
198445
def preprocess(input)
      input = input.to_s.encode('UTF-8',
        :invalid => :replace,
        :undef   => :replace)

      input.gsub!(/(?:\r\n|[\r\f])/, "\n")
      input.gsub!("\u0000", "\ufffd")
      input
    end
762721
def preprocess(source):
  
  doc = html5lib.parseFragment(source)
  source = ET.tostring(doc, encoding='utf-8', method='text').decode('utf-8')
  source = source.replace(u'\n', u'').strip()
  source = re.sub(r'\s\s+', u' ', source)
  return source
1280705
def tokenize(expr):
    
    tokens = []
    escape = False
    cur_token = ''

    for c in expr:
        if escape == True:
            cur_token += c
            escape = False
        else:
            if c == '\\':
                # Next char will be escaped
                escape = True
                continue
            elif c == '[':
                # Next token is of type index (list)
                if len(cur_token) > 0:
                    tokens.append(cur_token)
                    cur_token = ''
            elif c == ']':
                # End of index token. Next token defaults to a key (dict)
                if len(cur_token) > 0:
                    tokens.append(int(cur_token))
                    cur_token = ''
            elif c == '.':
                # End of key token. Next token defaults to a key (dict)
                if len(cur_token) > 0:
                    tokens.append(cur_token)
                    cur_token = ''
            else:
                # Append char to token name
                cur_token += c
    if len(cur_token) > 0:
        tokens.append(cur_token)

    return tokens
653654
def preprocessor(dom):
    ""
    options = {
        "processing_instructions": False,
        "remove_unknown_tags": False,
        "safe_attrs_only": False,
        "page_structure": False,
        "annoying_tags": False,
        "frames": False,
        "meta": False,
        "links": False,
        "javascript": False,
        "scripts": True,
        "comments": True,
        "style": True,
        "embedded": True,
        "forms": True,
        "kill_tags": ("head",),
    }
    cleaner = Cleaner(**options)

    return cleaner.clean_html(dom)
794735
def _preprocess_chars(self, chars):
        '''
        
        '''
        chars = self._normalize_dakuten(chars)
        chars = self._process_repeaters(chars)
        chars = self._perform_replacements(chars)

        return chars
487702
function tokensId(tokens, prepend) {
    if (!prepend) return null;

    return _.reduce(tokens, function(prev, token) {
        var tokId = [
            token.index,
            token.offset,
            token.value
        ].join('-');

        return [prev, tokId].join(':');
    }, prepend);
}
400241
function purify(input) {
    var result = [], indent = null, codeBlock = false;
    
    // We can have two types of lines:
    // - those beginning with *
    // - and those without it
    input.split("\n").forEach(function(line) {
        // chomp !
        line = line.replace(/(\n|\r)+$/, '');

        if (!codeBlock && /^\s*\*\s*```/.test(line))
          codeBlock = true;
        else if (codeBlock && /^\s*\*\s*```/.test(line))
          codeBlock = false;

        var m;
        if ( ( m = line.match(/^\s*\*\s?(.*)$/)) ) {
            // When comment contains *-lines, switch indent-trimming off
            indent = 0;
            
            if (codeBlock)
              result.push(m[1]);
            else
              result.push(m[1].replace(/^\s+/,""));
        }
        else if (/^\s*$/.test(line)) {
            // pass-through empty lines
            result.push(line);
        }
        else if (indent === undefined && (m = line.match(/^(\s*)(.*?$)/) ) ) {
            // When indent not measured, measure it and remember
            indent = m[1];
            result.push(m[2]);
        }
        else {
            // Trim away indent if available
            //result.push(line.replace(new RegExp("^\s{0," + indent + "}"), ""));
            result.push(line.replace(/^\s+/,""));
        }
    });

    return result.join("\n");
}
1145004
def markdownify(input)
      site = @context.registers[:site]
      converter = site.find_converter_instance(Bunto::Converters::Markdown)
      converter.convert(input.to_s)
    end
86031
function Tokenizer(str) {

    this.str = (str || '').toString();
    this.operatorCurrent = '';
    this.operatorExpecting = '';
    this.node = null;
    this.escaped = false;

    this.list = [];

}
486475
def convert(self, string, preprocess = None):
        
        string = unicode(preprocess(string) if preprocess else string, encoding="utf-8")
        if self.regex:
            return self.regex.sub(lambda x: self.substitutes[x.group()], string).encode('utf-8')
        else:
            return string
813732
def preprocess(string):
    
    string = unicode(string, encoding="utf-8")
    return regex.sub('', string).encode('utf-8')
530342
def input(self):
        
        def decorator(inputMethod):
            return MethodicalInput(automaton=self._automaton,
                                   method=inputMethod,
                                   symbol=self._symbol)
        return decorator
595373
function prepareInternalJSCode(source, options) {
	// Parse
	let tokens = parse(source)

	// Transform
	if (options.transformer) {
		tokens = options.transformer(tokens) || tokens
	}

	let reducedTokens = reduce(tokens, options)

	return createCode(reducedTokens, options, false)
}
551934
function(input, options) {
		this.preprocess(input, options);
		for (var inputIndex=0; inputIndex < this.inputList.length; ++inputIndex)
		{
			this.input = this.inputList[inputIndex][1][1].replace(/\\/g,'\\\\');
			
			// first stage : configurable crusher
			var output = this.findRedundancies(options);
			this.inputList[inputIndex].push(output);
			
			// second stage : convert token string to regexp
			output = packer.packToRegexpCharClass(options);
			this.inputList[inputIndex].push(output);
			
			// third stage : try a negated regexp instead
			output = packer.packToNegatedRegexpCharClass();
			this.inputList[inputIndex].push(output);
			
		}
	}
676764
def whitespace_token
      if md=WHITESPACE.match(@chunk)
        input = md.to_a[0]
        input.length
      else
        did_not_match
      end
    end
1102719
def process_input(self, paragraph):
        """"""
        try:
            lang, body = paragraph.split(None, 1)
        except ValueError:
            lang, body = paragraph, None

        if not lang.strip().startswith('%'):
            lang = 'scala'
            body = paragraph.strip()

        else:
            lang = lang.strip()[1:]

        if lang == 'md':
            self.build_markdown(lang, body)
        else:
            self.build_code(lang, body)
1321558
def span_tokenize(self, s):
        
        if self._tokenizer is None:
            self._tokenizer = load_model(self.model)
        # for debug in tokenizer.debug_decisions(s):
        #     log.debug(format_debug_decision(debug))
        return self._tokenizer.span_tokenize(s)