* What you can do
    - It converts input-text into normalized-text which is good for tokenizer input.

    * Params
    - new_line_replaced: a string which replaces from \n string.
424912
def normalize_text(input_text,
                   dictionary_mode='ipadic',
                   new_line_replaced='。',
                   is_replace_eos=True,
                   is_kana=True,
                   is_ascii=True,
                   is_digit=True):
    # type: (text_type,text_type,text_type,bool,bool,bool,bool)->text_type
    
    if is_replace_eos:
        without_new_line = input_text.replace('\n', new_line_replaced)
    else:
        without_new_line = new_line_replaced

    if dictionary_mode=='neologd' and is_neologdn_valid:
        return neologdn.normalize(normalize_text_normal_ipadic(without_new_line))
    elif dictionary_mode=='neologd' and is_neologdn_valid == False:
        raise Exception("You could not call neologd dictionary bacause you do NOT install the package neologdn.")
    else:
        return normalize_text_normal_ipadic(without_new_line, kana=is_kana, ascii=is_ascii, digit=is_digit)
675853
function rawLineToText (line) {
  const cut = line.indexOf(' ')
  return cut === -1
    ? ''
    : line.slice(cut + 1).trim()
}
893676
def fix_line_breaks(s):
    
    l = s.splitlines()
    x = [i.strip() for i in l]
    x = [i for i in x if i]  # remove blank lines
    return "\n".join(x)
919767
def _wrap_splitter_with_newline(msg_body, delimiter, content_type='text/plain'):
    
    def splitter_wrapper(splitter):
        """Wraps splitter with new line"""
        if splitter.start() and msg_body[splitter.start() - 1] != '\n':
            return '%s%s' % (delimiter, splitter.group())
        else:
            return splitter.group()

    if content_type == 'text/plain':
        msg_body = re.sub(RE_ON_DATE_SMB_WROTE, splitter_wrapper, msg_body)

    return msg_body
857577
protected function replaceParam($line, array $param)
    {
        if (! empty($param) and is_array($param)) {
            foreach ($param as $key => $value) {
                $line = str_replace(':'.$key, $value, $line);
            }
        }

        return $line;
    }
693912
def tokenize(self, string):
        '''
        
        '''

        new_lexer = ply.lex.lex(module=self, debug=self.debug, errorlog=logger)
        new_lexer.latest_newline = 0
        new_lexer.string_value = None
        new_lexer.input(string)

        while True:
            t = new_lexer.token()
            if t is None: break
            t.col = t.lexpos - new_lexer.latest_newline
            yield t

        if new_lexer.string_value is not None:
            raise JsonPathLexerError('Unexpected EOF in string literal or identifier')
828377
def _prepare_text(self, text):
        
        # Remove characters that should be escaped for XML input (but
        # which cause problems when escaped, since they become
        # tokens).
        text = re.sub(r'[<>&]', '', text)
        pattern = r'({})'.format(self._tokenizer.pattern)
        return re.sub(pattern, self._base_token_markup, text)
727170
function parse(text, language) {
  return text.split(newlines)
             .map(parseLine(language))
             .reduce(assimilateSimilarTokens, []) }
140829
def normalize_linefeeds(self, a_string):
        """"""
        newline = re.compile(r"(\r\r\n|\r\n)")
        # NX-OS fix for incorrect MD5 on 9K (due to strange <enter> patterns on NX-OS)
        return newline.sub(self.RESPONSE_RETURN, a_string).replace("\r", "\n")
1296178
private String replaceParameters(String text) {
    	Map<String, Object> params  = bean.getParametersBean().getParamValues();
        for  (String param : params.keySet()) {
             text = StringUtil.replace(text, "\\$P\\{" + param + "\\}", StringUtil.getValueAsString(params.get(param), null));
        }
        return text;
    }
1138370
def replace(input, string, replacement = ''.freeze)
      input.to_s.gsub(string.to_s, replacement.to_s)
    end
1074831
def convert_line(line_num)
      return '' unless @callouts.key? line_num

      @callouts[line_num]
        .map { |num| convert_callout(num) }
        .join(' ')
    end
322688
def tokenize(self, text: str, customize=True, disable=[]) -> List[Token]:
        

        """Tokenize text"""
        if not self.keep_multi_space:
            text = re.sub(' +', ' ', text)
        # disable spacy parsing, tagging etc as it takes a long time if the text is short
        tokens = self.nlp(text, disable=disable)
        if customize:
            tokens = [self.custom_token(a_token) for a_token in tokens]

        return tokens
543116
def _shorten_line_at_tokens_new(tokens, source, indentation,
                                max_line_length):
    
    # Yield the original source so to see if it's a better choice than the
    # shortened candidate lines we generate here.
    yield indentation + source

    parsed_tokens = _parse_tokens(tokens)

    if parsed_tokens:
        # Perform two reflows. The first one starts on the same line as the
        # prefix. The second starts on the line after the prefix.
        fixed = _reflow_lines(parsed_tokens, indentation, max_line_length,
                              start_on_prefix_line=True)
        if fixed and check_syntax(normalize_multiline(fixed.lstrip())):
            yield fixed

        fixed = _reflow_lines(parsed_tokens, indentation, max_line_length,
                              start_on_prefix_line=False)
        if fixed and check_syntax(normalize_multiline(fixed.lstrip())):
            yield fixed
1145870
def _input_as_multiline_string(self, data):
        
        if data:
            self.Parameters['--in']\
                .on(super(Clearcut,self)._input_as_multiline_string(data))
        return ''
174305
def __normalize(self,text):
        
        return self._normalizer_regex.sub(lambda mo:
                self._normalizer[mo.string[mo.start():mo.end()]],
                    text.lower())
778837
function tokenizeText(text, callback) {
    "use strict";
    processTokens(tokenizer.tokenize(text) || [], function (error, tokens) {
        process.nextTick(function () {
            callback(error, tokens);
        });
    });
}