Normalizes text.
            Converts to lowercase,
            Unicode NFC normalization
            and removes mentions and links

            :param text: Text to be normalized.
482060
def normalize(self, text):
        
        #print 'Normalize...\n'
        text = text.lower()
        text = unicodedata.normalize('NFC', text)
        text = self.strip_mentions_links(text)
        return text
755519
def normalize_nfc(txt):
    
    if isinstance(txt, bytes):
        txt = txt.decode()
    return unicodedata.normalize("NFC", txt).encode()
451224
private String normalizeText(String text) {
        String[] tokens = StringUtils.split(text, "\n");
        for (int i = 0; i < tokens.length; i++) {
            tokens[i] = tokens[i].trim();
        }

        return StringUtils.join(tokens, " ").trim();
    }
1121305
def parse(text, **options):
    
    _options = copy.copy(DEFAULT_OPTIONS)
    _options.update(options)

    return _normalize(_parse(text, **_options), **_options)
999493
protected static String preNormalize(String text) {
    String normalizedText = text.toLowerCase();
    normalizedText = Chronic.numericizeNumbers(normalizedText);
    normalizedText = normalizedText.replaceAll("['\"\\.]", "");
    normalizedText = normalizedText.replaceAll("([/\\-,@])", " $1 ");
    normalizedText = normalizedText.replaceAll("\\btoday\\b", "this day");
    normalizedText = normalizedText.replaceAll("\\btomm?orr?ow\\b", "next day");
    normalizedText = normalizedText.replaceAll("\\byesterday\\b", "last day");
    normalizedText = normalizedText.replaceAll("\\bnoon\\b", "12:00");
    normalizedText = normalizedText.replaceAll("\\bmidnight\\b", "24:00");
    normalizedText = normalizedText.replaceAll("\\bbefore now\\b", "past");
    normalizedText = normalizedText.replaceAll("\\bnow\\b", "this second");
    normalizedText = normalizedText.replaceAll("\\b(ago|before)\\b", "past");
    normalizedText = normalizedText.replaceAll("\\bthis past\\b", "last");
    normalizedText = normalizedText.replaceAll("\\bthis last\\b", "last");
    normalizedText = normalizedText.replaceAll("\\b(?:in|during) the (morning)\\b", "$1");
    normalizedText = normalizedText.replaceAll("\\b(?:in the|during the|at) (afternoon|evening|night)\\b", "$1");
    normalizedText = normalizedText.replaceAll("\\btonight\\b", "this night");
    normalizedText = normalizedText.replaceAll("(?=\\w)([ap]m|oclock)\\b", " $1");
    normalizedText = normalizedText.replaceAll("\\b(hence|after|from)\\b", "future");
    normalizedText = Chronic.numericizeOrdinals(normalizedText);
    return normalizedText;
  }
1336307
def clean_string(text):
    
    if isinstance(text, six.string_types):
        return text.translate(UNI_SPECIAL_CHARS).strip()
    return text.translate(None, STR_SPECIAL_CHARS).strip()
361523
def decamelise(text):
    """"""
    s = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', text)
    return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s).lower()
944393
private void normaliseTextNodes(Element element) {
        List<Node> toMove = new ArrayList<>();
        for (Node node: element.childNodes) {
            if (node instanceof TextNode) {
                TextNode tn = (TextNode) node;
                if (!tn.isBlank())
                    toMove.add(tn);
            }
        }

        for (int i = toMove.size()-1; i >= 0; i--) {
            Node node = toMove.get(i);
            element.removeChild(node);
            body().prependChild(new TextNode(" "));
            body().prependChild(node);
        }
    }
366215
def normalize_eols(text, eol='\n'):
    """"""
    for eol_char, _ in EOL_CHARS:
        if eol_char != eol:
            text = text.replace(eol_char, eol)
    return text
1298495
def normalize_value(text):
    
    result = text.replace('\n', ' ')
    result = re.subn('[ ]{2,}', ' ', result)[0]
    return result
557459
def _normalize_string(raw_str):
  
  return " ".join(
      token.strip()
      for token in tokenizer.encode(text_encoder.native_to_unicode(raw_str)))
513953
def _normalize_str(self, string):
        
        if string:
            if not isinstance(string, str):
                string = str(string, 'utf-8', 'replace')

            return unicodedata.normalize('NFKD', string).encode(
                'ASCII', 'ignore').decode('ASCII')
        return ''
435855
public static String clear(String text) {
        text = StringCleaner.tokenizeURLs(text);
        text = StringCleaner.tokenizeSmileys(text);
        text = StringCleaner.removeAccents(text);
        text = StringCleaner.removeSymbols(text);
        text = StringCleaner.removeExtraSpaces(text);
        
        return text.toLowerCase(Locale.ENGLISH);
    }
466337
def clean_text(text):
    """"""
    # Replace a few nasty unicode characters with their ASCII equivalent
    maps = {u'×': u'x', u'–': u'-', u'−': '-'}
    for element in maps:
        text = text.replace(element, maps[element])

    # Replace genitives
    text = re.sub(r'(?<=\w)\'s\b|(?<=\w)s\'(?!\w)', '  ', text)

    logging.debug(u'Clean text: "%s"', text)

    return text
1147675
def normalize_text(self, text):
        
        if not self.editor.free_format:
            text = ' ' * 6 + text[6:]
        return text.upper()
828377
def _prepare_text(self, text):
        
        # Remove characters that should be escaped for XML input (but
        # which cause problems when escaped, since they become
        # tokens).
        text = re.sub(r'[<>&]', '', text)
        pattern = r'({})'.format(self._tokenizer.pattern)
        return re.sub(pattern, self._base_token_markup, text)
710178
def normalise_string(string):
    
    string = (string.strip()).lower()
    return re.sub(r'\W+', '_', string)