Apply text standardization following original implementation.
487270
def text_standardize(text):
    
    text = text.replace('—', '-')
    text = text.replace('–', '-')
    text = text.replace('―', '-')
    text = text.replace('…', '...')
    text = text.replace('´', "'")
    text = re.sub(r'''(-+|~+|!+|"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)''', r' \1 ', text)
    text = re.sub(r'\s*\n\s*', ' \n ', text)
    text = re.sub(r'[^\S\n]+', ' ', text)
    return text.strip()
40827
public void fitOnTexts(String[] texts) {
        String[] sequence;
        for (String text : texts) {
            if (documentCount == null)
                documentCount = 1;
            else
                documentCount += 1;
            if (charLevel) {
                if (lower)
                    text = text.toLowerCase();
                sequence = text.split("");
            } else {
                sequence = textToWordSequence(text, filters, lower, split);
            }
            for (String word : sequence) {
                if (wordCounts.containsKey(word))
                    wordCounts.put(word, wordCounts.get(word) + 1);
                else
                    wordCounts.put(word, 1);
            }
            Set<String> sequenceSet = new HashSet<>(Arrays.asList(sequence));
            for (String word: sequenceSet) {
                if (wordDocs.containsKey(word))
                    wordDocs.put(word, wordDocs.get(word) + 1);
                else
                    wordDocs.put(word, 1);
            }
        }
        Map<String, Integer> sortedWordCounts = reverseSortByValues((HashMap) wordCounts);

        ArrayList<String> sortedVocabulary = new ArrayList<>();
        if (outOfVocabularyToken != null)
            sortedVocabulary.add(outOfVocabularyToken);
        for (String word: sortedWordCounts.keySet()) {
            sortedVocabulary.add(word);
        }

        for (int i = 0; i < sortedVocabulary.size(); i++)
            wordIndex.put(sortedVocabulary.get(i), i+1);

        for(String key : wordIndex.keySet()){
            indexWord.put(wordIndex.get(key), key);
        }

        for (String key: wordDocs.keySet())
            indexDocs.put(wordIndex.get(key), wordDocs.get(key));
    }
1187373
def apply(self):
        """"""
        # In this phase, we look for hyperlinks (references nodes)
        # that contain substitutions (of the form "|foo|").
        # We then add actual "substitution"s nodes to those references,
        # so that they can be replaced by the substitution processor.
        subst_re = re.compile(self.subst_pattern)

        for link in self.document.traverse(self._maybe_hyperlink):
            if 'refuri' not in link:
                continue

            # Note: "target" nodes do not have a "name" attribute.
            if '|' not in link['refuri'] and '|' not in link.get('name', ''):
                continue

            # This list acts as a cache so that only one substitution node
            # is added as a child for each substitution name.
            substitutions = []

            matches = subst_re.findall(link['refuri']) + \
                subst_re.findall(link.get('name', ''))
            for subref_text in matches:
                if subref_text in substitutions:
                    continue

                substitutions.append(subref_text)
                subref_node = nodes.substitution_reference(subref_text)
                link.append(subref_node)
                self.document.note_substitution_ref(subref_node, subref_text)

            # Build a map of substitutions names to child indices
            # (minus one since the actual link label is in link[0]).
            link['varlinks'] = \
                dict(zip(substitutions, range(len(substitutions))))
539511
def iter_text_fixups(data, encoding):
    """"""

    yield data
    if encoding == Encoding.UTF16BE:
        # wrong termination
        yield data + b"\x00"
    elif encoding == Encoding.UTF16:
        # wrong termination
        yield data + b"\x00"
        # utf-16 is missing BOM, content is usually utf-16-le
        yield codecs.BOM_UTF16_LE + data
        # both cases combined
        yield codecs.BOM_UTF16_LE + data + b"\x00"
706677
def edit_message_text(self, *args, **kwargs):
        """"""
        return edit_message_text(*args, **self._merge_overrides(**kwargs)).run()
1223716
def convert(self, text, encoding):
        """"""

        if self.normalize in ('NFC', 'NFKC', 'NFD', 'NFKD'):
            text = unicodedata.normalize(self.normalize, text)
        if self.convert_encoding:
            text = text.encode(self.convert_encoding, self.errors).decode(self.convert_encoding)
            encoding = self.convert_encoding
        return text, encoding
59445
def sanitize_text(text):
    """"""
    # First replace characters that have specific effects with their repr #
    text = re.sub("(\s)", lambda m: repr(m.group(0)).strip("'"), text)
    # Make it a unicode string (the try supports python 2 and 3) #
    try: text = text.decode('utf-8')
    except AttributeError: pass
    # Normalize it “
    text = unicodedata.normalize('NFC', text)
    return text
1105460
def legitimize(text):
    
    import platform
    os_ = platform.system()
    # POSIX systems
    text = text.translate({
        0: None,
        ord('/'): '-',
        ord('|'): '-',
    })

    if os_ == 'Windows':
        # Windows (non-POSIX namespace)
        text = text.translate({
            # Reserved in Windows VFAT and NTFS
            ord(':'): '-',
            ord('*'): '-',
            ord('?'): '-',
            ord('\\'): '-',
            ord('\"'): '\'',
            # Reserved in Windows VFAT
            ord('+'): '-',
            ord('<'): '-',
            ord('>'): '-',
            ord('['): '(',
            ord(']'): ')',
        })
    else:
        # *nix
        if os_ == 'Darwin':
            # Mac OS HFS+
            text = text.translate({
                ord(':'): '-',
            })

        # Remove leading .
        if text.startswith("."):
            text = text[1:]

    text = text[:82]  # Trim to 82 Unicode characters long
    return text
469159
def text(self, normalize_spaces=False):
        """"""
        return super(TextContent,self).text(normalize_spaces=normalize_spaces)
885372
def setText( self, text ):
        
        if text is None:
            text = ''
        
        super(XLineEdit, self).setText(projex.text.encoded(self.formatText(text), self.encoding()))
1003105
def convert(self, text):
        """"""
        # Main function. The order in which other subs are called here is
        # essential. Link and image substitutions need to happen before
        # _EscapeSpecialChars(), so that any *'s or _'s in the <a>
        # and <img> tags get encoded.

        # Clear the global hashes. If we don't clear these, you get conflicts
        # from other articles when generating a page which contains more than
        # one article (e.g. an index page that shows the N most recent
        # articles):
        self.reset()

        if not isinstance(text, unicode):
            #TODO: perhaps shouldn't presume UTF-8 for string input?
            text = unicode(text, 'utf-8')

        if self.use_file_vars:
            # Look for emacs-style file variable hints.
            emacs_vars = self._get_emacs_vars(text)
            if "markdown-extras" in emacs_vars:
                splitter = re.compile("[ ,]+")
                for e in splitter.split(emacs_vars["markdown-extras"]):
                    if '=' in e:
                        ename, earg = e.split('=', 1)
                        try:
                            earg = int(earg)
                        except ValueError:
                            pass
                    else:
                        ename, earg = e, None
                    self.extras[ename] = earg

        # Standardize line endings:
        text = re.sub("\r\n|\r", "\n", text)

        # Make sure $text ends with a couple of newlines:
        text += "\n\n"

        # Convert all tabs to spaces.
        text = self._detab(text)

        # Strip any lines consisting only of spaces and tabs.
        # This makes subsequent regexen easier to write, because we can
        # match consecutive blank lines with /\n+/ instead of something
        # contorted like /[ \t]*\n+/ .
        text = self._ws_only_line_re.sub("", text)

        # strip metadata from head and extract
        if "metadata" in self.extras:
            text = self._extract_metadata(text)

        text = self.preprocess(text)

        if self.safe_mode:
            text = self._hash_html_spans(text)

        # Turn block-level HTML blocks into hash entries
        text = self._hash_html_blocks(text, raw=True)

        # Strip link definitions, store in hashes.
        if "footnotes" in self.extras:
            # Must do footnotes first because an unlucky footnote defn
            # looks like a link defn:
            #   [^4]: this "looks like a link defn"
            text = self._strip_footnote_definitions(text)
        text = self._strip_link_definitions(text)

        text = self._run_block_gamut(text)

        if "footnotes" in self.extras:
            text = self._add_footnotes(text)

        text = self.postprocess(text)

        text = self._unescape_special_chars(text)

        if self.safe_mode:
            text = self._unhash_html_spans(text)

        if "nofollow" in self.extras:
            text = self._a_nofollow.sub(r'<\1 rel="nofollow"\2', text)

        text += "\n"

        rv = UnicodeWithAttrs(text)
        if "toc" in self.extras:
            rv._toc = self._toc
        if "metadata" in self.extras:
            rv.metadata = self.metadata
        return rv
775580
private void adaptHelperText() {
        if (textInputLayout != null && TextUtils.isEmpty(getText())) {
            textInputLayout.setHelperText(helperText);
            textInputLayout.setHelperTextEnabled(true);
        }
    }
585067
def text_normalize(text):
    
    # if not isinstance(text, str):
    if isinstance(text, bytes):
        # -- MAYBE: command.ouput => bytes, encoded stream output.
        text = codecs.decode(text)
    lines = [ line.strip()  for line in text.splitlines()  if line.strip() ]
    return "\n".join(lines)
1033843
function normalizeToBase(text) {
		var rExps = [
			{re: /[\xC0-\xC6]/g, ch: "A"},
			{re: /[\xE0-\xE6]/g, ch: "a"},
			{re: /[\xC8-\xCB]/g, ch: "E"},
			{re: /[\xE8-\xEB]/g, ch: "e"},
			{re: /[\xCC-\xCF]/g, ch: "I"},
			{re: /[\xEC-\xEF]/g, ch: "i"},
			{re: /[\xD2-\xD6]/g, ch: "O"},
			{re: /[\xF2-\xF6]/g, ch: "o"},
			{re: /[\xD9-\xDC]/g, ch: "U"},
			{re: /[\xF9-\xFC]/g, ch: "u"},
			{re: /[\xC7-\xE7]/g, ch: "c"},
			{re: /[\xD1]/g, ch: "N"},
			{re: /[\xF1]/g, ch: "n"}
		];
		$.each(rExps, function () {
			text = text.replace(this.re, this.ch);
		});
		return text;
	}
1164241
@Override
    protected void handleTransliterate(Replaceable text,
            Position offsets, boolean isIncremental) {
        // start and limit of the input range
        int start = offsets.start;
        int limit = offsets.limit;
        if(start >= limit) {
            return;
        }

        /*
         * Normalize as short chunks at a time as possible even in
         * bulk mode, so that styled text is minimally disrupted.
         * In incremental mode, a chunk that ends with offsets.limit
         * must not be normalized.
         *
         * If it was known that the input text is not styled, then
         * a bulk mode normalization could be used.
         * (For details, see the comment in the C++ version.)
         */
        StringBuilder segment = new StringBuilder();
        StringBuilder normalized = new StringBuilder();
        int c = text.char32At(start);
        do {
            int prev = start;
            // Skip at least one character so we make progress.
            // c holds the character at start.
            segment.setLength(0);
            do {
                segment.appendCodePoint(c);
                start += Character.charCount(c);
            } while(start < limit && !norm2.hasBoundaryBefore(c = text.char32At(start)));
            if(start == limit && isIncremental && !norm2.hasBoundaryAfter(c)) {
                // stop in incremental mode when we reach the input limit
                // in case there are additional characters that could change the
                // normalization result
                start=prev;
                break;
            }
            norm2.normalize(segment, normalized);
            if(!Normalizer2Impl.UTF16Plus.equal(segment, normalized)) {
                // replace the input chunk with its normalized form
                text.replace(prev, start, normalized.toString());

                // update all necessary indexes accordingly
                int delta = normalized.length() - (start - prev);
                start += delta;
                limit += delta;
            }
        } while(start < limit);

        offsets.start = start;
        offsets.contextLimit += limit - offsets.limit;
        offsets.limit = limit;
    }
782279
def insert_text(self, text):
        """"""
        # Eventually this maybe should wrap to insert_text_to if
        # backspace-handling is required
        self.textCursor().insertText(text, self.default_style.format)
1109040
def canonicalize_spec(spec, parent_context):
  """"""

  test_specs = {k:v for (k,v) in spec.items() if k.startswith("Test")}
  local_context = {k:v for (k,v) in spec.items() if not k.startswith("Test")}

  context = reduce_contexts(parent_context, local_context)

  if test_specs:
    return {k: canonicalize_spec(v, context) for (k,v) in test_specs.items()}
  else:
    program_chunks = sum([resolve_module(m,context['Definitions']) for m in context['Modules']],[]) + [context['Program']]
    test_spec = {
      'Arguments': context['Arguments'],
      'Program': "\n".join(program_chunks),
      'Expect': context['Expect'],
    }
    return test_spec