rem_umlat: Remove umlaut from text
432472
def stem_helper(word, rem_umlaut = True):
	""""""
	
	#Define R1 and R2 regions
	
	#R1 is defined as the region after the first consonant followed by a vowel
	
	try:
		R1 = list(re.finditer(r"[aëeiouäöüâêîôûæœ][bdghfcjklmnspqrtvwz]",word))[0].start() + 2
	except:
		R1 = len(word)
		
	#R2 is defined as the region within R1 after the first consonant followed by a vowel
	
	try:
		R2 = list(re.finditer(r"[aëeiouäöüâêîôûæœ][bdghfcjklmnspqrtvwz]",word[R1:]))[0].start() + 2 + R1
	except:
		R2 = len(word)
		
	#Make sure the index of R1 is at least 3. 
	
	if R1<3:
		try:
			R1 = list(re.finditer(r"[aëeiouäöüâêîôûæœ][bdghfcjklmnspqrtvwz]",word[1:]))[0].start() + 2
		except:
			R1 = len(word)
	
	if rem_umlaut:
		word = remove_umlaut(word)
	
	word = word[:R1] + re.sub(r'(wes|wen|est|ern|em|en|er|es|eȥ(?=[klmrt])s|(?=[lr])n|e)$',"",word[R1:])
	word = word[:R1] + re.sub(r'(est|er|en|re|in|iu|(?=.{3})st,word[R1:])$',"",word[R1:])
	word = word[:R2] + re.sub(r'(lich?.?.|keit|inc|isch?.?.)$',"",word[R2:])
	
	return word
270497
def _run_strip_accents(self, text):
        """"""
        text = unicodedata.normalize("NFD", text)
        output = []
        for char in text:
            cat = unicodedata.category(char)
            if cat == "Mn":
                continue
            output.append(char)
        return "".join(output)
972125
def clean_asciidoc(text):
    
    """
    text = re.sub(r'(\b|^)[\[_*]{1,2}([a-zA-Z0-9])', r'"\2', text)
    text = re.sub(r'([a-zA-Z0-9])[\]_*]{1,2}', r'\1"', text)
    return text
205101
def _delete_unwanted_caracters(self, chain):
        
        try:
            chain = chain.decode('utf8', 'replace')
        except UnicodeEncodeError:
            # If it is still encoded correctly, ignore...
            pass
        except AttributeError:
            # Python 3 will raise an exception because the line is still unicode
            pass
        for char in self.illegal_macro_output_chars:
            chain = chain.replace(char, '')
        return chain
1019700
def censor(input_text):
    
    ret = input_text
    words = get_words()
    for word in words:
        curse_word = re.compile(re.escape(word), re.IGNORECASE)
        cen = "".join(get_censor_char() for i in list(word))
        ret = curse_word.sub(cen, ret)
    return ret
716018
def remove_invalid_char(text):
    
    invalid_char_list = ['/', '\\', ':', '*', '?', '"', '<', '>', '|', '\n']
    res = ''
    for char in text:
        if char not in invalid_char_list:
            res += char
    return res
1114855
def unescape(text):
    
    def fixup(m):
        text = m.group(0)
        if text[:2] == "&#":
            # character reference
            try:
                if text[:3] == "&#x":
                    return unichr(int(text[3:-1], 16))
                else:
                    return unichr(int(text[2:-1]))
            except ValueError:
                pass
        else:
            # named entity
            try:
                text = unichr(htmlentitydefs.name2codepoint[text[1:-1]])
            except KeyError:
                pass
        return text # leave as is

    return re.sub("&#?\w+;", fixup, text)
274468
def unapostrophe(text):
    """"""
    text = re.sub(r'[%s]s?$' % ''.join(APOSTROPHES), '', text)
    return text
538885
def fix_text(s, ignore_sym_pat=False):

    if not ignore_sym_pat:
        # Fix descriptions like D7TDB1 (
        s = re.sub("([A-Z0-9]){6} \(", "", s)
        s = s.split(";")[0]

    # Fix parantheses containing names
    s = s.translate(None, "[]")
    s = s.replace("(-)", "[-]")
    s = s.replace("(+)", "[+]")
    s = s.replace("(Uncharacterized protein)", "")
    if not ignore_sym_pat:
        s = s.translate(None, "()")

    # fix minor typos, seen in `autonaming` output
    # change 'protei ' to 'protein '
    # change 'hypthetical' to 'hypothetical'
    # fix string starting with 'ytochrome'
    if 'protei ' in s: s = s.replace('protei ', 'protein ')
    if 'hypthetical' in s: s = s.replace('hypthetical', 'hypothetical')
    if s.startswith('ytochrome'): s = s.replace('ytochrome', 'cytochrome')

    # before trimming off at the first ";", check if name has glycosidic
    # linkage information (e.g 1,3 or 1,4). If so, also check if multiple
    # linkages are separated by ";". If so, replace ";" by "-"
    m = re.findall(glycosidic_link_pat, s)
    if m and ";" in s:
        s = re.sub(";\s*", "-", s)

    # remove underscore from description
    s = re.sub("_", " ", s)

    # Cellular locations
    # Any word that matches e.g. AT5G54690
    # Any word that matches e.g. Os02g0234800
    # (fragment)
    # UPF
    # Remove 'DDB_G\d+' ID
    # '_At[0-9]+g[0-9]+' to ''
    for pat in (loc_pat, osg_pat, frag_pat, upf_pat, ddb_pat):
        # below is a hack since word boundaries don't work on /
        s = s.strip() + " "
        s = re.sub(pat, "", s)

    # &apos;? => '
    s = re.sub(apos_pat, "'", s)
    # &gt => none
    s = re.sub(gt_pat, "", s)
    # reduce runs such as -- '''
    s = re.sub(r"[-]+", "-", s)
    s = re.sub(r"[']+", "'", s)

    s = s.strip()

    # -like to -like protein
    s = re.sub(like_pat, "-like protein", s)

    # 'repeat$' to 'repeat protein'
    if re.search(repeat_pat, s):
        s += "-containing protein"

    # 'binding$' to 'binding protein'
    if re.search(binding_pat, s):
        s += " protein"
        if re.match(Protein_pat, s):
            s = re.sub(Protein_pat, "", s)

    # 'domain$' to 'domain-containing protein'
    if re.search(domain_pat, s):
        s += "-containing protein"
        if re.search(r"-domain", s):
            s = re.sub(r"-domain", " domain", s)
        if re.match(Protein_pat, s):
            s = re.sub(Protein_pat, "", s)

    # 'related$' to '-like protein'
    if re.search(related_pat, s):
        s = re.sub(related_pat, "-like protein", s)
        if re.match(Protein_pat, s) and not re.match(r"Protein kinase", s):
            s = re.sub(Protein_pat, "", s)

    # '[0-9]+ homolog' to '-like protein'
    if re.search(homolog_pat1, s):
        s = re.sub(homolog_pat1, "-like protein", s)
        if re.match(Protein_pat, s):
            s = re.sub(Protein_pat, "", s)

    # 'Protein\s+(.*)\s+homolog' to '$1-like protein'
    match = re.search(homolog_pat2, s)
    if match and not re.match(r"Protein kinase", s):
        ret = match.group(1)
        s = re.sub(homolog_pat2, ret + "-like protein", s)
        s = re.sub(r"^\s+", "", s)
        s = s.capitalize()

    # 'homolog protein' to '-like protein'
    # 'homologue$' to '-like protein'
    # 'homolog$' to '-like protein'
    for pat in (homolog_pat3, homolog_pat5, homolog_pat6):
        if re.search(pat, s):
            s = re.sub(pat, "-like protein", s)

    # 'Agenet domain-containing protein / bromo-adjacent homology (BAH) domain-containing protein'
    # to 'Agenet and bromo-adjacent homology (BAH) domain-containing protein'
    if re.search(agenet_pat, s):
        s = re.sub(agenet_pat, "Agenet and ", s)

    # plural to singular
    if re.search(plural_pat, s):
        if (s.find('biogenesis') == -1 and s.find('Topors') == -1) or (not re.search(with_and_pat, s)):
            s = re.sub(r"s$", "", s)

    # 'like_TBP' or 'likeTBP' to 'like TBP'
    if re.search(tbp_pat, s):
        s = re.sub(tbp_pat, "like TBP", s)

    # 'protein protein' to 'protein'
    if re.search(prot_pat, s):
        s = re.sub(prot_pat, "protein", s)

    # 'dimerisation' to 'dimerization'
    if re.search(dimer_pat, s):
        s = re.sub(dimer_pat, "dimerization", s)

    # Any AHRD that matches e.g. "AT5G54690-like protein"
    # Any AHRD that contains the words '^Belongs|^Encoded|^Expression|^highly'
    for pat in (atg_pat, athila_pat1):
        if re.search(pat, s):
            s = Unknown

    # remove 'arabidopsis[ thaliana]' and/or embedded Atg IDs
    for pat in (atg_id_pat, athila_pat2, athila_pat3, athila_pat4):
        # below is a hack since word boundaries don't work on /
        s = s.strip() + " "
        s = re.sub(pat, "", s)

    # remove "\s+LENGTH=\d+" from TAIR deflines
    if re.search(length_pat, s):
        s = re.sub(length_pat, "", s)

    # if name has a dot followed by a space (". ") in it and contains multiple
    # parts separated by a comma, strip name starting from first occurrence of ","
    if re.search(r"\. ", s):
        if re.search(r",", s):
            s = s.split(",")[0]

    # if name contains any of the disallowed words,
    # remove word occurrence from name
    # if name contains references to any other organism, trim name upto
    # that occurrence
    for pat in (disallow_pat, organism_pat):
        if re.search(pat, s):
            s = re.sub(pat, "", s)

    s = s.strip()

    if not ignore_sym_pat:
        # 'homolog \d+' to '-like protein'
        if re.search(homolog_pat4, s):
            s = re.sub(homolog_pat4, "", s)

        # Trailing protein numeric copy (e.g. Myb 1)
        if re.search(trail_pat, s):
            s = re.sub(trail_pat, "", s)

        # if name is entirely a gene symbol-like (all capital letters, maybe followed by numbers)
        # add a "-like protein" at the end
        if (re.search(sym_pat, s) or re.search(lc_sym_pat, s)) \
                and not re.search(spada_pat, s):
            s = s + "-like protein"

        # if gene symbol in parantheses at EOL, remove symbol
        if re.search(eol_sym_pat, s):
            s = re.sub(eol_sym_pat, "", s)

        # if name terminates at a symbol([^A-Za-z0-9_]), trim it off
        if re.search(r"\W{1,}$", s) and not re.search(r"\)$", s):
            s = re.sub("\W{1,}$", "", s)

        if "uncharacterized" in s:
            s = "uncharacterized protein"

    # change sulfer to sulfur
    if re.search(sulfer_pat, s):
        s = re.sub(sulfer_pat, "sulfur", s)

    # change sulph to sulf
    if re.search(sulph_pat, s):
        s = re.sub(sulph_pat, "sulf", s)

    # change monoxy to monooxy
    if re.search(monoxy_pat, s):
        s = re.sub(monoxy_pat, "monooxy", s)

    # change proteine to protein
    if re.search(proteine_pat, s):
        s = re.sub(proteine_pat, "protein", s)

    # change signalling to signaling
    if re.search(signalling_pat, s):
        s = re.sub(signalling_pat, "signaling", s)

    # change aluminium to aluminum
    if re.search(aluminium_pat, s):
        s = re.sub(aluminium_pat, "aluminum", s)

    # change haem to heme
    if re.search(haem_pat, s):
        s = re.sub(haem_pat, "heme", s)

    # chage haemo to hemo
    if re.search(haemo_pat, s):
        s = re.sub(haemo_pat, "hemo", s)

    # change assessory to accessory
    if re.search(assessory_pat, s):
        s = re.sub(assessory_pat, "accessory", s)

    # change -ise/-ised/-isation to -ize/-ized/-ization
    match = re.search(ise_pat, s)
    if match:
        ret = match.group(1)
        if match.group(2):
            suff = match.group(2)
            s = re.sub(ise_pat, "{0}ize{1}".format(ret, suff), s)
        else:
            s = re.sub(ise_pat, "{0}ize".format(ret), s)

    match = re.search(isation_pat, s)
    if match:
        ret = match.group(1)
        s = re.sub(isation_pat, "{0}ization".format(ret), s)

    # change -bre to -ber
    match = re.search(bre_pat, s)
    if match:
        ret = match.group(1)
        s = re.sub(bre_pat, "{0}ber".format(ret), s)

    if not s.startswith(Hypothetical):
        # 'Candidate|Hypothetical|Novel|Predicted|Possible|Probable|Uncharacterized' to 'Putative'
        if s.startswith('Uncharacterized') and any(pat in s for pat in ('UCP', 'UPF', 'protein')):
            pass
        else:
            if re.search(put_pat, s):
                s = re.sub(put_pat, "Putative", s)

    
    sl = s.lower()

    # Any mention of `clone` or `contig` is not informative
    if "clone" in sl or "contig" in sl:
        s = Unknown

    # All that's left is `protein` is not informative
    if sl in ("protein", "protein, putative", ""):
        s = Unknown

    if Unknown.lower() in sl:
        s = Unknown

    if "FUNCTIONS IN".lower() in sl and "unknown" in sl:
        s = Unknown

    if "LOCATED IN".lower() in sl:
        s = Unknown

    s = re.sub(r"[,]*\s+putative$", "", s)

    if s == Unknown or s.strip() == "protein":
        s = Hypothetical

    # Compact all spaces
    s = ' '.join(s.split())

    assert s.strip()

    return s
275482
def _utf_strip_bom(self, encoding):
        """"""

        if encoding is None:
            pass
        elif encoding.lower() == 'utf-8':
            encoding = 'utf-8-sig'
        elif encoding.lower().startswith('utf-16'):
            encoding = 'utf-16'
        elif encoding.lower().startswith('utf-32'):
            encoding = 'utf-32'
        return encoding
1332718
def slugify_unicode(s):
    
    chars = []
    for char in str(smart_text(s)):
        cat = unicodedata.category(char)[0]
        if cat in "LN" or char in "-_~":
            chars.append(char)
        elif cat == "Z":
            chars.append(" ")
    return re.sub("[-\s]+", "-", "".join(chars).strip()).lower()
657747
def convert_text( text)
      text = text.to_s
      return text if text.empty?
      raise "Text kein String >#{text}< (#{text.class})" unless text.kind_of?( String ) # not possible
      text.gsub!('Ä', 'AE')
      text.gsub!('Ü', 'UE')
      text.gsub!('Ö', 'OE')
      text.gsub!('ä', 'AE')
      text.gsub!('ü', 'UE')
      text.gsub!('ö', 'OE')
      text.gsub!('ß', 'SS')

      text = I18n.transliterate(text)

      text.gsub! /[^a-zA-Z0-9\ \.\,\&\-\/\+\*\$\%]/, '' # Remove all invalid chars
      text.upcase.strip
    end
1298495
def normalize_value(text):
    
    result = text.replace('\n', ' ')
    result = re.subn('[ ]{2,}', ' ', result)[0]
    return result
972291
def deramise(r):
    
    return r.replace('J', 'I') \
            .replace('j', 'i') \
            .replace('v', 'u') \
            .replace("æ", "ae") \
            .replace("Æ", "Ae") \
            .replace("œ", "oe") \
            .replace("Œ", "Oe") \
            .replace("ụ", 'u') \
            .replace('V', 'U')
742614
def normalize_text(self, txt):
        ""
        # - for TTF unicode fonts: unicode object (utf8 encoding)
        # - for built-in fonts: string instances (latin 1 encoding)
        if self.unifontsubset and isinstance(txt, str) and not PY3K:
            txt = txt.decode('utf8')
        elif not self.unifontsubset and isinstance(txt, unicode) and not PY3K:
            txt = txt.encode('latin1')
        return txt
193166
function removeAnchors(text) {
        if(text.charAt(0) == '\x02')
            text = text.substr(1);
        if(text.charAt(text.length - 1) == '\x03')
            text = text.substr(0, text.length - 1);
        return text;
    }
195811
def strip_accents(text):
    

    normalized_str = unicodedata.normalize('NFD', text)

    return ''.join([
        c for c in normalized_str if unicodedata.category(c) != 'Mn'])