Revise wording to match canonical and expected forms.
338948
def _preprocess(inp):
        """"""
        inp = re.sub(r'(\b)a(\b)', r'\g<1>one\g<2>', inp)
        inp = re.sub(r'to the (.*) power', r'to \g<1>', inp)
        inp = re.sub(r'to the (.*?)(\b)', r'to \g<1>\g<2>', inp)
        inp = re.sub(r'log of', r'log', inp)
        inp = re.sub(r'(square )?root( of)?', r'sqrt', inp)
        inp = re.sub(r'squared', r'to two', inp)
        inp = re.sub(r'cubed', r'to three', inp)
        inp = re.sub(r'divided?( by)?', r'divide', inp)
        inp = re.sub(r'(\b)over(\b)', r'\g<1>divide\g<2>', inp)
        inp = re.sub(r'(\b)EE(\b)', r'\g<1>e\g<2>', inp)
        inp = re.sub(r'(\b)E(\b)', r'\g<1>e\g<2>', inp)
        inp = re.sub(r'(\b)pie(\b)', r'\g<1>pi\g<2>', inp)
        inp = re.sub(r'(\b)PI(\b)', r'\g<1>pi\g<2>', inp)

        def findImplicitMultiplications(inp):
            """Replace omitted 'times' references."""

            def findConstantMultiplications(inp):
                split = inp.split(' ')
                revision = ""

                converter = NumberService()
                for i, w in enumerate(split):
                    if i > 0 and w in MathService.__constants__:
                        if converter.isValid(split[i - 1]):
                            revision += " times"
                    if not revision:
                        revision = w
                    else:
                        revision += " " + w

                return revision

            def findUnaryMultiplications(inp):
                split = inp.split(' ')
                revision = ""

                for i, w in enumerate(split):
                    if i > 0 and w in MathService.__unaryOperators__:
                        last_op = split[i - 1]

                        binary = last_op in MathService.__binaryOperators__
                        unary = last_op in MathService.__unaryOperators__

                        if last_op and not (binary or unary):
                            revision += " times"
                    if not revision:
                        revision = w
                    else:
                        revision += " " + w

                return revision

            return findUnaryMultiplications(findConstantMultiplications(inp))

        return findImplicitMultiplications(inp)
662063
private static String canonicalBraceAsNecessary(Type type) {
		String str = type.toCanonicalString();
		if (needsBraces(type)) {
			return "(" + str + ")";
		} else {
			return str;
		}
	}
291658
def raise_infinitive_error(sentence_str):
    """"""
    sent_doc = textacy.Doc(sentence_str, lang='en_core_web_lg')
    inf_pattern = r'<PART|ADP><VERB>' # To aux/auxpass* csubj
    infinitives = textacy.extract.pos_regex_matches(sent_doc, inf_pattern)
    for inf in infinitives:
        if inf[0].text.lower() != 'to':
            continue
        if inf[-1].tag_ != 'VB':
            raise Exception('InfinitivePhraseError')
918565
def correct(self, **kwargs):
        """"""

        if 'insertindex_offset' in kwargs:
            del kwargs['insertindex_offset'] #dealt with in an earlier stage

        if 'confidence' in kwargs and kwargs['confidence'] is None:
            del kwargs['confidence']

        if 'reuse' in kwargs:
            #reuse an existing correction instead of making a new one
            if isinstance(kwargs['reuse'], Correction):
                c = kwargs['reuse']
            else: #assume it's an index
                try:
                    c = self.doc.index[kwargs['reuse']]
                    assert isinstance(c, Correction)
                except:
                    raise ValueError("reuse= must point to an existing correction (id or instance)! Got " + str(kwargs['reuse']))

            suggestionsonly = (not c.hasnew(True) and not c.hasoriginal(True) and c.hassuggestions(True))

            if 'new' in kwargs and c.hascurrent():
                #can't add new if there's current, so first set original to current, and then delete current

                if 'current' in kwargs:
                    raise Exception("Can't set both new= and current= !")
                if 'original' not in kwargs:
                    kwargs['original'] = c.current()

                c.remove(c.current())
        else:
            if 'id' not in kwargs and 'generate_id_in' not in kwargs:
                kwargs['generate_id_in'] = self
            kwargs2 = copy(kwargs)
            for x in ['new','original','suggestion', 'suggestions','current', 'insertindex','nooriginal']:
                if x in kwargs2:
                    del kwargs2[x]
            c = Correction(self.doc, **kwargs2)

        addnew = False
        if 'insertindex' in kwargs:
            insertindex = int(kwargs['insertindex'])
            del kwargs['insertindex']
        else:
            insertindex = -1 #append

        if 'nooriginal' in kwargs and kwargs['nooriginal']:
            nooriginal = True
            del kwargs['nooriginal']
        else:
            nooriginal = False

        if 'current' in kwargs:
            if 'original' in kwargs or 'new' in kwargs: raise Exception("When setting current=, original= and new= can not be set!")
            if not isinstance(kwargs['current'], list) and not isinstance(kwargs['current'], tuple): kwargs['current'] = [kwargs['current']] #support both lists (for multiple elements at once), as well as single element
            c.replace(Current(self.doc, *kwargs['current']))
            for o in kwargs['current']: #delete current from current element
                if o in self and isinstance(o, AbstractElement): #pylint: disable=unsupported-membership-test
                    if insertindex == -1: insertindex = self.data.index(o)
                    self.remove(o)
            del kwargs['current']
        if 'new' in kwargs:
            if not isinstance(kwargs['new'], list) and not isinstance(kwargs['new'], tuple): kwargs['new'] = [kwargs['new']] #support both lists (for multiple elements at once), as well as single element
            addnew = New(self.doc, *kwargs['new']) #pylint: disable=redefined-variable-type
            c.replace(addnew)
            for current in c.select(Current): #delete current if present
                c.remove(current)
            del kwargs['new']
        if 'original' in kwargs and kwargs['original']:
            if not isinstance(kwargs['original'], list) and not isinstance(kwargs['original'], tuple): kwargs['original'] = [kwargs['original']] #support both lists (for multiple elements at once), as well as single element
            c.replace(Original(self.doc, *kwargs['original']))
            for o in kwargs['original']: #delete original from current element
                if o in self and isinstance(o, AbstractElement): #pylint: disable=unsupported-membership-test
                    if insertindex == -1: insertindex = self.data.index(o)
                    self.remove(o)
            for o in kwargs['original']: #make sure IDs are still properly set after removal
                o.addtoindex()
            for current in c.select(Current):  #delete current if present
                c.remove(current)
            del kwargs['original']
        elif addnew and not nooriginal:
            #original not specified, find automagically:
            original = []
            for new in addnew:
                kwargs2 = {}
                if isinstance(new, TextContent):
                    kwargs2['cls'] = new.cls
                try:
                    set = new.set
                except AttributeError:
                    set = None
                #print("DEBUG: Finding replaceables within " + str(repr(self)) + " for ", str(repr(new)), " set " ,set , " args " ,repr(kwargs2),file=sys.stderr)
                replaceables = new.__class__.findreplaceables(self, set, **kwargs2)
                #print("DEBUG: " , len(replaceables) , " found",file=sys.stderr)
                original += replaceables
            if not original:
                #print("DEBUG: ", self.xmlstring(),file=sys.stderr)
                raise Exception("No original= specified and unable to automatically infer on " + str(repr(self)) + " for " + str(repr(new)) + " with set " + set)
            else:
                c.replace( Original(self.doc, *original))
                for current in c.select(Current):  #delete current if present
                    c.remove(current)

        if addnew and not nooriginal:
            for original in c.original():
                if original in self: #pylint: disable=unsupported-membership-test
                    self.remove(original)

        if 'suggestion' in kwargs:
            kwargs['suggestions'] = [kwargs['suggestion']]
            del kwargs['suggestion']
        if 'suggestions' in kwargs:
            for suggestion in kwargs['suggestions']:
                if isinstance(suggestion, Suggestion):
                    c.append(suggestion)
                elif isinstance(suggestion, list) or isinstance(suggestion, tuple):
                    c.append(Suggestion(self.doc, *suggestion))
                else:
                    c.append(Suggestion(self.doc, suggestion))
            del kwargs['suggestions']




        if 'reuse' in kwargs:
            if addnew and suggestionsonly:
                #What was previously only a suggestion, now becomes a real correction
                #If annotator, annotatortypes
                #are associated with the correction as a whole, move it to the suggestions
                #correction-wide annotator, annotatortypes might be overwritten
                for suggestion in c.suggestions():
                    if c.annotator and not suggestion.annotator:
                        suggestion.annotator = c.annotator
                    if c.annotatortype and not suggestion.annotatortype:
                        suggestion.annotatortype = c.annotatortype

            if 'annotator' in kwargs:
                c.annotator = kwargs['annotator'] #pylint: disable=attribute-defined-outside-init
            if 'annotatortype' in kwargs:
                c.annotatortype = kwargs['annotatortype'] #pylint: disable=attribute-defined-outside-init
            if 'confidence' in kwargs:
                c.confidence = float(kwargs['confidence']) #pylint: disable=attribute-defined-outside-init
            c.addtoindex()
            del kwargs['reuse']
        else:
            c.addtoindex()
            if insertindex == -1:
                self.append(c)
            else:
                self.insert(insertindex, c)
        return c
760244
def variations(word):
    """"""

    if len(word) == 1:
        return [[word[0]]]
    elif word == 'aa':
        return [['A']]
    elif word == 'ee':
        return [['i']]
    elif word == 'ei':
        return [['ei']]
    elif word in ['oo', 'ou']:
        return [['u']]
    elif word == 'kha':
        return [['kha'], ['kh', 'a']]
    elif word in ['kh', 'gh', 'ch', 'sh', 'zh', 'ck']:
        return [[word]]
    elif word in ["'ee", "'ei"]:
        return [["'i"]]
    elif word in ["'oo", "'ou"]:
        return [["'u"]]
    elif word in ["a'", "e'", "o'", "i'", "u'", "A'"]:
        return [[word[0] + "'"]]
    elif word in ["'a", "'e", "'o", "'i", "'u", "'A"]:
        return [["'" + word[1]]]
    elif len(word) == 2 and word[0] == word[1]:
        return [[word[0]]]

    if word[:2] == 'aa':
        return [['A'] + i for i in variations(word[2:])]
    elif word[:2] == 'ee':
        return [['i'] + i for i in variations(word[2:])]
    elif word[:2] in ['oo', 'ou']:
        return [['u'] + i for i in variations(word[2:])]
    elif word[:3] == 'kha':
        return \
            [['kha'] + i for i in variations(word[3:])] + \
            [['kh', 'a'] + i for i in variations(word[3:])] + \
            [['k', 'h', 'a'] + i for i in variations(word[3:])]
    elif word[:2] in ['kh', 'gh', 'ch', 'sh', 'zh', 'ck']:
        return \
            [[word[:2]] + i for i in variations(word[2:])] + \
            [[word[0]] + i for i in variations(word[1:])]
    elif word[:2] in ["a'", "e'", "o'", "i'", "u'", "A'"]:
        return [[word[:2]] + i for i in variations(word[2:])]
    elif word[:3] in ["'ee", "'ei"]:
        return [["'i"] + i for i in variations(word[3:])]
    elif word[:3] in ["'oo", "'ou"]:
        return [["'u"] + i for i in variations(word[3:])]
    elif word[:2] in ["'a", "'e", "'o", "'i", "'u", "'A"]:
        return [[word[:2]] + i for i in variations(word[2:])]
    elif len(word) >= 2 and word[0] == word[1]:
        return [[word[0]] + i for i in variations(word[2:])]
    else:
        return [[word[0]] + i for i in variations(word[1:])]
377039
def adjust(self, ln):
        """"""
        adj_ln = ln
        need_unskipped = 0
        for i in self.skips:
            if i <= ln:
                need_unskipped += 1
            elif adj_ln + need_unskipped < i:
                break
            else:
                need_unskipped -= i - adj_ln - 1
                adj_ln = i
        return adj_ln + need_unskipped
775163
function properize(word, options) {
  options = _.extend({inflect: false}, options);
  if (!/\./.test(word)) {
    word = changeCase(word);
    if (options.inflect === false) {
      return word;
    }
    return inflection.singularize(word);
  }
  return word;
}
782991
def normalize_UNTL(subject):
    """"""
    subject = subject.strip()
    subject = re.sub(r'[\s]+', ' ', subject)
    return subject
335147
def process_match(match, fixed_text, cur, cur_end):
    """"""
    # Set our tools
    # -- Initial/default value for replace
    replace = True
    # -- Set check cursor depending on match['type']
    if match['type'] == 'prefix':
        chk = cur - 1
    else:
        # suffix
        chk = cur_end
    # -- Set scope based on whether scope is negative
    if match['scope'].startswith('!'):
        scope = match['scope'][1:]
        negative = True
    else:
        scope = match['scope']
        negative = False

    # Let the matching begin
    # -- Punctuations
    if scope == 'punctuation':
        # Conditions: XORd with negative
        if (not ((chk < 0 and match['type'] == 'prefix') or
                 (chk >= len(fixed_text) and match['type'] == 'suffix') or
                 validate.is_punctuation(fixed_text[chk]))
            ^ negative):
            replace = False
    # -- Vowels -- Checks: 1. Cursor should not be at first character
    # -- if prefix or last character if suffix, 2. Character at chk
    # -- should be a vowel. 3. 'negative' will invert the value of 1
    # -- AND 2
    elif scope == 'vowel':
        if (not (((chk >= 0 and match['type'] == 'prefix') or
                  (chk < len(fixed_text) and match['type'] == 'suffix'))
                 and validate.is_vowel(fixed_text[chk]))
            ^ negative):
            replace =  False
    # -- Consonants -- Checks: 1. Cursor should not be at first
    # -- character if prefix or last character if suffix, 2. Character
    # -- at chk should be a consonant. 3. 'negative' will invert the
    # -- value of 1 AND 2
    elif scope == 'consonant':
        if (not (((chk >= 0 and match['type'] == 'prefix') or
                  (chk < len(fixed_text) and match['type'] == 'suffix'))
                 and validate.is_consonant(fixed_text[chk]))
            ^ negative):
            replace = False
    # -- Exacts
    elif scope == 'exact':
        # Prepare cursor for exact search
        if match['type'] == 'prefix':
            exact_start = cur - len(match['value'])
            exact_end = cur
        else:
            # suffix
            exact_start = cur_end
            exact_end = cur_end + len(match['value'])
        # Validate exact find.
        if not validate.is_exact(match['value'], fixed_text, exact_start,
                                 exact_end, negative):
            replace = False
    # Return replace, which will be true if none of the checks above match
    return replace
52703
def canonicalize(assess_id):
    
    
    hash_regex = re.compile(r'\w{32}')
    
    lines = assess_id.split('\n')
    canon_lines = []
  
    parsing_code = False
    for line in lines:
        line = line.strip()

        if not parsing_code and len(line) > 0:
            for prompt in DICT_PROMPT_TO_CHARACTER:
                if line.startswith(prompt):
                    parsing_code = True
                    comment_character = DICT_PROMPT_TO_CHARACTER[prompt]
                    break

        # If False still in preamble and do not include in canonicalized lines
        if parsing_code:
            # Remove any comments
            comment_index = line.find(comment_character)
            if comment_index >= 0:
                line = line[0:comment_index].strip()

            # If a hashed answer, replace with constant since these vary by semester
            if hash_regex.match(line):
                line = 'LOCKED_ANSWER'

            # Remove any '# locked' text since these are here regardless of language
            if line == '# locked':
                line = ''

            if len(line) > 0:
                canon_lines.append(line)

    return '\n'.join(canon_lines) + '\n'
932389
def pluralize(word, pos=NOUN, gender=MALE, role=SUBJECT, custom={}):
    
    w = word.lower().capitalize()
    if word in custom:
        return custom[word]
    if pos == NOUN:
        for a, b in plural_inflections:
            if w.endswith(a):
                return w[:-len(a)] + b
        # Default rules (baseline = 69%).
        if w.startswith("ge"):
            return w
        if w.endswith("gie"):
            return w
        if w.endswith("e"):
            return w + "n"
        if w.endswith("ien"):
            return w[:-2] + "um"
        if w.endswith(("au", "ein", "eit", "er", "en", "el", "chen", "mus", u"tät", "tik", "tum", "u")):
            return w
        if w.endswith(("ant", "ei", "enz", "ion", "ist", "or", "schaft", "tur", "ung")):
            return w + "en"
        if w.endswith("in"):
            return w + "nen"
        if w.endswith("nis"):
            return w + "se"
        if w.endswith(("eld", "ild", "ind")):
            return w + "er"
        if w.endswith("o"):
            return w + "s"
        if w.endswith("a"):
            return w[:-1] + "en"
        # Inflect common umlaut vowels: Kopf => Köpfe.
        if w.endswith(("all", "and", "ang", "ank", "atz", "auf", "ock", "opf", "uch", "uss")):
            umlaut = w[-3]
            umlaut = umlaut.replace("a", u"ä")
            umlaut = umlaut.replace("o", u"ö")
            umlaut = umlaut.replace("u", u"ü")
            return w[:-3] + umlaut + w[-2:] + "e"
        for a, b in (
          ("ag",  u"äge"), 
          ("ann", u"änner"), 
          ("aum", u"äume"), 
          ("aus", u"äuser"), 
          ("zug", u"züge")):
            if w.endswith(a):
                return w[:-len(a)] + b
        return w + "e"
    return w
711988
def correct(self):
    
    # regex matches: contraction or word or punctuation or whitespace
    tokens = nltk.tokenize.regexp_tokenize(self.raw, "\w*('\w*)+|\w+|[^\w\s]|\s")
    corrected = (Word(w).correct() for w in tokens)
    ret = ''.join(corrected)
    return self.__class__(ret)
384802
def _in(self, *lst):
        """"""
        self.terms.append('in (%s)' % ', '.join(['"%s"' % x for x in lst]))
        return self
1239258
def substitute_infinitives_as_subjects(sent_str):
    """"""
    sent_doc = textacy.Doc(sent_str, lang='en_core_web_lg')
    #inf_pattern = r'<PART><VERB>+' # To aux/auxpass* csubj
    inf_pattern = r'<PART><VERB>' # To aux/auxpass* csubj
    infinitives = textacy.extract.pos_regex_matches(sent_doc, inf_pattern)
    inf_subjs = [] # => [[0,1],...]
    for inf in infinitives:
        if inf[0].text.lower() != 'to':
            continue
        if ('csubj' not in [w.dep_ for w in inf] and sent_doc[inf[-1].i + 1].dep_
                != 'csubj'):
            continue
        if inf[-1].tag_ != 'VB':
            continue
        inf_subj = []
        for v in inf:
            inf_subj.append(v.i)
        inf_subjs.append(inf_subj) 
    new_sent_str = sent_str
    unusual_char = '形'
    for inf_subj in inf_subjs:
        start_inf = sent_doc[inf_subj[0]].idx 
        end_inf = sent_doc[inf_subj[-1]].idx + len(sent_doc[inf_subj[-1]])
        inf_len = end_inf - start_inf
        sub = (unusual_char * inf_len)
        new_sent_str = new_sent_str[:start_inf] + sub + new_sent_str[end_inf:]
    new_sent_str = re.sub('形+', '{}', new_sent_str)
    repl = [conjugate(sent_doc[i_s[-1]].text, tense='presentparticiple') for i_s in inf_subjs]
    return new_sent_str.format(*repl)
684213
def url_fix_common_typos (url):
    """"""
    if url.startswith("http//"):
        url = "http://" + url[6:]
    elif url.startswith("https//"):
        url = "https://" + url[7:]
    return url
854195
def hastext(self, cls='current',strict=True, correctionhandling=CorrectionHandling.CURRENT):
        """"""
        if cls == 'original': correctionhandling = CorrectionHandling.ORIGINAL #backward compatibility
        if correctionhandling in (CorrectionHandling.CURRENT, CorrectionHandling.EITHER):
            for e in self:
                if isinstance(e, New) or isinstance(e, Current):
                    return e.hastext(cls,strict, correctionhandling)
        if correctionhandling in (CorrectionHandling.ORIGINAL, CorrectionHandling.EITHER):
            for e in self:
                if isinstance(e, Original):
                    return e.hastext(cls,strict, correctionhandling)
        return False
780809
def mixedcase(path):
    """"""
    words = path.split('_')
    return words[0] + ''.join(word.title() for word in words[1:])