An agglutination diphthong that ends in /u, y/ usually contains a
    syllable boundary when -C# or -CCV follow, e.g., [lau.ka.us],
    [va.ka.ut.taa].
328634
def apply_T4(word):  # OPTIMIZE
    ''''''
    WORD = _split_consonants_and_vowels(word)

    for k, v in WORD.iteritems():

        if len(v) == 2 and v.endswith(('u', 'y')):

            if WORD.get(k + 2, 0):

                if not WORD.get(k + 3, 0):
                    if len(WORD[k + 2]) == 1 and is_consonant(WORD[k + 2]):
                        WORD[k] = v[0] + '.' + v[1]

                elif len(WORD[k + 1]) == 1 and WORD.get(k + 3, 0):
                    if is_consonant(WORD[k + 3][0]):
                        WORD[k] = v[0] + '.' + v[1]

                elif len(WORD[k + 2]) == 2:
                    WORD[k] = v[0] + '.' + v[1]

    word = _compile_dict_into_word(WORD)

    return word
316829
def _suffix_rules(token, tag="NN"):
    
    if isinstance(token, (list, tuple)):
        token, tag = token
    if token.endswith("ing"):
        tag = "VBG"
    if token.endswith("ly"):
        tag = "RB"
    if token.endswith("s") and not token.endswith(("is", "ous", "ss")):
        tag = "NNS"
    if token.endswith(("able", "al", "ful", "ible", "ient", "ish", "ive", "less", "tic", "ous")) or "-" in token:
        tag = "JJ"
    if token.endswith("ed"):
        tag = "VBN"
    if token.endswith(("ate", "ify", "ise", "ize")):
        tag = "VBP"
    return [token, tag]
825690
def stem(self, word):
        
        word = word.lower()

        r1 = self.__r1_hungarian(word, self.__vowels, self.__digraphs)

        # STEP 1: Remove instrumental case
        if r1.endswith(self.__step1_suffixes):
            for double_cons in self.__double_consonants:
                if word[-2-len(double_cons):-2] == double_cons:
                    word = "".join((word[:-4], word[-3]))

                    if r1[-2-len(double_cons):-2] == double_cons:
                        r1 = "".join((r1[:-4], r1[-3]))
                    break

        # STEP 2: Remove frequent cases
        for suffix in self.__step2_suffixes:
            if word.endswith(suffix):
                if r1.endswith(suffix):
                    word = word[:-len(suffix)]
                    r1 = r1[:-len(suffix)]

                    if r1.endswith("\xE1"):
                        word = "".join((word[:-1], "a"))
                        r1 = "".join((r1[:-1], "a"))

                    elif r1.endswith("\xE9"):
                        word = "".join((word[:-1], "e"))
                        r1 = "".join((r1[:-1], "e"))
                break

        # STEP 3: Remove special cases
        for suffix in self.__step3_suffixes:
            if r1.endswith(suffix):
                if suffix == "\xE9n":
                    word = "".join((word[:-2], "e"))
                    r1 = "".join((r1[:-2], "e"))
                else:
                    word = "".join((word[:-len(suffix)], "a"))
                    r1 = "".join((r1[:-len(suffix)], "a"))
                break

        # STEP 4: Remove other cases
        for suffix in self.__step4_suffixes:
            if r1.endswith(suffix):
                if suffix == "\xE1stul":
                    word = "".join((word[:-5], "a"))
                    r1 = "".join((r1[:-5], "a"))

                elif suffix == "\xE9st\xFCl":
                    word = "".join((word[:-5], "e"))
                    r1 = "".join((r1[:-5], "e"))
                else:
                    word = word[:-len(suffix)]
                    r1 = r1[:-len(suffix)]
                break

        # STEP 5: Remove factive case
        for suffix in self.__step5_suffixes:
            if r1.endswith(suffix):
                for double_cons in self.__double_consonants:
                    if word[-1-len(double_cons):-1] == double_cons:
                        word = "".join((word[:-3], word[-2]))

                        if r1[-1-len(double_cons):-1] == double_cons:
                            r1 = "".join((r1[:-3], r1[-2]))
                        break

        # STEP 6: Remove owned
        for suffix in self.__step6_suffixes:
            if r1.endswith(suffix):
                if suffix in ("\xE1k\xE9", "\xE1\xE9i"):
                    word = "".join((word[:-3], "a"))
                    r1 = "".join((r1[:-3], "a"))

                elif suffix in ("\xE9k\xE9", "\xE9\xE9i",
                                "\xE9\xE9"):
                    word = "".join((word[:-len(suffix)], "e"))
                    r1 = "".join((r1[:-len(suffix)], "e"))
                else:
                    word = word[:-len(suffix)]
                    r1 = r1[:-len(suffix)]
                break

        # STEP 7: Remove singular owner suffixes
        for suffix in self.__step7_suffixes:
            if word.endswith(suffix):
                if r1.endswith(suffix):
                    if suffix in ("\xE1nk", "\xE1juk", "\xE1m",
                                  "\xE1d", "\xE1"):
                        word = "".join((word[:-len(suffix)], "a"))
                        r1 = "".join((r1[:-len(suffix)], "a"))

                    elif suffix in ("\xE9nk", "\xE9j\xFCk",
                                    "\xE9m", "\xE9d", "\xE9"):
                        word = "".join((word[:-len(suffix)], "e"))
                        r1 = "".join((r1[:-len(suffix)], "e"))
                    else:
                        word = word[:-len(suffix)]
                        r1 = r1[:-len(suffix)]
                break

        # STEP 8: Remove plural owner suffixes
        for suffix in self.__step8_suffixes:
            if word.endswith(suffix):
                if r1.endswith(suffix):
                    if suffix in ("\xE1im", "\xE1id", "\xE1i",
                                  "\xE1ink", "\xE1itok", "\xE1ik"):
                        word = "".join((word[:-len(suffix)], "a"))
                        r1 = "".join((r1[:-len(suffix)], "a"))

                    elif suffix in ("\xE9im", "\xE9id", "\xE9i",
                                    "\xE9ink", "\xE9itek", "\xE9ik"):
                        word = "".join((word[:-len(suffix)], "e"))
                        r1 = "".join((r1[:-len(suffix)], "e"))
                    else:
                        word = word[:-len(suffix)]
                        r1 = r1[:-len(suffix)]
                break

        # STEP 9: Remove plural suffixes
        for suffix in self.__step9_suffixes:
            if word.endswith(suffix):
                if r1.endswith(suffix):
                    if suffix == "\xE1k":
                        word = "".join((word[:-2], "a"))
                    elif suffix == "\xE9k":
                        word = "".join((word[:-2], "e"))
                    else:
                        word = word[:-len(suffix)]
                break


        return word
1282918
protected IWord getNextMixedWord(char[] chars, int cjkidx) throws IOException
    {
        IStringBuffer buff = new IStringBuffer();
        buff.clear().append(chars, cjkidx);
        String tstring = buff.toString();
        if ( ! dic.match(ILexicon.MIX_ASSIST_WORD, tstring) ) {
            return null;
        }
        
        /*
         * check and append the behind Latin string 
        */
        if ( behindLatin == null ) {
            behindLatin = nextLatinString(readNext());
        }
        
        IWord wd = null;
        buff.append(behindLatin);
        tstring = buff.toString();
        if ( dic.match(ILexicon.CJK_WORD, tstring) ) {
            wd = dic.get(ILexicon.CJK_WORD, tstring);
        }
        
        if ( (ctrlMask & ISegment.CHECK_EC_MASK) != 0 
                || dic.match(ILexicon.MIX_ASSIST_WORD, tstring) ) {
            ialist.clear();
            int chr = -1, j, mc = 0;
            for ( j = 0; j < dic.mixSuffixLength && (chr = readNext()) != -1; j++ ) {
                buff.append((char)chr);
                ialist.add(chr);
                tstring = buff.toString();
                if ( dic.match(ILexicon.CJK_WORD, tstring) ) {
                    wd = dic.get(ILexicon.CJK_WORD, tstring);
                    mc = j + 1;
                }
            }
            
            //push back the read chars.
            for ( int i = j - 1; i >= mc; i-- ) {
                pushBack(ialist.get(i));
            }
        }
        
        buff.clear();
        buff = null;
        if ( wd != null ) {
            behindLatin = null;
        }
        
        return wd;
    }
1109563
def verb_chain_ends(self):
        """"""
        if not self.is_tagged(VERB_CHAINS):
            self.tag_verb_chains()
        return self.ends(VERB_CHAINS)
844956
def pk_prom(word):
    ''''''
    violations = 0
    stressed = []

    for w in extract_words(word):
        stressed += w.split('.')[2:-1:2]  # odd syllables, excl. word-initial

    # (CVV = light)
    for syll in stressed:
        if phon.is_vowel(syll[-1]):
            violations += 1

    # # (CVV = heavy)
    # for syll in stressed:
    #     if re.search(
    #             ur'^[^ieaouäöy]*[ieaouäöy]{1}$',  syll, flags=re.I | re.U):
    #         violations += 1

    return violations
290933
def T4(word, rules):
    ''''''
    WORD = re.split(
        r'([ieaouäöy]+[^ieaouäöy]+\.*[ieaoäö]{1}(?:u|y)(?:\.*[^ieaouäöy]+|$))',  # noqa
        word, flags=re.I | re.U)

    PARTS = [[] for part in range(len(WORD))]

    for i, v in enumerate(WORD):

        if i != 0:
            vv = u_y_final_diphthongs(v)

            if vv:
                I = vv.start(1) + 1
                PARTS[i].append(v[:I] + '.' + v[I:])

        # include original form (non-application of rule)
        PARTS[i].append(v)

    WORDS = [w for w in product(*PARTS)]

    for WORD in WORDS:
        WORD = ''.join(WORD)
        RULES = rules + ' T4' if word != WORD else rules

        yield WORD, RULES
8581
def flag_dipthongs(self, syllables: List[str]) -> List[int]:
        
        long_positions = []
        for idx, syl in enumerate(syllables):
            for dipthong in self.constants.DIPTHONGS:
                if dipthong in syllables[idx]:
                    if not string_utils.starts_with_qu(syllables[idx]):
                        long_positions.append(idx)
        return long_positions
89323
protected void appendLatinSyn( IWord w )
    {
        IWord ew;
        
        /*
         * @added 2014-07-07
         * w maybe EC_MIX_WORD, so check its syn first
         * and make sure it is not a EC_MIX_WORD then check the EN_WORD 
        */
        if ( w.getSyn() == null ) {
            ew = dic.get(ILexicon.CJK_WORD, w.getValue());
        } else {
            ew = w;
        }
        
        if (  ew != null && ew.getSyn() != null ) {
            ew.setPosition(w.getPosition());
            SegKit.appendSynonyms(wordPool, ew);
        }
    }
926438
def _ending_consonants_only(self, letters: List[str]) -> List[int]:
        """"""
        reversed_letters = list(reversed(letters))
        length = len(letters)
        for idx, letter in enumerate(reversed_letters):
            if not self._contains_vowels(letter) and self._contains_consonants(letter):
                return [(length - idx) - 1]
            if self._contains_vowels(letter):
                return []
            if self._contains_vowels(letter) and self._contains_consonants(letter):
                return []
        return []
1254492
def apply_T1(word):
    ''''''
    # split consonants and vowels: 'balloon' -> ['b', 'a', 'll', 'oo', 'n']
    WORD = [w for w in re.split('([ieAyOauo]+)', word) if w]
    count = 0

    for i, v in enumerate(WORD):

        if i == 0 and is_consonant(v[0]):
            continue

        elif is_consonant(v[0]) and i + 1 != len(WORD):
            if is_cluster(v):  # WSP
                if count % 2 == 0:
                    WORD[i] = v[0] + '.' + v[1:]  # CC > C.C, CCC > C.CC

                else:
                    WORD[i] = '.' + v  # CC > .CC, CCC > .CCC

            # elif is_sonorant(v[0]) and is_cluster(v[1:]):  # NEW
            #     if count % 2 == 0:
            #         WORD[i] = v[0:2] + '.' + v[2:]

            #     else:
            #         WORD[i] = v[0] + '.' + v[1:]

            else:
                WORD[i] = v[:-1] + '.' + v[-1]  # CC > C.C, CCC > CC.C

            count += 1

    WORD = ''.join(WORD)
    RULE = ' T1' if word != WORD else ''

    return WORD, RULE
817873
def T2(word, rules):
    ''''''
    WORD = word
    offset = 0

    for vv in vv_sequences(WORD):
        seq = vv.group(1)

        if not phon.is_diphthong(seq) and not phon.is_long(seq):
            i = vv.start(1) + 1 + offset
            WORD = WORD[:i] + '.' + WORD[i:]
            offset += 1

    rules += ' T2' if word != WORD else ''

    return WORD, rules
1157865
def apply_T4(word):
    ''''''
    T4 = ''
    WORD = word.split('.')

    for i, v in enumerate(WORD):

        # i % 2 != 0 prevents this rule from applying to first, third, etc.
        # syllables, which receive stress (WSP)
        if is_consonant(v[-1]) and i % 2 != 0:

            if i + 1 == len(WORD) or is_consonant(WORD[i + 1][0]):

                if contains_Vu_diphthong(v):
                    I = v.rfind('u')
                    WORD[i] = v[:I] + '.' + v[I:]
                    T4 = ' T4'

                elif contains_Vy_diphthong(v):
                    I = v.rfind('y')
                    WORD[i] = v[:I] + '.' + v[I:]
                    T4 = ' T4'

    word = '.'.join(WORD)

    return word, T4
489590
def same_syllabic_feature(ch1, ch2):
    ''''''
    if ch1 == '.' or ch2 == '.':
        return False

    ch1 = 'V' if ch1 in VOWELS else 'C' if ch1 in CONSONANTS else None
    ch2 = 'V' if ch2 in VOWELS else 'C' if ch2 in CONSONANTS else None

    return ch1 == ch2
388862
def apply_T5(word):
    ''''''
    WORD = _split_consonants_and_vowels(word)

    for k, v in WORD.iteritems():

        if len(v) >= 3 and is_vowel(v[0]):
            vv = [v.find(i) for i in i_DIPHTHONGS if v.find(i) > 0]

            if any(vv):
                vv = vv[0]

                if vv == v[0]:
                    WORD[k] = v[:2] + '.' + v[2:]

                else:
                    WORD[k] = v[:vv] + '.' + v[vv:]

    word = _compile_dict_into_word(WORD)

    return word
667646
def wsp(word):
    ''''''
    violations = 0
    unstressed = []

    for w in extract_words(word):
        unstressed += w.split('.')[1::2]  # even syllables

        # include extrametrical odd syllables as potential WSP violations
        if w.count('.') % 2 == 0:
            unstressed += [w.rsplit('.', 1)[-1], ]

    # SHSP
    for syll in unstressed:
        if re.search(r'[ieaouäöy]{2}[^$ieaouäöy]+', syll, flags=FLAGS):
            violations += 1

    # # WSP (CVV = heavy)
    # for syll in unstressed:
    #     if re.search(
    #             ur'[ieaouäöy]{2}|[ieaouäöy]+[^ieaouäöy]+',
    #             syll, flags=re.I | re.U):
    #         violations += 1

    return violations
1221473
def detect_pattern_format(pattern_filename, encoding, on_word_boundaries):
    '''
    
    '''
    tsv = True
    boundaries = on_word_boundaries
    with open_file(pattern_filename) as input_file:
        for line in input_file:
            line = line.decode(encoding)
            if line.count('\t') != 1:
                tsv = False
            if '\\b' in line:
                boundaries = True
            if boundaries and not tsv:
                break
    return tsv, boundaries