Stem each word of the Latin text.
199810
def stem(self, text):
        """"""

        stemmed_text = ''

        for word in text.split(' '):
            if word not in self.stops:

                # remove '-que' suffix
                word, in_que_pass_list = self._checkremove_que(word)
                if not in_que_pass_list:

                    # remove the simple endings from the target word
                    word, was_stemmed = self._matchremove_simple_endings(word)

                    # if word didn't match the simple endings, try verb endings
                    if not was_stemmed:
                        word = self._matchremove_verb_endings(word)

            # add the stemmed word to the text
            stemmed_text += word + ' '


        return stemmed_text
767580
def stem(self, word):
        
        # set initial values
        self.is_verb = True
        self.is_noun = True
        self.is_defined = False

        self.suffix_verb_step2a_success = False
        self.suffix_verb_step2b_success = False
        self.suffix_noun_step2c2_success = False
        self.suffix_noun_step1a_success = False
        self.suffix_noun_step2a_success = False
        self.suffix_noun_step2b_success = False
        self.suffixe_noun_step1b_success = False
        self.prefix_step2a_success = False
        self.prefix_step3a_noun_success = False
        self.prefix_step3b_noun_success = False

        modified_word = word
        # guess type and properties
        # checks1
        self.__checks_1(modified_word)
        # checks2
        self.__checks_2(modified_word)
        # Pre_Normalization
        modified_word = self.__normalize_pre(modified_word)
        # Start stemming
        if self.is_verb:
            modified_word = self.__Suffix_Verb_Step1(modified_word)
            if self.suffixes_verb_step1_success:
                modified_word = self.__Suffix_Verb_Step2a(modified_word)
                if not self.suffix_verb_step2a_success:
                    modified_word = self.__Suffix_Verb_Step2c(modified_word)
                # or next TODO: How to deal with or next instruction
            else:
                modified_word = self.__Suffix_Verb_Step2b(modified_word)
                if not self.suffix_verb_step2b_success:
                    modified_word = self.__Suffix_Verb_Step2a(modified_word)
        if self.is_noun:
            modified_word = self.__Suffix_Noun_Step2c2(modified_word)
            if not self.suffix_noun_step2c2_success:
                if not self.is_defined:
                    modified_word = self.__Suffix_Noun_Step1a(modified_word)
                    # if self.suffix_noun_step1a_success:
                    modified_word = self.__Suffix_Noun_Step2a(modified_word)
                    if not self.suffix_noun_step2a_success:
                        modified_word = self.__Suffix_Noun_Step2b(modified_word)
                    if (
                        not self.suffix_noun_step2b_success
                        and not self.suffix_noun_step2a_success
                    ):
                        modified_word = self.__Suffix_Noun_Step2c1(modified_word)
                    # or next ? todo : how to deal with or next
                else:
                    modified_word = self.__Suffix_Noun_Step1b(modified_word)
                    if self.suffixe_noun_step1b_success:
                        modified_word = self.__Suffix_Noun_Step2a(modified_word)
                        if not self.suffix_noun_step2a_success:
                            modified_word = self.__Suffix_Noun_Step2b(modified_word)
                        if (
                            not self.suffix_noun_step2b_success
                            and not self.suffix_noun_step2a_success
                        ):
                            modified_word = self.__Suffix_Noun_Step2c1(modified_word)
                    else:
                        if not self.is_defined:
                            modified_word = self.__Suffix_Noun_Step2a(modified_word)
                        modified_word = self.__Suffix_Noun_Step2b(modified_word)
            modified_word = self.__Suffix_Noun_Step3(modified_word)
        if not self.is_noun and self.is_verb:
            modified_word = self.__Suffix_All_alef_maqsura(modified_word)

        # prefixes
        modified_word = self.__Prefix_Step1(modified_word)
        modified_word = self.__Prefix_Step2a(modified_word)
        if not self.prefix_step2a_success:
            modified_word = self.__Prefix_Step2b(modified_word)
        modified_word = self.__Prefix_Step3a_Noun(modified_word)
        if not self.prefix_step3a_noun_success and self.is_noun:
            modified_word = self.__Prefix_Step3b_Noun(modified_word)
        else:
            if not self.prefix_step3b_noun_success and self.is_verb:
                modified_word = self.__Prefix_Step3_Verb(modified_word)
                modified_word = self.__Prefix_Step4_Verb(modified_word)

        # post normalization stemming
        modified_word = self.__normalize_post(modified_word)
        stemmed_word = modified_word
        return stemmed_word
1214425
public function stem($text)
    {
        $normalizedText = Filter\TextNormalizer::normalizeText($text);

        $words = explode(' ', $normalizedText);
        $stems = array();

        foreach ($words as $word) {
            $stems[] = $this->stemWord($word);
        }

        return implode(' ', $stems);
    }
538327
def tokenize_documents(docs):
    """"""
    sep = '\n\n'
    texts = []
    for doc in docs:
        text = '\n\n'.join(['\n'.join(para[SENTENCES]) for para in doc[PARAGRAPHS]])
        doc[TEXT] = text
        del doc[PARAGRAPHS]
        texts.append(Text(doc))
    return texts
340567
function lex(text) {
  let root = new StyleNode();
  root['start'] = 0;
  root['end'] = text.length
  let n = root;
  for (let i = 0, l = text.length; i < l; i++) {
    if (text[i] === OPEN_BRACE) {
      if (!n['rules']) {
        n['rules'] = [];
      }
      let p = n;
      let previous = p['rules'][p['rules'].length - 1] || null;
      n = new StyleNode();
      n['start'] = i + 1;
      n['parent'] = p;
      n['previous'] = previous;
      p['rules'].push(n);
    } else if (text[i] === CLOSE_BRACE) {
      n['end'] = i + 1;
      n = n['parent'] || root;
    }
  }
  return root;
}
418737
def wordify(text):
    
    stopset = set(nltk.corpus.stopwords.words('english'))
    tokens = nltk.WordPunctTokenizer().tokenize(text)
    return [w for w in tokens if w not in stopset]
656045
def forward_word_extend_selection(self, e): # 
        """
        self.l_buffer.forward_word_extend_selection(self.argument_reset)
        self.finalize()
1059706
def stem(self, word):
        
        word = word.lower()

        step3_success = False

        r1, r2 = self._r1r2_standard(word, self.__vowels)

        # STEP 1: Particles etc.
        for suffix in self.__step1_suffixes:
            if r1.endswith(suffix):
                if suffix == "sti":
                    if suffix in r2:
                        word = word[:-3]
                        r1 = r1[:-3]
                        r2 = r2[:-3]
                else:
                    if word[-len(suffix)-1] in "ntaeiouy\xE4\xF6":
                        word = word[:-len(suffix)]
                        r1 = r1[:-len(suffix)]
                        r2 = r2[:-len(suffix)]
                break

        # STEP 2: Possessives
        for suffix in self.__step2_suffixes:
            if r1.endswith(suffix):
                if suffix == "si":
                    if word[-3] != "k":
                        word = word[:-2]
                        r1 = r1[:-2]
                        r2 = r2[:-2]

                elif suffix == "ni":
                    word = word[:-2]
                    r1 = r1[:-2]
                    r2 = r2[:-2]
                    if word.endswith("kse"):
                        word = "".join((word[:-3], "ksi"))

                    if r1.endswith("kse"):
                        r1 = "".join((r1[:-3], "ksi"))

                    if r2.endswith("kse"):
                        r2 = "".join((r2[:-3], "ksi"))

                elif suffix == "an":
                    if (word[-4:-2] in ("ta", "na") or
                        word[-5:-2] in ("ssa", "sta", "lla", "lta")):
                        word = word[:-2]
                        r1 = r1[:-2]
                        r2 = r2[:-2]

                elif suffix == "\xE4n":
                    if (word[-4:-2] in ("t\xE4", "n\xE4") or
                        word[-5:-2] in ("ss\xE4", "st\xE4",
                                        "ll\xE4", "lt\xE4")):
                        word = word[:-2]
                        r1 = r1[:-2]
                        r2 = r2[:-2]

                elif suffix == "en":
                    if word[-5:-2] in ("lle", "ine"):
                        word = word[:-2]
                        r1 = r1[:-2]
                        r2 = r2[:-2]
                else:
                    word = word[:-3]
                    r1 = r1[:-3]
                    r2 = r2[:-3]
                break

        # STEP 3: Cases
        for suffix in self.__step3_suffixes:
            if r1.endswith(suffix):
                if suffix in ("han", "hen", "hin", "hon", "h\xE4n",
                              "h\xF6n"):
                    if ((suffix == "han" and word[-4] == "a") or
                        (suffix == "hen" and word[-4] == "e") or
                        (suffix == "hin" and word[-4] == "i") or
                        (suffix == "hon" and word[-4] == "o") or
                        (suffix == "h\xE4n" and word[-4] == "\xE4") or
                        (suffix == "h\xF6n" and word[-4] == "\xF6")):
                        word = word[:-3]
                        r1 = r1[:-3]
                        r2 = r2[:-3]
                        step3_success = True

                elif suffix in ("siin", "den", "tten"):
                    if (word[-len(suffix)-1] == "i" and
                        word[-len(suffix)-2] in self.__restricted_vowels):
                        word = word[:-len(suffix)]
                        r1 = r1[:-len(suffix)]
                        r2 = r2[:-len(suffix)]
                        step3_success = True
                    else:
                        continue

                elif suffix == "seen":
                    if word[-6:-4] in self.__long_vowels:
                        word = word[:-4]
                        r1 = r1[:-4]
                        r2 = r2[:-4]
                        step3_success = True
                    else:
                        continue

                elif suffix in ("a", "\xE4"):
                    if word[-2] in self.__vowels and word[-3] in self.__consonants:
                        word = word[:-1]
                        r1 = r1[:-1]
                        r2 = r2[:-1]
                        step3_success = True

                elif suffix in ("tta", "tt\xE4"):
                    if word[-4] == "e":
                        word = word[:-3]
                        r1 = r1[:-3]
                        r2 = r2[:-3]
                        step3_success = True

                elif suffix == "n":
                    word = word[:-1]
                    r1 = r1[:-1]
                    r2 = r2[:-1]
                    step3_success = True

                    if word[-2:] == "ie" or word[-2:] in self.__long_vowels:
                        word = word[:-1]
                        r1 = r1[:-1]
                        r2 = r2[:-1]
                else:
                    word = word[:-len(suffix)]
                    r1 = r1[:-len(suffix)]
                    r2 = r2[:-len(suffix)]
                    step3_success = True
                break

        # STEP 4: Other endings
        for suffix in self.__step4_suffixes:
            if r2.endswith(suffix):
                if suffix in ("mpi", "mpa", "mp\xE4", "mmi", "mma",
                              "mm\xE4"):
                    if word[-5:-3] != "po":
                        word = word[:-3]
                        r1 = r1[:-3]
                        r2 = r2[:-3]
                else:
                    word = word[:-len(suffix)]
                    r1 = r1[:-len(suffix)]
                    r2 = r2[:-len(suffix)]
                break

        # STEP 5: Plurals
        if step3_success and len(r1) >= 1 and r1[-1] in "ij":
            word = word[:-1]
            r1 = r1[:-1]

        elif (not step3_success and len(r1) >= 2 and
              r1[-1] == "t" and r1[-2] in self.__vowels):
            word = word[:-1]
            r1 = r1[:-1]
            r2 = r2[:-1]
            if r2.endswith("imma"):
                word = word[:-4]
                r1 = r1[:-4]
            elif r2.endswith("mma") and r2[-5:-3] != "po":
                word = word[:-3]
                r1 = r1[:-3]

        # STEP 6: Tidying up
        if r1[-2:] in self.__long_vowels:
            word = word[:-1]
            r1 = r1[:-1]

        if (len(r1) >= 2 and r1[-2] in self.__consonants and
            r1[-1] in "a\xE4ei"):
            word = word[:-1]
            r1 = r1[:-1]

        if r1.endswith(("oj", "uj")):
            word = word[:-1]
            r1 = r1[:-1]

        if r1.endswith("jo"):
            word = word[:-1]
            r1 = r1[:-1]

        # If the word ends with a double consonant
        # followed by zero or more vowels, the last consonant is removed.
        for i in range(1, len(word)):
            if word[-i] in self.__vowels:
                continue
            else:
                if i == 1:
                    if word[-i-1:] in self.__double_consonants:
                        word = word[:-1]
                else:
                    if word[-i-1:-i+1] in self.__double_consonants:
                        word = "".join((word[:-i], word[-i+1:]))
                break


        return word
501607
def _tokenize(sentence):
    ''''''

    _tokens = nltk.word_tokenize(sentence)
    tokens = [stemmer.stem(tk) for tk in _tokens]
    return tokens
1091562
def gen_docs(corpus, lemmatize, rm_stops):
    

    assert corpus in ['phi5', 'tlg']

    if corpus == 'phi5':
        language = 'latin'
        filepaths = assemble_phi5_author_filepaths()
        jv_replacer = JVReplacer()
        text_cleaner = phi5_plaintext_cleanup
        word_tokenizer = nltk_tokenize_words
        if rm_stops:
            stops = latin_stops
        else:
            stops = None
    elif corpus == 'tlg':
        language = 'greek'
        filepaths = assemble_tlg_author_filepaths()
        text_cleaner = tlg_plaintext_cleanup
        word_tokenizer = nltk_tokenize_words

        if rm_stops:
            stops = latin_stops
        else:
            stops = None

    if lemmatize:
        lemmatizer = LemmaReplacer(language)

    sent_tokenizer = TokenizeSentence(language)

    for filepath in filepaths:
        with open(filepath) as f:
            text = f.read()
        # light first-pass cleanup, before sentence tokenization (which relies on punctuation)
        text = text_cleaner(text, rm_punctuation=False, rm_periods=False)
        sent_tokens = sent_tokenizer.tokenize_sentences(text)
        # doc_sentences = []
        for sentence in sent_tokens:
            # a second cleanup at sentence-level, to rm all punctuation
            sentence = text_cleaner(sentence, rm_punctuation=True, rm_periods=True)
            sentence = word_tokenizer(sentence)
            sentence = [s.lower() for s in sentence]
            sentence = [w for w in sentence if w]
            if language == 'latin':
                sentence = [w[1:] if w.startswith('-') else w for w in sentence]

            if stops:
                sentence = [w for w in sentence if w not in stops]

            sentence = [w for w in sentence if len(w) > 1]  # rm short words

            if sentence:
                sentence = sentence

            if lemmatize:
                sentence = lemmatizer.lemmatize(sentence)
            if sentence and language == 'latin':
                sentence = [jv_replacer.replace(word) for word in sentence]
            if sentence:
                yield sentence
70198
function getKeyphrases(results, maximum) {
  var stemmedPhrases = {}
  var initialWords = []
  var stemmedPhrase
  var index
  var length
  var otherIndex
  var keyword
  var matches
  var phrase
  var stems
  var score
  var first
  var match

  // Iterate over all grouped important words...
  for (keyword in results) {
    matches = results[keyword].matches
    length = matches.length
    index = -1

    // Iterate over every occurence of a certain keyword...
    while (++index < length) {
      phrase = findPhrase(matches[index])
      stemmedPhrase = stemmedPhrases[phrase.value]
      first = phrase.nodes[0]

      match = {
        nodes: phrase.nodes,
        parent: matches[index].parent
      }

      // If we've detected the same stemmed phrase somewhere.
      if (own.call(stemmedPhrases, phrase.value)) {
        // Add weight per phrase to the score of the phrase.
        stemmedPhrase.score += stemmedPhrase.weight

        // If this is the first time we walk over the phrase (exact match but
        // containing another important word), add it to the list of matching
        // phrases.
        if (initialWords.indexOf(first) === -1) {
          initialWords.push(first)
          stemmedPhrase.matches.push(match)
        }
      } else {
        otherIndex = -1
        score = -1
        stems = phrase.stems

        initialWords.push(first)

        // For every stem in phrase, add its score to score.
        while (stems[++otherIndex]) {
          score += results[stems[otherIndex]].score
        }

        stemmedPhrases[phrase.value] = {
          score: score,
          weight: score,
          stems: stems,
          value: phrase.value,
          matches: [match]
        }
      }
    }
  }

  for (stemmedPhrase in stemmedPhrases) {
    phrase = stemmedPhrases[stemmedPhrase]

    // Modify its score to be the rounded result of multiplying it with the
    // number of occurances, and dividing it by the ammount of words in the
    // phrase.
    phrase.score = Math.round(
      (phrase.score * phrase.matches.length) / phrase.stems.length
    )
  }

  return filterResults(stemmedPhrases, maximum)
}
441680
def tokenize(self):

        

        self.tokens = []
        self.terms = OrderedDict()

        # Generate tokens.
        for token in utils.tokenize(self.text):

            # Ignore stopwords.
            if token['unstemmed'] in self.stopwords:
                self.tokens.append(None)

            else:

                # Token:
                self.tokens.append(token)

                # Term:
                offsets = self.terms.setdefault(token['stemmed'], [])
                offsets.append(token['offset'])
916127
def get_stem(self, noun, gender, mimation=True):
        """"""
        stem = ''
        if mimation and noun[-1:] == 'm':
            # noun = noun[:-1]
            pass
        # Take off ending
        if gender == 'm':
            if noun[-2:] in list(self.endings['m']['singular'].values()) + \
                    list(self.endings['m']['dual'].values()):
                stem = noun[:-2]
            elif noun[-1] in list(self.endings['m']['plural'].values()):
                stem = noun[:-1]
            else:
                print("Unknown masculine noun: {}".format(noun))
        elif gender == 'f':
            if noun[-4:] in self.endings['f']['plural']['nominative'] + \
                    self.endings['f']['plural']['oblique']:
                stem = noun[:-4] + 't'
            elif noun[-3:] in list(self.endings['f']['singular'].values()) + \
                    list(self.endings['f']['dual'].values()):
                stem = noun[:-3] + 't'
            elif noun[-2:] in list(self.endings['m']['singular'].values()) + \
                    list(self.endings['m']['dual'].values()):
                stem = noun[:-2]
            else:
                print("Unknown feminine noun: {}".format(noun))
        else:
            print("Unknown noun: {}".format(noun))
        return stem
432472
def stem_helper(word, rem_umlaut = True):
	""""""
	
	#Define R1 and R2 regions
	
	#R1 is defined as the region after the first consonant followed by a vowel
	
	try:
		R1 = list(re.finditer(r"[aëeiouäöüâêîôûæœ][bdghfcjklmnspqrtvwz]",word))[0].start() + 2
	except:
		R1 = len(word)
		
	#R2 is defined as the region within R1 after the first consonant followed by a vowel
	
	try:
		R2 = list(re.finditer(r"[aëeiouäöüâêîôûæœ][bdghfcjklmnspqrtvwz]",word[R1:]))[0].start() + 2 + R1
	except:
		R2 = len(word)
		
	#Make sure the index of R1 is at least 3. 
	
	if R1<3:
		try:
			R1 = list(re.finditer(r"[aëeiouäöüâêîôûæœ][bdghfcjklmnspqrtvwz]",word[1:]))[0].start() + 2
		except:
			R1 = len(word)
	
	if rem_umlaut:
		word = remove_umlaut(word)
	
	word = word[:R1] + re.sub(r'(wes|wen|est|ern|em|en|er|es|eȥ(?=[klmrt])s|(?=[lr])n|e)$',"",word[R1:])
	word = word[:R1] + re.sub(r'(est|er|en|re|in|iu|(?=.{3})st,word[R1:])$',"",word[R1:])
	word = word[:R2] + re.sub(r'(lich?.?.|keit|inc|isch?.?.)$',"",word[R2:])
	
	return word
661874
def tokenize(self, text):
        '''
        
        '''
        start = -1
        tokens = []
        for i, character in enumerate(text):
            if character == ' ' or character == '\t':
                if start >= 0:
                    word = text[start:i]
                    tokens.append({
                        'word': word,
                        'original_text': word,
                        'char_begin': start,
                        'char_end': i})
                    start = -1
            else:
                if start < 0:
                    start = i
        if start >= 0:
            tokens.append({
                'word': text[start:len(text)],
                'original_text': text[start:len(text)],
                'char_begin': start,
                'char_end': len(text)
            })
        return tokens
583081
def stem(self, word):
        """"""
        if self.stemmer:
            return unicode_to_ascii(self._stemmer.stem(word))
        else:
            return word
846189
def stem_word(self, p, i=0, j=None):
        
        ## --NLTK--
        if j is None and i == 0:
            word = p
        else:
            if j is None:
                j = len(p) - 1
            word = p[i:j+1]

        if word in self.pool:
            return self.pool[word]

        if len(word) <= 2:
            return word # --DEPARTURE--
        # With this line, strings of length 1 or 2 don't go through the
        # stemming process, although no mention is made of this in the
        # published algorithm. Remove the line to match the published
        # algorithm.

        word = self._step1ab(word)
        word = self._step1c(word)
        word = self._step2(word)
        word = self._step3(word)
        word = self._step4(word)
        word = self._step5(word)
        return word