Returns a list of words that occur in the text. Eliminates stopwords.
556505
def eliminate_repeats(text):
    ''''''
    
    bannedwords = read_file('stopwords.txt')
    alphabet = 'abcdefghijklmnopqrstuvwxyz'
    words = text.split()
    standardwords = []
    for word in words:
        newstr = ''
        for char in word:
            if char in alphabet or char in alphabet.upper():
               newstr += char
        if newstr not in standardwords and newstr != '' and newstr not in bannedwords:
            standardwords.append(newstr)
        
    return map(lambda x: x.lower(),standardwords)
1230235
public static List<Term> segment(char[] text)
    {
        List<Term> resultList = SEGMENT.seg(text);
        ListIterator<Term> listIterator = resultList.listIterator();
        while (listIterator.hasNext())
        {
            if (!CoreStopWordDictionary.shouldInclude(listIterator.next()))
            {
                listIterator.remove();
            }
        }

        return resultList;
    }
1248496
def word_counts(self):
        """"""
        counts = defaultdict(int)
        stripped_words = [lowerstrip(word) for word in self.words]
        for word in stripped_words:
            counts[word] += 1
        return counts
471399
function containsWords(wordlist, str) {
    var i;
    var matches = [];
    for (i = 0; i < wordlist.length; i++) {
        var re = new RegExp("\\b" + wordlist[i] + "\\b", "i");
        if (re.exec(str)) {
            matches.push(wordlist[i]);
        }
    }
    return matches;
}
902807
def word_spans(self):
        """"""
        if not self.is_tagged(WORDS):
            self.tokenize_words()
        return self.spans(WORDS)
531369
def keywords(text):
    
    text = split_words(text)
    numWords = len(text)  # of words before removing blacklist words
    freq = Counter(x for x in text if x not in stopWords)

    minSize = min(10, len(freq))  # get first 10
    keywords = {x: y for x, y in freq.most_common(minSize)}  # recreate a dict

    for k in keywords:
        articleScore = keywords[k]*1.0 / numWords
        keywords[k] = articleScore * 1.5 + 1

    return keywords
570982
def _clean_text(self, text):
        
        clean = []
        for char in text:
            if char in self.punc_stops:
                clean += '.'
            elif char not in self.punc:
                clean += char
            else:
                pass
        return (''.join(clean)).lower()
1286572
def clean_text_by_word(text, language="english", deacc=False, additional_stopwords=None):
    
    init_textcleanner(language, additional_stopwords)
    text_without_acronyms = replace_with_separator(text, "", [AB_ACRONYM_LETTERS])
    original_words = list(tokenize(text_without_acronyms, lowercase=True, deacc=deacc))
    filtered_words = filter_words(original_words)
    if HAS_PATTERN:
        tags = tag(" ".join(original_words))  # tag needs the context of the words in the text
    else:
        tags = None
    units = merge_syntactic_units(original_words, filtered_words, tags)
    return { unit.text : unit for unit in units }
559051
def get_matches(self, word):
        
        matches = self.gen_matches(word)
        # defend this against bad user input for regular expression patterns
        try:
            matches = self.exclude_matches(matches)
        except Exception:
            sys.stderr.write(traceback.format_exc())
            return None
        else:
            if self.use_suffix:
                matches = [self.inflect(match) for match in matches]
            return list(matches)
1295499
def complete_ls(self, text, *_):
        
        return [t + " " for t in self.engine.cached_descriptions if t.startswith(text)]
1345583
@SuppressWarnings("MethodMayBeStatic")
    public NerResultSet process(final String _text) {
        // tokenization
        final List<List<Token>> tokens = TOKENIZER.process(_text);
        // extract name phrase:
        final List<List<Phrase>> phrases = new ArrayList<>();

        for (final List<Token> tokenList : tokens) {
            // for each sentence
            final List<Phrase> sentencePhrase = new ArrayList<>();
            int wordPtr = 0;
            int phraseStubStartPtr = 0;
            while (wordPtr < tokenList.size()) {

                // iterate through every words in the sentence
                final List<Token> currentPhrase = new ArrayList<>();

                // get the phrase stub
                while (true) {
                    // we merge the names combined with the word 'and'
                    // if the second phrase is much shorter than the first phrase
                    // example: Canberra Institution of Science and Technology
                    boolean mergeAndFlag = false;
                    if ("and".equalsIgnoreCase(tokenList.get(wordPtr).text)
                        && !currentPhrase.isEmpty()
                        && wordPtr + 1 < tokenList.size()
                        && detectNameWordInSentenceByPosition(tokenList, wordPtr + 1)) {
                        if (wordPtr + currentPhrase.size() - 1 >= tokenList.size()) {
                            mergeAndFlag = true;
                        }
                        else {
                            for (int i = wordPtr + 1; i <= wordPtr + currentPhrase.size() - 1; i++) {
                                if (!detectNameWordInSentenceByPosition(tokenList, i)) {
                                    mergeAndFlag = true;
                                    break;
                                }
                            }
                        }
                    }

                    if (detectNameWordInSentenceByPosition(tokenList, wordPtr)
                        || (equalsIgnoreCase(tokenList.get(wordPtr).text, "of", "the", "'s")
                            && !currentPhrase.isEmpty()
                            && wordPtr + 1 < tokenList.size()
                            && detectNameWordInSentenceByPosition(tokenList, wordPtr + 1))
                        || mergeAndFlag) {
                        if (currentPhrase.isEmpty()) phraseStubStartPtr = wordPtr; // record the index of the first word
                        currentPhrase.add(tokenList.get(wordPtr));
                        wordPtr++;
                        if (wordPtr == tokenList.size()) break;
                    }
                    else break;
                }

                //get the attached 'the' or 'a'
                int phrasePos = phraseStubStartPtr;
                int phraseLen = currentPhrase.size();

                // if phrasePos is zero, then it is the first word, there won't be any 'the' or 'a' attached to this word.
                if (phrasePos > 0) {
                    final Token attachedArticle = tokenList.get(phrasePos - 1);
                    if ("the".equalsIgnoreCase(attachedArticle.text) || "a".equalsIgnoreCase(attachedArticle.text)) {
                        phrasePos--;
                        phraseLen++;
                    }
                }

                //handling non-name single words such as: Monday, January, etc.
                if (currentPhrase.size() == 1 && phraseStubStartPtr == phrasePos) {
                    if (NON_NAME_WORDS.contains(currentPhrase.get(0).text)) {
                        currentPhrase.clear();
                        wordPtr--;
                    }
                }

                if (currentPhrase.isEmpty()) {
                    // if the current pointed word is not a name phrase
                    // ESTIMATE IF IT IS A TIME/DATE
                    final List<Token> currentDatePhrase = new ArrayList<>();
                    final DatePhraseModel dpm = new DatePhraseModel();
                    int tempPtr = wordPtr;
                    while (true) {
                        final Token word = tokenList.get(tempPtr);
                        final DateType type = getDateType(word.text);
                        if (type != NONE
                            || equalsIgnoreCase(word.text, "of", ",", "the")
                               && !currentDatePhrase.isEmpty()
                               && tempPtr + 1 != tokenList.size()
                               && getDateType(tokenList.get(tempPtr + 1).text) != NONE
                               && getDateType(tokenList.get(tempPtr - 1).text) != NONE) {
                            // is date word:
                            currentDatePhrase.add(word);
                            dpm.addType(type);
                            tempPtr++;
                            if (tempPtr == tokenList.size()) break;
                        }
                        else break;
                    }
                    if (dpm.isDate()) {
                        // add the phrase to phrase array
                        sentencePhrase.add(new Date(currentDatePhrase, wordPtr, currentDatePhrase.size(), wordPtr));
                        wordPtr = tempPtr;
                    }
                }
                else if (!(currentPhrase.size() == 1
                           && (isPastTense(currentPhrase.get(0).text) || isPlural(currentPhrase.get(0).text)))) {
                    sentencePhrase.add(new Name(currentPhrase, phrasePos, phraseLen, phraseStubStartPtr));
                }

                wordPtr++;
            }

            phrases.add(sentencePhrase);
        }

        // find attached words(preps):
        for (int si = 0; si < tokens.size(); si++) {
            final List<Token> sentenceToken = tokens.get(si);
            final List<Phrase> sentencePhrase = phrases.get(si);
            for (int pi = 0; pi < sentencePhrase.size(); pi++) {
                getAttachedPrep(sentenceToken, sentencePhrase, pi);
            }
        }

        return new NerResultSet(phrases, tokens);
    }
199810
def stem(self, text):
        """"""

        stemmed_text = ''

        for word in text.split(' '):
            if word not in self.stops:

                # remove '-que' suffix
                word, in_que_pass_list = self._checkremove_que(word)
                if not in_que_pass_list:

                    # remove the simple endings from the target word
                    word, was_stemmed = self._matchremove_simple_endings(word)

                    # if word didn't match the simple endings, try verb endings
                    if not was_stemmed:
                        word = self._matchremove_verb_endings(word)

            # add the stemmed word to the text
            stemmed_text += word + ' '


        return stemmed_text
545882
def _lastWord(self, text):
        
        for index, char in enumerate(text[::-1]):
            if char.isspace() or \
               char in ('(', ')'):
                return text[len(text) - index :]
        else:
            return text
84788
public List<String> getWords() {
        List<String> result = new ArrayList<>();
        List<Word> words = WordParser.parse(title + snippet);
        for (Word word : words) {
            result.add(word.getText());
        }
        return result;
    }
1251833
def known(self, words: List[str]) -> List[str]:
        
        return list(w for w in words if w in self.__WORDS)
17585
def tokenize_words(self):
        
        if not self.is_tagged(SENTENCES):
            self.tokenize_sentences()
        tok = self.__word_tokenizer
        text = self.text
        dicts = []
        for sentence in self[SENTENCES]:
            sent_start, sent_end = sentence[START], sentence[END]
            sent_text = text[sent_start:sent_end]
            spans = tok.span_tokenize(sent_text)
            for start, end in spans:
                dicts.append({START: start+sent_start, END: end+sent_start, TEXT: sent_text[start:end]})
        self[WORDS] = dicts
        return self
776656
def get_wordlist(stanzas):
    
    return sorted(list(set().union(*[stanza.words for stanza in stanzas])))