Returns a list of English words which can be used to filter out
    code-switched sentences.
971641
def get_en_words() -> Set[str]:
    

    pull_en_words()

    with open(config.EN_WORDS_PATH) as words_f:
        raw_words = words_f.readlines()
    en_words = set([word.strip().lower() for word in raw_words])
    NA_WORDS_IN_EN_DICT = set(["kore", "nani", "karri", "imi", "o", "yaw", "i",
                           "bi", "aye", "imi", "ane", "kubba", "kab", "a-",
                           "ad", "a", "mak", "selim", "ngai", "en", "yo",
                           "wud", "mani", "yak", "manu", "ka-", "mong",
                           "manga", "ka-", "mane", "kala", "name", "kayo",
                           "kare", "laik", "bale", "ni", "rey", "bu",
                           "re", "iman", "bom", "wam",
                           "alu", "nan", "kure", "kuri", "wam", "ka", "ng",
                           "yi", "na", "m", "arri", "e", "kele", "arri", "nga",
                           "kakan", "ai", "ning", "mala", "ti", "wolk",
                           "bo", "andi", "ken", "ba", "aa", "kun", "bini",
                           "wo", "bim", "man", "bord", "al", "mah", "won",
                           "ku", "ay", "belen", "wen", "yah", "muni",
                           "bah", "di", "mm", "anu", "nane", "ma", "kum",
                           "birri", "ray", "h", "kane", "mumu", "bi", "ah",
                           "i-", "n", "mi", "bedman", "rud", "le", "babu",
                           "da", "kakkak", "yun", "ande", "naw", "kam", "bolk",
                           "woy", "u", "bi-",
                           ])
    EN_WORDS_NOT_IN_EN_DICT = set(["screenprinting"])
    en_words = en_words.difference(NA_WORDS_IN_EN_DICT)
    en_words = en_words | EN_WORDS_NOT_IN_EN_DICT
    return en_words
125923
def _cldf2wordlist(dataset, row='parameter_id', col='language_id'):
    """"""
    return lingpy.Wordlist(_cldf2wld(dataset), row=row, col=col)
1125787
public List<VocabularyWord> updateHuffmanCodes() {
        int min1i;
        int min2i;
        int b;
        int i;
        // get vocabulary as sorted list
        List<VocabularyWord> vocab = this.words();
        int count[] = new int[vocab.size() * 2 + 1];
        int parent_node[] = new int[vocab.size() * 2 + 1];
        byte binary[] = new byte[vocab.size() * 2 + 1];

        // at this point vocab is sorted, with descending order
        for (int a = 0; a < vocab.size(); a++)
            count[a] = vocab.get(a).getCount();
        for (int a = vocab.size(); a < vocab.size() * 2; a++)
            count[a] = Integer.MAX_VALUE;
        int pos1 = vocab.size() - 1;
        int pos2 = vocab.size();
        for (int a = 0; a < vocab.size(); a++) {
            // First, find two smallest nodes 'min1, min2'
            if (pos1 >= 0) {
                if (count[pos1] < count[pos2]) {
                    min1i = pos1;
                    pos1--;
                } else {
                    min1i = pos2;
                    pos2++;
                }
            } else {
                min1i = pos2;
                pos2++;
            }
            if (pos1 >= 0) {
                if (count[pos1] < count[pos2]) {
                    min2i = pos1;
                    pos1--;
                } else {
                    min2i = pos2;
                    pos2++;
                }
            } else {
                min2i = pos2;
                pos2++;
            }
            count[vocab.size() + a] = count[min1i] + count[min2i];
            parent_node[min1i] = vocab.size() + a;
            parent_node[min2i] = vocab.size() + a;
            binary[min2i] = 1;
        }

        // Now assign binary code to each vocabulary word
        byte[] code = new byte[MAX_CODE_LENGTH];
        int[] point = new int[MAX_CODE_LENGTH];

        for (int a = 0; a < vocab.size(); a++) {
            b = a;
            i = 0;
            byte[] lcode = new byte[MAX_CODE_LENGTH];
            int[] lpoint = new int[MAX_CODE_LENGTH];
            while (true) {
                code[i] = binary[b];
                point[i] = b;
                i++;
                b = parent_node[b];
                if (b == vocab.size() * 2 - 2)
                    break;
            }

            lpoint[0] = vocab.size() - 2;
            for (b = 0; b < i; b++) {
                lcode[i - b - 1] = code[b];
                lpoint[i - b] = point[b] - vocab.size();
            }

            vocab.get(a).setHuffmanNode(new HuffmanNode(lcode, lpoint, a, (byte) i));
        }

        idxMap.clear();
        for (VocabularyWord word : vocab) {
            idxMap.put(word.getHuffmanNode().getIdx(), word);
        }

        return vocab;
    }
688546
function (s) {
            var c = clean( s );
            var all = _.words( c, wordDelimiter );
            var words = _(all).chain()
                    // remove stop words
                    .filter( function (w) {
                        return (stopWords.indexOf( w ) === -1) ;
                    })
                    // apply stemmer
                    .map( function (w) {
                        return stemmer( w );
                    })
                    // collect word frequencies
                    .reduce( function (collect, w) {
                        collect[w] = collect[w] ? collect[w] + 1 : 1 ;
                        return collect;
                    }, {}).value();
            // remove a word from this sentence to reduce redundancy in results
            var remove = function (w) {
                return delete words[w];
            };
            return {
                orig: s,
                words: words,
                remove: remove
            };
        }
1325896
def negated(input_words, include_nt=True):
    
    input_words = [str(w).lower() for w in input_words]
    neg_words = []
    neg_words.extend(NEGATE)
    for word in neg_words:
        if word in input_words:
            return True
    if include_nt:
        for word in input_words:
            if "n't" in word:
                return True
    if "least" in input_words:
        i = input_words.index("least")
        if i > 0 and input_words[i - 1] != "at":
            return True
    return False
1317975
function filterTranslate (translateList, words) {
  if (translateList && translateList.length === 1 && translateList[0].trans === words) {
    return []
  }

  return translateList
}
418737
def wordify(text):
    
    stopset = set(nltk.corpus.stopwords.words('english'))
    tokens = nltk.WordPunctTokenizer().tokenize(text)
    return [w for w in tokens if w not in stopset]
950603
def get_filter_list(p_expression):
    
    result = []
    for arg in p_expression:
        # when a word starts with -, it should be negated
        is_negated = len(arg) > 1 and arg[0] == '-'
        arg = arg[1:] if is_negated else arg

        argfilter = None
        for match, _filter in MATCHES:
            if re.match(match, arg):
                argfilter = _filter(arg)
                break

        if not argfilter:
            argfilter = GrepFilter(arg)

        if is_negated:
            argfilter = NegationFilter(argfilter)

        result.append(argfilter)

    return result
206486
def remove_stopwords(self, tokens):
        """"""
        if isinstance(tokens, (list, tuple)):
            return [word for word in tokens if word.lower() not in self._stopwords]
        else:
            return ' '.join(
                [word for word in tokens.split(' ') if word.lower() not in self._stopwords]
            )
1181479
def scrub_sentence(sentence)
    sentence.split(/([\s]|--)+/).collect {|a| scrub_word(a)}.select {|a| a.length>0}
  end
522008
def regex_matches_to_indexed_words(matches):
    
    words = [dict(v, **dict([('sentence', i)]))
             for i, s in enumerate(matches['sentences'])
             for k, v in s.items() if k != 'length']
    return words
1188821
def drop_stopwords(list):
    
    stopwords_list = []
    with open(get_default_stop_words_file(), encoding='utf-8') as stopwords:
        for line in stopwords:
            stopwords_list.append(line.replace('\n', ''))

    list_clean = []
    for i in list:
        if i not in stopwords_list:
            list_clean.append(i)
    return list_clean
377736
public List<String> getSpannedWords() {
    if (isTerminal()) {
      return words;
    } else {
      List<String> words = Lists.newArrayList();
      words.addAll(left.getSpannedWords());
      words.addAll(right.getSpannedWords());
      return words;
    }
  }
482352
def p_word_list(p):
    ''''''
    parserobj = p.context
    if len(p) == 2:
        p[0] = [_expandword(parserobj, p.slice[1])]
    else:
        p[0] = p[1]
        p[0].append(_expandword(parserobj, p.slice[2]))
201659
def _load_words(self):
        """"""
        with open(self._words_file, 'r') as f:
            self._censor_list = [line.strip() for line in f.readlines()]
613034
def segment_sentences(self, sentences, lower=True, use_stop_words=True, use_speech_tags_filter=False):
        

        res = []
        for sentence in sentences:
            res.append(self.segment(text=sentence,
                                    lower=lower,
                                    use_stop_words=use_stop_words,
                                    use_speech_tags_filter=use_speech_tags_filter))
        return res
530252
def spell(self, word: str) -> List[str]:
        
        if not word:
            return ""

        candidates = (
            self.known([word])
            or self.known(_edits1(word))
            or self.known(_edits2(word))
            or [word]
        )
        candidates.sort(key=self.freq, reverse=True)

        return candidates