Get a set of valid words.

    If :path_to_user_dictionary: is specified, then the newline-separated
    words in that file will be added to the word set.
492334
def valid_words_set(path_to_user_dictionary=None,
                    user_dictionary_words=None):
    
    def read_file(binary_file):
        """Read a binary file for its text lines."""
        return binary_file.read().decode("ascii").splitlines()

    try:
        valid = _valid_words_cache[path_to_user_dictionary]
        return valid
    except KeyError:
        words = set()
        with resource_stream("polysquarelinter", "en_US.txt") as words_file:
            words |= set(["".join(l).lower() for l in read_file(words_file)])

        if path_to_user_dictionary:
            # Add both case-sensitive and case-insensitive variants
            # of words in user dictionary as they may be checked as
            # though they are a regular word and a technical word.
            words |= set([w.lower() for w in user_dictionary_words])
            words |= user_dictionary_words

        _valid_words_cache[path_to_user_dictionary] = words
        return words
1070459
def _get_raw_data(subset):
  """"""
  raw_data = _LOADED.get(subset)
  if raw_data is not None:
    return raw_data, _LOADED["vocab"]
  else:
    train_data, valid_data, test_data, vocab = ptb_reader.ptb_raw_data(
        FLAGS.data_path)
    _LOADED.update({
        "train": np.array(train_data),
        "valid": np.array(valid_data),
        "test": np.array(test_data),
        "vocab": vocab
    })
    return _LOADED[subset], vocab
186917
def get_wordlist(lang, wl_dir, po_path):
        #print("Looking for Wordlist in:\nlang {}\nwl_dir {}\npo_path {}".format(lang, wl_dir, po_path))
        po_path = os.path.abspath(po_path)

        
        if wl_dir is not None:
            wl_path = os.path.join(wl_dir, lang + '.txt')
            if os.path.isfile(wl_path):
                return wl_path

        """
        If wl_dir is not given, the wordlist should live in a file named
        "wordlist.txt" either in the locales_dir for the default language or in
        the same directory as the .po-files
        """
        if po_path.endswith("po"):
            # translated language
            po_dir = os.path.dirname(po_path)
            for f in os.scandir(po_dir):
                if f.name == "wordlist.txt":
                    #print("found wordlist in", f.path)
                    return f.path
            #print("Checked po-dir, None Found")

            """
            If no file was found so far, the po-files seem to lie in
            <lang>/LC_MESSAGES, and the wordlist should be in the directory
            above.
            """
            if os.path.basename(po_dir) == "LC_MESSAGES":
                for f in os.scandir(os.path.join(po_dir, "..")):
                    if f.name == "wordlist.txt":
                        #print("found wordlist in", f.path)
                        return f.path
            #print("Checked LC_MESSAGES-dir. none found")
        #print("Checked lang-specific files")

        if os.path.isdir(po_path):
            # default language
            for f in os.scandir(po_path):
                if f.name == "wordlist.txt":
                    #print("found wordlist in", f.path)
                    return f.path
        #print("If this shows up, no wordlist was found")
        return None
857979
def get_known_words(self, lang):
        """"""
        words = []
        for topic in self.user_data.language_data[lang]['skills']:
            if topic['learned']:
                words += topic['words']
        return set(words)
373360
def words(self, key):
        
        if self.default_mappings is not None and key not in self.mappings:
            return self.default_mappings[key].encode('utf').split("|")
        else:
            return self.mappings[key].encode('utf').split("|")
947067
def get_or_generate_vocab(data_dir, tmp_dir, vocab_filename, vocab_size,
                          sources, file_byte_budget=1e6,
                          max_subtoken_length=None):
  """"""

  vocab_generator = generate_lines_for_vocab(tmp_dir, sources, file_byte_budget)
  return get_or_generate_vocab_inner(data_dir, vocab_filename, vocab_size,
                                     vocab_generator, max_subtoken_length)
783008
def custom_stopwords(stopwords)
      unless stopwords.is_a?(Enumerable)
        if stopwords.strip.empty?
          stopwords = []
        elsif File.exist?(stopwords)
          stopwords = File.read(stopwords).force_encoding('utf-8').split
        else
          return # Do not overwrite the default
        end
      end
      TokenFilter::Stopword::STOPWORDS[@language] = Set.new stopwords
    end
796603
private function loadDictionaries()
    {
        if (null !== $this->words) {
            // Dictionaries already loaded
            return;
        }

        $this->words = array();

        $dictionaryFiles = glob($this->dictionaryDir . '/*');

        foreach ($dictionaryFiles as $dictionaryFile) {
            $fileName = substr($dictionaryFile, strlen($this->dictionaryDir) + 1);

            preg_match('/^(.+)\./', $fileName, $parts);
            $lang = $parts[1];

            $words = include $dictionaryFile;

            foreach ($words as $word) {
                if (!isset($this->words[$word])) {
                    $this->words[$word] = array();
                }

                if (!in_array($lang, $this->words[$word])) {
                    $this->words[$word][] = $lang;
                }
            }
        }
    }
990886
def _create_technical_words_dictionary(spellchecker_cache_path,
                                       relative_path,
                                       user_words,
                                       shadow):
    """"""
    technical_terms_set = (user_words |
                           technical_words_from_shadow_contents(shadow))
    technical_words = Dictionary(technical_terms_set,
                                 "technical_words_" +
                                 relative_path.replace(os.path.sep, "_"),
                                 [os.path.realpath(relative_path)],
                                 spellchecker_cache_path)
    return technical_words
642550
def _load_sentence_list(self, path):
        

        result = {}

        for entry in textfile.read_separated_lines_generator(path, separator='\t', max_columns=3):
            if self.include_languages is None or entry[1] in self.include_languages:
                result[entry[0]] = entry[1:]

        return result
1001920
def __get_keywords(self):
        
        txt = self.text
        for line in txt:
            for word in split_words(line):
                yield(word)
321676
def get_word_level_vocab(self):
        

        def simple_tokenize(source_str, token_delim=' ', seq_delim='\n'):
            return list(filter(None, re.split(token_delim + '|' + seq_delim, source_str)))

        return VocabProvider._create_squad_vocab(simple_tokenize, self._dataset)
1274143
def get_user_sets(client_id, user_id):
    """"""
    data = api_call('get', 'users/{}/sets'.format(user_id), client_id=client_id)
    return [WordSet.from_dict(wordset) for wordset in data]
580735
def create(spellchecker_cache_path):
    """"""
    user_dictionary = os.path.join(os.getcwd(), "DICTIONARY")
    user_words = read_dictionary_file(user_dictionary)

    valid_words = Dictionary(valid_words_set(user_dictionary, user_words),
                             "valid_words",
                             [user_dictionary],
                             spellchecker_cache_path)

    return (user_words, valid_words)
451723
def getWordList(ipFile, delim):
    
    indexedWords = {}
    totWords = 0
    totLines = 0
    with codecs.open(ipFile, "r",encoding='utf-8', errors='replace') as f:
        for line in f:
            totLines = totLines + 1
            words = multi_split(line, delim)
            totWords = totWords + len(words)
            for word in words:
                cleanedWord = word.lower().strip()
                if cleanedWord not in indexedWords:
                    indexedWords[cleanedWord] =  str(totLines)
                else:
                    indexedWords[cleanedWord] = indexedWords[cleanedWord] + ' ' + str(totLines)
    return totWords, totLines, indexedWords
1312427
def lookup(word):
	'''
	
	'''
	_patch_wordnik()
	# Jason's key - do not abuse
	key = 'edc4b9b94b341eeae350e087c2e05d2f5a2a9e0478cefc6dc'
	client = wordnik.swagger.ApiClient(key, 'http://api.wordnik.com/v4')
	words = wordnik.WordApi.WordApi(client)
	definitions = words.getDefinitions(word, limit=1)
	if not definitions:
		return
	definition = definitions[0]
	return str(definition.text)
1085567
def _updateWordSet(self):
        
        self._wordSet = set(self._keywords) | set(self._customCompletions)

        start = time.time()

        for line in self._qpart.lines:
            for match in _wordRegExp.findall(line):
                self._wordSet.add(match)
            if time.time() - start > self._WORD_SET_UPDATE_MAX_TIME_SEC:
                """It is better to have incomplete word set, than to freeze the GUI"""
                break