If the vocabulary path is defined, the vocabulary is loaded from the path.
    Otherwise, it is built from the data file. No writing to disk occurs.
429778
def load_or_create_vocab(data: str, vocab_path: Optional[str], num_words: int, word_min_count: int,
                         pad_to_multiple_of: Optional[int] = None) -> Vocab:
    
    if vocab_path is None:
        return build_from_paths(paths=[data], num_words=num_words, min_count=word_min_count,
                                pad_to_multiple_of=pad_to_multiple_of)
    else:
        return vocab_from_json(vocab_path)
658845
def make_vocab_from_args(args: argparse.Namespace):
    
    parameter_path = args.param_path
    overrides = args.overrides
    serialization_dir = args.serialization_dir

    params = Params.from_file(parameter_path, overrides)

    make_vocab_from_params(params, serialization_dir)
823079
def load_vocabulary(lang="en", type="wiki"):
  
  src_dir = "{}_vocab".format(type)
  p = locate_resource(src_dir, lang)
  return CountedVocabulary.from_vocabfile(p)
965691
def get_or_generate_vocabulary(data_dir,
                               tmp_dir,
                               data_prefix,
                               max_page_size_exp,
                               approx_vocab_size=32768,
                               strip=True):
  
  num_pages_for_vocab_generation = approx_vocab_size // 3
  vocab_file = vocab_filename(approx_vocab_size, strip)

  def my_generator(data_prefix):
    """Line generator for vocab."""
    count = 0
    for page in corpus_page_generator(
        all_corpus_files(data_prefix)[::-1], tmp_dir, max_page_size_exp):
      revisions = page["revisions"]
      if revisions:
        text = get_text(revisions[-1], strip=strip)
        yield text
        count += 1
        if count % 100 == 0:
          tf.logging.info("reading pages for vocab %d" % count)
        if count > num_pages_for_vocab_generation:
          break

  return generator_utils.get_or_generate_vocab_inner(data_dir, vocab_file,
                                                     approx_vocab_size,
                                                     my_generator(data_prefix))
914929
def from_vocabfile(cls, filename):
    
    words = [x.strip() for x in _open(filename, 'r').read().splitlines()]
    return cls(words=words)
764061
def save_to_files(self, directory: str) -> None:
        
        os.makedirs(directory, exist_ok=True)
        if os.listdir(directory):
            logging.warning("vocabulary serialization directory %s is not empty", directory)

        with codecs.open(os.path.join(directory, NAMESPACE_PADDING_FILE), 'w', 'utf-8') as namespace_file:
            for namespace_str in self._non_padded_namespaces:
                print(namespace_str, file=namespace_file)

        for namespace, mapping in self._index_to_token.items():
            # Each namespace gets written to its own file, in index order.
            with codecs.open(os.path.join(directory, namespace + '.txt'), 'w', 'utf-8') as token_file:
                num_tokens = len(mapping)
                start_index = 1 if mapping[0] == self._padding_token else 0
                for i in range(start_index, num_tokens):
                    print(mapping[i].replace('\n', '@@NEWLINE@@'), file=token_file)
587146
def get_vocabulary(preprocess_output_dir, name):
  
  vocab_file = os.path.join(preprocess_output_dir, CATEGORICAL_ANALYSIS % name)
  if not file_io.file_exists(vocab_file):
    raise ValueError('File %s not found in %s' %
                     (CATEGORICAL_ANALYSIS % name, preprocess_output_dir))

  labels = python_portable_string(
      file_io.read_file_to_string(vocab_file)).split('\n')
  label_values = [x for x in labels if x]  # remove empty lines

  return label_values
798152
def write(self, path=None):
        
        if not self._path and not path:
            raise ConfigException('no config path given')
        if path:
            self._path = path
        if '~' in self._path:
            self._path = os.path.expanduser(self._path)
        f = open(self._path, 'w')
        f.write(json.dumps(self._data))
        f.close()
564879
def load_from_file(cls, filename_prefix):
    """"""
    filename = cls._filename(filename_prefix)
    lines, _ = cls._read_lines_from_file(filename)
    # Strip wrapping single quotes
    vocab_list = [line[1:-1] for line in lines]
    return cls(vocab_list=vocab_list)
1223729
def load(self, path):
        
        with open(path, "rb") as f:
            content = f.read().decode("utf-8")

        section_text_list = list()

        lines = list()
        for line in content.split("\n"):
            if line.startswith("["):
                section_text_list.append("\n".join(lines))
                lines = list()
                lines.append(line)
            else:
                lines.append(line)

        section_text_list.append("\n".join(lines))

        self._sections = OrderedDict()
        for text in section_text_list[1:]:
            section = Section.from_text(text)
            self._sections[section.name] = section
768198
def words(query):
	
	filename = support.get_file_name(query)
	if(os.path.isfile(filename)):
		with open(filename) as openfile:
			print len(openfile.read().split())
	else:
		print 'File not found : ' + filename
186917
def get_wordlist(lang, wl_dir, po_path):
        #print("Looking for Wordlist in:\nlang {}\nwl_dir {}\npo_path {}".format(lang, wl_dir, po_path))
        po_path = os.path.abspath(po_path)

        
        if wl_dir is not None:
            wl_path = os.path.join(wl_dir, lang + '.txt')
            if os.path.isfile(wl_path):
                return wl_path

        """
        If wl_dir is not given, the wordlist should live in a file named
        "wordlist.txt" either in the locales_dir for the default language or in
        the same directory as the .po-files
        """
        if po_path.endswith("po"):
            # translated language
            po_dir = os.path.dirname(po_path)
            for f in os.scandir(po_dir):
                if f.name == "wordlist.txt":
                    #print("found wordlist in", f.path)
                    return f.path
            #print("Checked po-dir, None Found")

            """
            If no file was found so far, the po-files seem to lie in
            <lang>/LC_MESSAGES, and the wordlist should be in the directory
            above.
            """
            if os.path.basename(po_dir) == "LC_MESSAGES":
                for f in os.scandir(os.path.join(po_dir, "..")):
                    if f.name == "wordlist.txt":
                        #print("found wordlist in", f.path)
                        return f.path
            #print("Checked LC_MESSAGES-dir. none found")
        #print("Checked lang-specific files")

        if os.path.isdir(po_path):
            # default language
            for f in os.scandir(po_path):
                if f.name == "wordlist.txt":
                    #print("found wordlist in", f.path)
                    return f.path
        #print("If this shows up, no wordlist was found")
        return None
435986
def load_file(path, key_word)
      begin
        data = MultiJson.load(IO.read(File.expand_path(path)), symbolize_keys: true)
      rescue RuntimeError, Errno::ENOENT => e
        raise e.message
      rescue MultiJson::LoadError => e
        raise e.message
      end
        if data.has_key?(key_word)
        data[key_word].map { |k,v| data[key_word][k] = v } if data[key_word].is_a? Hash
      end
      data
    end
185345
def load_and_set_file_content(self, file_system_path):
        
        semantic_data = load_data_file(os.path.join(file_system_path, storage.SEMANTIC_DATA_FILE))
        self.model.state.semantic_data = semantic_data
979741
def _HasOOOWrite(self, path):
    """"""
    # Check the sizes of each path before the current one.
    size = tf.io.gfile.stat(path).length
    old_size = self._finalized_sizes.get(path, None)
    if size != old_size:
      if old_size is None:
        logger.error('File %s created after file %s even though it\'s '
                         'lexicographically earlier', path, self._path)
      else:
        logger.error('File %s updated even though the current file is %s',
                         path, self._path)
      return True
    else:
      return False
548609
def train_phrases(paths, out='data/bigram_model.phrases', tokenizer=word_tokenize, **kwargs):
    
    n = 0
    for path in paths:
        print('Counting lines for {0}...'.format(path))
        n += sum(1 for line in open(path, 'r'))
    print('Processing {0} lines...'.format(n))

    # Change to use less memory. Default is 40m.
    kwargs = {
        'max_vocab_size': 40000000,
        'threshold': 8.
    }.update(kwargs)

    print('Training bigrams...')
    bigram = Phrases(_phrase_doc_stream(paths, n, tokenizer=word_tokenize), **kwargs)

    print('Saving...')
    bigram.save(out)
782008
def _load(pathtovector,
              wordlist,
              num_to_load=None,
              truncate_embeddings=None,
              sep=" "):
        """"""
        vectors = []
        addedwords = set()
        words = []

        try:
            wordlist = set(wordlist)
        except ValueError:
            wordlist = set()

        logger.info("Loading {0}".format(pathtovector))

        firstline = open(pathtovector).readline().strip()
        try:
            num, size = firstline.split(sep)
            num, size = int(num), int(size)
            logger.info("Vector space: {} by {}".format(num, size))
            header = True
        except ValueError:
            size = len(firstline.split(sep)) - 1
            logger.info("Vector space: {} dim, # items unknown".format(size))
            word, rest = firstline.split(sep, 1)
            # If the first line is correctly parseable, set header to False.
            header = False

        if truncate_embeddings is None or truncate_embeddings == 0:
            truncate_embeddings = size

        for idx, line in enumerate(open(pathtovector, encoding='utf-8')):

            if header and idx == 0:
                continue

            word, rest = line.rstrip(" \n").split(sep, 1)

            if wordlist and word not in wordlist:
                continue

            if word in addedwords:
                raise ValueError("Duplicate: {} on line {} was in the "
                                 "vector space twice".format(word, idx))

            if len(rest.split(sep)) != size:
                raise ValueError("Incorrect input at index {}, size "
                                 "is {}, expected "
                                 "{}".format(idx+1,
                                             len(rest.split(sep)), size))

            words.append(word)
            addedwords.add(word)
            vectors.append(np.fromstring(rest, sep=sep)[:truncate_embeddings])

            if num_to_load is not None and len(addedwords) >= num_to_load:
                break

        vectors = np.array(vectors).astype(np.float32)

        logger.info("Loading finished")
        if wordlist:
            diff = wordlist - addedwords
            if diff:
                logger.info("Not all items from your wordlist were in your "
                            "vector space: {}.".format(diff))

        return vectors, words