Get or generate the vocabulary.

  Args:
    data_dir: a string
    tmp_dir: a string
    data_prefix: a string
    max_page_size_exp: an integer
    approx_vocab_size: an integer
    strip: a boolean

  Returns:
    a TextEncoder
965691
def get_or_generate_vocabulary(data_dir,
                               tmp_dir,
                               data_prefix,
                               max_page_size_exp,
                               approx_vocab_size=32768,
                               strip=True):
  
  num_pages_for_vocab_generation = approx_vocab_size // 3
  vocab_file = vocab_filename(approx_vocab_size, strip)

  def my_generator(data_prefix):
    """Line generator for vocab."""
    count = 0
    for page in corpus_page_generator(
        all_corpus_files(data_prefix)[::-1], tmp_dir, max_page_size_exp):
      revisions = page["revisions"]
      if revisions:
        text = get_text(revisions[-1], strip=strip)
        yield text
        count += 1
        if count % 100 == 0:
          tf.logging.info("reading pages for vocab %d" % count)
        if count > num_pages_for_vocab_generation:
          break

  return generator_utils.get_or_generate_vocab_inner(data_dir, vocab_file,
                                                     approx_vocab_size,
                                                     my_generator(data_prefix))
956748
def words(min_size, max_size = None):
        
        max_size = max_size if max_size != None else min_size
        result = ""
        
        count = RandomInteger.next_integer(min_size, max_size)
        for i in range(count):
            result += random.choice(_all_words)

        return result
990572
def _get_words(data_dir):
    
    words_file = data_dir.joinpath("words.txt")
    word_list = []
    if not words_file.is_file():
        return word_list
    with open(words_file, mode="r") as f:
        for l in f:
            line = l.rstrip()
            word_list.append(line)
    return word_list
1035579
def _get_decoder(self, units, vocab_size, embed, prefix):
        
        with self.name_scope():
            decoder = nn.HybridSequential(prefix=prefix)
            decoder.add(nn.Dense(units, flatten=False))
            decoder.add(GELU())
            decoder.add(BERTLayerNorm(in_channels=units))
            decoder.add(nn.Dense(vocab_size, flatten=False, params=embed.collect_params()))
        assert decoder[3].weight == list(embed.collect_params().values())[0], \
            'The weights of word embedding are not tied with those of decoder'
        return decoder
291587
def extend_from_instances(self,
                              params: Params,
                              instances: Iterable['adi.Instance'] = ()) -> None:
        
        min_count = params.pop("min_count", None)
        max_vocab_size = pop_max_vocab_size(params)
        non_padded_namespaces = params.pop("non_padded_namespaces", DEFAULT_NON_PADDED_NAMESPACES)
        pretrained_files = params.pop("pretrained_files", {})
        min_pretrained_embeddings = params.pop("min_pretrained_embeddings", None)
        only_include_pretrained_words = params.pop_bool("only_include_pretrained_words", False)
        tokens_to_add = params.pop("tokens_to_add", None)
        params.assert_empty("Vocabulary - from dataset")

        logger.info("Fitting token dictionary from dataset.")
        namespace_token_counts: Dict[str, Dict[str, int]] = defaultdict(lambda: defaultdict(int))
        for instance in Tqdm.tqdm(instances):
            instance.count_vocab_items(namespace_token_counts)
        self._extend(counter=namespace_token_counts,
                     min_count=min_count,
                     max_vocab_size=max_vocab_size,
                     non_padded_namespaces=non_padded_namespaces,
                     pretrained_files=pretrained_files,
                     only_include_pretrained_words=only_include_pretrained_words,
                     tokens_to_add=tokens_to_add,
                     min_pretrained_embeddings=min_pretrained_embeddings)
239175
def generate_lines_for_vocab(tmp_dir, sources, file_byte_budget=1e6):
  """"""
  tf.logging.info("Generating vocab from: %s", str(sources))
  for source in sources:
    url = source[0]
    filename = os.path.basename(url)
    compressed_file = maybe_download(tmp_dir, filename, url)

    for lang_file in source[1]:
      tf.logging.info("Reading file: %s" % lang_file)
      filepath = os.path.join(tmp_dir, lang_file)

      # Extract from tar if needed.
      if not tf.gfile.Exists(filepath):
        read_type = "r:gz" if filename.endswith("tgz") else "r"
        with tarfile.open(compressed_file, read_type) as corpus_tar:
          corpus_tar.extractall(tmp_dir)

      # For some datasets a second extraction is necessary.
      if lang_file.endswith(".gz"):
        new_filepath = os.path.join(tmp_dir, lang_file[:-3])
        if tf.gfile.Exists(new_filepath):
          tf.logging.info(
              "Subdirectory %s already exists, skipping unpacking" % filepath)
        else:
          tf.logging.info("Unpacking subdirectory %s" % filepath)
          gunzip_file(filepath, new_filepath)
        filepath = new_filepath

      with tf.gfile.GFile(filepath, mode="r") as source_file:
        file_byte_budget_ = file_byte_budget
        counter = 0
        countermax = int(source_file.size() / file_byte_budget_ / 2)
        for line in source_file:
          if counter < countermax:
            counter += 1
          else:
            if file_byte_budget_ <= 0:
              break
            line = line.strip()
            file_byte_budget_ -= len(line)
            counter = 0
            yield line
98479
def feature_encoders(self, data_dir):
    
    encoders = (super(BabiQa, self).feature_encoders(data_dir))
    label_encoder = self.get_labels_encoder(data_dir)
    encoders["targets"] = label_encoder  # bAbi as a classification task
    return encoders
213983
def load_dataset(data_name):
    """"""
    if data_name == 'MR' or data_name == 'Subj':
        train_dataset, output_size = _load_file(data_name)
        vocab, max_len = _build_vocab(data_name, train_dataset, [])
        train_dataset, train_data_lengths = _preprocess_dataset(train_dataset, vocab, max_len)
        return vocab, max_len, output_size, train_dataset, train_data_lengths
    else:
        train_dataset, test_dataset, output_size = _load_file(data_name)
        vocab, max_len = _build_vocab(data_name, train_dataset, test_dataset)
        train_dataset, train_data_lengths = _preprocess_dataset(train_dataset, vocab, max_len)
        test_dataset, test_data_lengths = _preprocess_dataset(test_dataset, vocab, max_len)
        return vocab, max_len, output_size, train_dataset, train_data_lengths, test_dataset, \
               test_data_lengths
384376
def _maybe_download_corpora(tmp_dir):
  
  mnli_filename = "MNLI.zip"
  mnli_finalpath = os.path.join(tmp_dir, "MNLI")
  if not tf.gfile.Exists(mnli_finalpath):
    zip_filepath = generator_utils.maybe_download(
        tmp_dir, mnli_filename, _MNLI_URL)
    zip_ref = zipfile.ZipFile(zip_filepath, "r")
    zip_ref.extractall(tmp_dir)
    zip_ref.close()

  return mnli_finalpath
820322
def text2lm(text, output_file, vocab_file=None, text2idngram_kwargs={}, idngram2lm_kwargs={}):
    
    if vocab_file:
        used_vocab_file = vocab_file
    else:
        # Create temporary vocab file
        with tempfile.NamedTemporaryFile(suffix='.vocab', delete=False) as f:
            used_vocab_file = f.name
        text2vocab(text, used_vocab_file)

    # Create temporary idngram file
    with tempfile.NamedTemporaryFile(suffix='.idngram', delete=False) as f:
        idngram_file = f.name

    try:
        output1 = text2idngram(text, vocab_file=used_vocab_file, output_file=idngram_file, **text2idngram_kwargs)
        output2 = idngram2lm(idngram_file, vocab_file=used_vocab_file, output_file=output_file, **idngram2lm_kwargs)
    except ConversionError:
        output = (None, None)
        raise
    else:
        output = (output1, output2)
    finally:
        # Remove temporary files
        if not vocab_file:
            os.remove(used_vocab_file)
        os.remove(idngram_file)
    return output
751115
def _generate_examples(tmp_dir, dataset_split):
  
  if dataset_split == problem.DatasetSplit.TRAIN:
    file_name = _TRAINING_SET
  else:
    file_name = _DEV_SET
  squad_file = generator_utils.maybe_download(tmp_dir,
                                              file_name,
                                              os.path.join(_URL, file_name))
  with tf.gfile.GFile(squad_file, mode="r") as fp:
    squad = json.load(fp)

  version = squad["version"]
  for article in squad["data"]:
    if "title" in article:
      title = article["title"].strip()
    else:
      title = "no title"
    for paragraph in article["paragraphs"]:
      context = paragraph["context"].strip()
      for qa in paragraph["qas"]:
        question = qa["question"].strip()
        id_ = qa["id"]
        answer_starts = [answer["answer_start"] for answer in qa["answers"]]
        answers = [answer["text"].strip() for answer in qa["answers"]]

        # Features currently used are "context", "question", and "answers".
        # Others are extracted here for the ease of future expansions.
        example = {
            "version": version,
            "title": title,
            "context": context,
            "question": question,
            "id": id_,
            "answer_starts": answer_starts,
            "answers": answers,
            "num_answers": len(answers),
            "is_supervised": True,
        }
        yield example
947067
def get_or_generate_vocab(data_dir, tmp_dir, vocab_filename, vocab_size,
                          sources, file_byte_budget=1e6,
                          max_subtoken_length=None):
  """"""

  vocab_generator = generate_lines_for_vocab(tmp_dir, sources, file_byte_budget)
  return get_or_generate_vocab_inner(data_dir, vocab_filename, vocab_size,
                                     vocab_generator, max_subtoken_length)
129992
def load(fin, dtype=np.float32, max_vocab=None):
    
    vocab = {}
    line = next(fin)
    data = line.strip().split(b' ')
    assert len(data) == 2
    words = int(data[0])
    if max_vocab is not None:
        words = min(max_vocab, words)
    size = int(data[1])
    arr = np.empty((words, size), dtype=dtype)
    i = 0
    for n_line, line in enumerate(fin):
        if i >= words:
            break
        token, v = _load_line(line, dtype, size)
        if token in vocab:
            parse_warn(b'Duplicated vocabulary ' + token)
            continue
        arr[i, :] = v
        vocab[token] = i
        i += 1
    if i != words:
        # Use + instead of formatting because python 3.4.* does not allow
        # format with bytes
        parse_warn(
            b'EOF before the defined size (read ' + bytes(i) + b', expected '
            + bytes(words) + b')'
        )
        arr = arr[:i, :]
    return arr, vocab
781779
def get_labels_encoder(self, data_dir):
    
    label_filepath = os.path.join(data_dir, self.vocab_filename)
    return text_encoder.TokenTextEncoder(label_filepath)
763261
def load_with_vocab(fin, vocab, dtype=np.float32):
    
    line = next(fin)
    data = line.strip().split(b' ')
    assert len(data) == 2
    size = int(data[1])
    arr = np.empty((len(vocab), size), dtype=dtype)
    arr.fill(np.NaN)
    for line in fin:
        token, v = _load_line(line, dtype, size)
        if token in vocab:
            arr[vocab[token], :] = v
    if np.any(np.isnan(arr)):
        raise ParseError(b"Some of vocab was not found in word embedding file")
    return arr
823079
def load_vocabulary(lang="en", type="wiki"):
  
  src_dir = "{}_vocab".format(type)
  p = locate_resource(src_dir, lang)
  return CountedVocabulary.from_vocabfile(p)
22637
def _original_vocab(tmp_dir):
  
  vocab_url = ("http://download.tensorflow.org/models/LM_LSTM_CNN/"
               "vocab-2016-09-10.txt")
  vocab_filename = os.path.basename(vocab_url + ".en")
  vocab_filepath = os.path.join(tmp_dir, vocab_filename)
  if not os.path.exists(vocab_filepath):
    generator_utils.maybe_download(tmp_dir, vocab_filename, vocab_url)
  return set([
      text_encoder.native_to_unicode(l.strip())
      for l in tf.gfile.Open(vocab_filepath)
  ])