Generates TF-Records for problems using a global vocabulary file.
330621
def generate_data_with_shared_vocab(self, data_dir, tmp_dir, task_id=-1):
    """"""
    global_vocab_filename = os.path.join(data_dir, self.vocab_filename)
    if not tf.gfile.Exists(global_vocab_filename):
      raise ValueError(
          'Global vocabulary file: %s does not exist, '
          'please create one using build_vocab.py' % global_vocab_filename)
    # Before generating data, we copy the global vocabulary file to the children
    # locations. Although this is not the most disk efficient strategy, it
    # imposes the fewest changes to the text-to-text API.
    for p in self.problems:
      local_vocab_filename = os.path.join(data_dir, p.vocab_filename)
      if not tf.gfile.Exists(local_vocab_filename):
        tf.gfile.Copy(global_vocab_filename, local_vocab_filename)
      p.generate_data(data_dir, tmp_dir, task_id)
734838
def report_problems(filename, problems):
    
    problem_file = filename.replace(".po", ".prob")
    id_filler = textwrap.TextWrapper(width=79, initial_indent="  msgid: ", subsequent_indent=" " * 9)
    tx_filler = textwrap.TextWrapper(width=79, initial_indent="  -----> ", subsequent_indent=" " * 9)
    with codecs.open(problem_file, "w", encoding="utf8") as prob_file:
        for problem in problems:
            desc, msgid = problem[:2]
            prob_file.write(u"{}\n{}\n".format(desc, id_filler.fill(msgid)))
            info = u"{}\n{}\n".format(desc, id_filler.fill(msgid))
            for translation in problem[2:]:
                prob_file.write(u"{}\n".format(tx_filler.fill(translation)))
                info += u"{}\n".format(tx_filler.fill(translation))
            log.info(info)
            prob_file.write(u"\n")

    log.error(" %s problems in %s, details in .prob file", len(problems), filename)
467163
def _generate_examples(self, archive, validation_labels=None):
    """"""
    if validation_labels:  # Validation split
      for example in self._generate_examples_validation(archive,
                                                        validation_labels):
        yield example
    # Training split. Main archive contains archives names after a synset noun.
    # Each sub-archive contains pictures associated to that synset.
    for fname, fobj in archive:
      label = fname[:-4]  # fname is something like 'n01632458.tar'
      # TODO(b/117643231): in py3, the following lines trigger tarfile module
      # to call `fobj.seekable()`, which Gfile doesn't have. We should find an
      # alternative, as this loads ~150MB in RAM.
      fobj_mem = io.BytesIO(fobj.read())
      for image_fname, image_fobj in tfds.download.iter_archive(
          fobj_mem, tfds.download.ExtractMethod.TAR):
        yield {
            'file_name': image_fname,
            'image': image_fobj,
            'label': label,
        }
1141125
def prepare_to_generate(self, data_dir, tmp_dir):
    """"""
    self.get_or_create_vocab(data_dir, tmp_dir)
    self.train_text_filepaths(tmp_dir)
    self.dev_text_filepaths(tmp_dir)
92400
def record(self):
        # type: () -> bytes
        '''
        
        '''
        if not self._initialized:
            raise pycdlibexception.PyCdlibInternalError('TF record not yet initialized!')

        outlist = [b'TF', struct.pack('=BBB', RRTFRecord.length(self.time_flags), SU_ENTRY_VERSION, self.time_flags)]
        for fieldname in self.FIELDNAMES:
            field = getattr(self, fieldname)
            if field is not None:
                outlist.append(field.record())

        return b''.join(outlist)
437565
def generate(num, prompt_default=True):
    """"""
    p = Problem(num)

    problem_text = p.text

    msg = "Generate file for problem %i?" % num
    click.confirm(msg, default=prompt_default, abort=True)

    # Allow skipped problem files to be recreated
    if p.glob:
        filename = str(p.file)
        msg = '"{}" already exists. Overwrite?'.format(filename)
        click.confirm(click.style(msg, fg='red'), abort=True)
    else:
        # Try to keep prefix consistent with existing files
        previous_file = Problem(num - 1).file
        prefix = previous_file.prefix if previous_file else ''
        filename = p.filename(prefix=prefix)

    header = 'Project Euler Problem %i' % num
    divider = '=' * len(header)
    text = '\n'.join([header, divider, '', problem_text])
    content = '\n'.join(['"""', text, '"""'])

    with open(filename, 'w') as f:
        f.write(content + '\n\n\n')

    click.secho('Successfully created "{}".'.format(filename), fg='green')

    # Copy over problem resources if required
    if p.resources:
        p.copy_resources()
1008710
def _maybe_download_corpus(tmp_dir, vocab_type):
  
  filename = os.path.basename(PTB_URL)
  compressed_filepath = generator_utils.maybe_download(
      tmp_dir, filename, PTB_URL)
  ptb_files = []
  ptb_char_files = []

  with tarfile.open(compressed_filepath, "r:gz") as tgz:
    files = []
    # Selecting only relevant files.
    for m in tgz.getmembers():
      if "ptb" in m.name and ".txt" in m.name:
        if "char" in m.name:
          ptb_char_files += [m.name]
        else:
          ptb_files += [m.name]
        files += [m]

    tgz.extractall(tmp_dir, members=files)

  if vocab_type == text_problems.VocabType.CHARACTER:
    return ptb_char_files
  else:
    return ptb_files
1256496
def _generate_create_dict(record, record_type, data, ttl, **kwargs):
        """"""

        # Basic dns record structure
        resource_record = {
            'host': record,
            'data': data,
            'ttl': ttl,
            'type': record_type
        }

        for (key, value) in kwargs.items():
            resource_record.setdefault(key, value)

        return resource_record
443201
def build_input_data(sentences, labels, vocabulary):
    """"""
    x = np.array([[vocabulary[word] for word in sentence] for sentence in sentences])
    y = np.array(labels)
    return [x, y]
1125439
def _generate_examples(self, file_path):
    

    with tf.io.gfile.GFile(file_path) as f:
      raw_data = csv.DictReader(f)
      for row in raw_data:
        survive_val = row.pop("survived")
        yield {
            "survived": convert_to_label(survive_val, _SURVIVED_DICT),
            "features": {
                name: FEATURE_DICT[name][1](value)
                for name, value in row.items()
            }
        }
15694
def generate(self, output_dir, catalogue, results, label):
        """"""
        data = results.get_raw_data()
        labels = catalogue.ordered_labels
        ngrams = self._generate_results(output_dir, labels, data)
        ngram_table = self._generate_ngram_table(output_dir, labels, data)
        corpus_table = self._generate_corpus_table(labels, ngrams)
        context = {'corpus_table': corpus_table, 'focus_label': label,
                   'labels': labels, 'ngram_table': ngram_table, 'sep': os.sep}
        report_name = 'lifetime-{}.html'.format(label)
        self._write(context, output_dir, report_name)
1032951
def _write_tfrecords_from_generator(generator, output_files, shuffle=True):
  """"""
  if do_files_exist(output_files):
    raise ValueError(
        "Pre-processed files already exists: {}.".format(output_files))

  with _incomplete_files(output_files) as tmp_files:
    # Write all shards
    writers = [tf.io.TFRecordWriter(fname) for fname in tmp_files]
    with _close_on_exit(writers) as writers:
      logging.info("Writing TFRecords")
      _round_robin_write(writers, generator)
    # Shuffle each shard
    if shuffle:
      # WARNING: Using np instead of Python random because Python random
      # produce different values between Python 2 and 3 and between
      # architectures
      random_gen = np.random.RandomState(42)
      for path in utils.tqdm(
          tmp_files, desc="Shuffling...", unit=" shard", leave=False):
        _shuffle_tfrecord(path, random_gen=random_gen)
810203
def get_input_tensors(batch_size, tf_records, num_repeats=1,
                      shuffle_records=True, shuffle_examples=True,
                      shuffle_buffer_size=None,
                      filter_amount=0.05, random_rotation=True):
    
    print("Reading tf_records from {} inputs".format(len(tf_records)))
    dataset = read_tf_records(
        batch_size,
        tf_records,
        num_repeats=num_repeats,
        shuffle_records=shuffle_records,
        shuffle_examples=shuffle_examples,
        shuffle_buffer_size=shuffle_buffer_size,
        filter_amount=filter_amount,
        interleave=True)
    dataset = dataset.filter(lambda t: tf.equal(tf.shape(t)[0], batch_size))
    dataset = dataset.map(
        functools.partial(batch_parse_tf_example, batch_size))
    if random_rotation:
        dataset = dataset.map(_random_rotation_pyfunc)

    return dataset.make_one_shot_iterator().get_next()
818522
def _init_vocab_from_file(self, filename):
    
    with tf.gfile.Open(filename) as f:
      tokens = [token.strip() for token in f.readlines()]

    def token_gen():
      for token in tokens:
        yield token

    self._init_vocab(token_gen(), add_reserved_tokens=False)
947067
def get_or_generate_vocab(data_dir, tmp_dir, vocab_filename, vocab_size,
                          sources, file_byte_budget=1e6,
                          max_subtoken_length=None):
  """"""

  vocab_generator = generate_lines_for_vocab(tmp_dir, sources, file_byte_budget)
  return get_or_generate_vocab_inner(data_dir, vocab_filename, vocab_size,
                                     vocab_generator, max_subtoken_length)
1139643
def data_files(self):
    
    tf_record_pattern = os.path.join(FLAGS.data_dir, '%s-*' % self.subset)
    data_files = tf.gfile.Glob(tf_record_pattern)
    if not data_files:
      print('No files found for dataset %s/%s at %s' % (self.name,
                                                        self.subset,
                                                        FLAGS.data_dir))

      self.download_message()
      exit(-1)
    return data_files
58401
def _build_vocab(filename, vocab_dir, vocab_name):
  
  vocab_path = os.path.join(vocab_dir, vocab_name)
  if not tf.gfile.Exists(vocab_path):
    with tf.gfile.GFile(filename, "r") as f:
      data = f.read().split()
    counter = collections.Counter(data)
    count_pairs = sorted(counter.items(), key=lambda x: (-x[1], x[0]))
    words, _ = list(zip(*count_pairs))
    encoder = text_encoder.TokenTextEncoder(None, vocab_list=words)
    encoder.store_to_file(vocab_path)
  else:
    encoder = text_encoder.TokenTextEncoder(vocab_path)
  return encoder