Produce examples from shard_ids to out_filepaths.
487728
def produce_examples(shard_ids, wikis_dir, refs_dir, urls_dir, vocab_path,
                     out_filepaths):
  """"""
  # * Join the Wikipedia articles with their references
  # * Run Tf-idf to sort reference paragraphs
  # * Encode the Wikipedia and reference text with the vocabulary
  # * Write out TFRecords of tensorflow.Example
  tf.logging.info("Processing %d input shards into %d output files.",
                  len(shard_ids), len(out_filepaths))

  vocab = text_encoder.SubwordTextEncoder(vocab_path)
  eot_ids = vocab.encode(EOT)

  def example_generator():
    """Generate Example dicts."""
    stats = dict(total_original_wikis=0, total_original_refs=0,
                 total_found_refs=0, ref_lengths=[], wiki_original_refs=[],
                 wiki_found_refs=[], wikis_skipped_no_refs=0,
                 wikis_skipped_short_lead=0, num_wikis_written=0)
    ref_files_by_shard = _references_files_by_shard(refs_dir)
    for shard_id in shard_ids:
      tf.logging.info("Processing shard %d", shard_id)
      wiki_urls = _wiki_urls_for_shard(shard_id, urls_dir)
      tf.logging.info("Loaded wiki URLs for shard")
      refs_content = _references_content(ref_files_by_shard[shard_id])
      tf.logging.info("Loaded reference content for shard")
      for i, wiki in enumerate(_wiki_articles(shard_id, wikis_dir)):
        if not i % 1000:
          tf.logging.info("Processing wiki index %d for shard %d", i, shard_id)
        stats["total_original_wikis"] += 1

        # Get reference content
        wiki_ref_content = []
        ref_urls = wiki_urls[wiki.url]["refs"]
        stats["total_original_refs"] += len(ref_urls)
        stats_wiki_original_refs = len(ref_urls)
        stats_wiki_found_refs = 0
        for ref_url in ref_urls:
          ref_content = refs_content.get(ref_url)
          if not ref_content:
            continue
          stats["total_found_refs"] += 1
          stats["ref_lengths"].append(len(ref_content))
          stats_wiki_found_refs += 1
          wiki_ref_content.append(ref_content)

        stats["wiki_original_refs"].append(stats_wiki_original_refs)
        stats["wiki_found_refs"].append(stats_wiki_found_refs)
        if not wiki_ref_content or len(wiki_ref_content) < _MIN_REFS:
          # No/few refs were found
          stats["wikis_skipped_no_refs"] += 1
          continue

        # Rank reference paragraphs with TFIDF
        wiki_title = _normalize_text(wiki.title)
        ranked_paragraphs = rank_reference_paragraphs(wiki_title,
                                                      wiki_ref_content)

        # Construct inputs from Wiki title and references
        inputs = []
        inputs.extend(vocab.encode(wiki_title))
        inputs.extend(eot_ids)
        for paragraph in ranked_paragraphs:
          if len(inputs) >= 1e6:
            break
          paragraph += " "
          inputs.extend(vocab.encode(paragraph))

        # Construct targets from article sections
        targets, section_boundaries = _encode_wiki_sections(
            wiki.sections, vocab)

        # Skip if lead section is too short
        if (not section_boundaries or
            section_boundaries[0] < _MIN_LEADSECTION_TOKENS):
          stats["wikis_skipped_short_lead"] += 1
          continue

        inputs.append(text_encoder.EOS_ID)
        targets.append(text_encoder.EOS_ID)

        stats["num_wikis_written"] += 1
        yield {
            "inputs": inputs,
            "targets": targets,
            "section_boundaries": section_boundaries,
        }

    tf.logging.info("Total: %d, Skipped: %d",
                    stats["num_wikis_written"],
                    stats["total_original_wikis"] - stats["num_wikis_written"])
    tf.logging.info("Total refs: %d, Skipped refs: %d",
                    stats["total_found_refs"],
                    stats["total_original_refs"] - stats["total_found_refs"])
    stats_fname = os.path.join(os.path.split(out_filepaths[0])[0],
                               "stats.%d.json" % shard_ids[0])
    with tf.gfile.Open(stats_fname, "w") as f:
      f.write(json.dumps(stats))

  generator_utils.generate_files(example_generator(), out_filepaths)
151823
def to_dict(self):
    """"""
    result = {"mapreduce_spec": self.mapreduce_spec.to_json_str(),
              "shard_id": self.shard_id,
              "slice_id": str(self.slice_id),
              "input_reader_state": self.input_reader.to_json_str(),
              "initial_input_reader_state":
              self.initial_input_reader.to_json_str(),
              "retries": str(self.retries)}
    if self.output_writer:
      result["output_writer_state"] = self.output_writer.to_json_str()
    serialized_handler = util.try_serialize_handler(self.handler)
    if serialized_handler:
      result["serialized_handler"] = serialized_handler
    return result
268438
def finalize(self, ctx, shard_state):
    """"""
    filenames = []
    for filehandle in self._filehandles:
      filenames.append(filehandle.name)
      filehandle.close()
    shard_state.writer_state = {"shard_filenames": filenames}
540122
def get_examples(examples_dir="examples/"):  # pragma: no cover
    """"""
    all_files = os.listdir(examples_dir)
    python_files = [f for f in all_files if is_python_file(f)]
    basenames = [remove_suffix(f) for f in python_files]
    modules = [import_module(module) for module in pathify(basenames)]
    return [
        module for module in modules
        if getattr(module, 'app', None) is not None
    ]
666349
def filename_to_task_id(fname):
  """"""
  # This matches the order and size in WikisumBase.out_filepaths
  fname = os.path.basename(fname)
  shard_id_increment = {
      "train": 0,
      "dev": 800,
      "test": 900,
  }
  parts = fname.split("-")
  split = parts[1]
  shard_id = parts[2]
  task_id = int(shard_id) + shard_id_increment[split]
  return task_id
20796
def _generate_examples(self, data_path):
    
    with tf.io.gfile.GFile(data_path, "rb") as fp:
      images = np.load(fp)
    images = np.transpose(images, (1, 0, 2, 3))
    images = np.expand_dims(images, axis=-1)
    for sequence in images:
      yield dict(image_sequence=sequence)
619544
def example_path(cls, project, dataset, annotated_dataset, example):
        """"""
        return google.api_core.path_template.expand(
            "projects/{project}/datasets/{dataset}/annotatedDatasets/{annotated_dataset}/examples/{example}",
            project=project,
            dataset=dataset,
            annotated_dataset=annotated_dataset,
            example=example,
        )
1144384
def _w_between_shards(args: Dict[str, Any]):
    """"""
    shard_num = args['shard_num']
    state = _state_shard(args)
    num_shard_qubits = args['num_shard_qubits']
    index = args['index']
    half_turns = args['half_turns']

    axis_half_turns = args['axis_half_turns']

    perm_index = shard_num ^ (1 << (index - num_shard_qubits))
    perm_state = mem_manager.SharedMemManager.get_array(
        args['state_handle']).view(np.complex64)[perm_index]

    cos = np.cos(-0.5 * np.pi * half_turns)
    sin = np.sin(-0.5 * np.pi * half_turns)

    cos_axis = np.cos(np.pi * axis_half_turns)
    sin_axis = np.sin(np.pi * axis_half_turns)

    scratch = _scratch_shard(args)
    z_op = (1 - 2 * _kth_bit(shard_num, index - num_shard_qubits))
    np.copyto(scratch, state * cos + 1j * sin * perm_state *
              (cos_axis - 1j * sin_axis * z_op))
816189
def _generate_examples(self, images_dir_path):
    
    parent_dir = tf.io.gfile.listdir(images_dir_path)[0]
    walk_dir = os.path.join(images_dir_path, parent_dir)
    dirs = tf.io.gfile.listdir(walk_dir)

    for d in dirs:
      if tf.io.gfile.isdir(os.path.join(walk_dir, d)):
        for full_path, _, fname in tf.io.gfile.walk(os.path.join(walk_dir, d)):
          for image_file in fname:
            if image_file.endswith(".jpg"):
              image_path = os.path.join(full_path, image_file)
              yield {
                  "image": image_path,
                  "label": d.lower(),
              }
485574
def _get_samples_to_process(fn, out_dir, config, force_single, separators):
    """"""
    out_dir = os.path.abspath(out_dir)
    samples = defaultdict(list)
    with open(fn) as handle:
        for l in handle:
            if l.find("description") > 0:
                logger.info("Skipping header.")
                continue
            cols = l.strip().split(",")
            if len(cols) > 0:
                if len(cols) < 2:
                    raise ValueError("Line needs 2 values: file and name.")
                if utils.file_exists(cols[0]) or is_gsm(cols[0]) or is_srr(cols[0]):
                    if cols[0].find(" ") > -1:
                        new_name = os.path.abspath(cols[0].replace(" ", "_"))
                        logger.warning("Space finds in %s. Linked to %s." % (cols[0], new_name))
                        logger.warning("Please, avoid names with spaces in the future.")
                        utils.symlink_plus(os.path.abspath(cols[0]), new_name)
                        cols[0] = new_name
                    samples[cols[1]].append(cols)
                else:
                    logger.info("skipping %s, File doesn't exist." % cols[0])
    for sample, items in samples.items():
        if is_fastq(items[0][0], True):
            fn = "fq_merge"
            ext = ".fastq.gz"
        elif is_bam(items[0][0]):
            fn = "bam_merge"
            ext = ".bam"
        elif is_gsm(items[0][0]):
            fn = "query_gsm"
            ext = ".fastq.gz"
        elif is_srr(items[0][0]):
            fn = "query_srr"
            ext = ".fastq.gz"
        files = [os.path.abspath(fn_file[0]) if utils.file_exists(fn_file[0]) else fn_file[0] for fn_file in items]
        samples[sample] = [{'files': _check_paired(files, force_single, separators),
                            'out_file': os.path.join(out_dir, sample + ext),
                            'fn': fn, 'anno': items[0][2:], 'config': config,
                            'name': sample, 'out_dir': out_dir}]
    return [samples[sample] for sample in samples]
1304985
def convert_ipynbs(directory):
    """"""
    # The ipython_examples dir has to be in the same dir as this script
    for root, subfolders, files in os.walk(os.path.abspath(directory)):
        for f in files:
            if ".ipynb_checkpoints" not in root:
                if f.endswith("ipynb"):
                    ipynb_to_rst(root, f)
467163
def _generate_examples(self, archive, validation_labels=None):
    """"""
    if validation_labels:  # Validation split
      for example in self._generate_examples_validation(archive,
                                                        validation_labels):
        yield example
    # Training split. Main archive contains archives names after a synset noun.
    # Each sub-archive contains pictures associated to that synset.
    for fname, fobj in archive:
      label = fname[:-4]  # fname is something like 'n01632458.tar'
      # TODO(b/117643231): in py3, the following lines trigger tarfile module
      # to call `fobj.seekable()`, which Gfile doesn't have. We should find an
      # alternative, as this loads ~150MB in RAM.
      fobj_mem = io.BytesIO(fobj.read())
      for image_fname, image_fobj in tfds.download.iter_archive(
          fobj_mem, tfds.download.ExtractMethod.TAR):
        yield {
            'file_name': image_fname,
            'image': image_fobj,
            'label': label,
        }
1169850
def BatchConvert(self, metadata_value_pairs, token=None):
    """"""

    urns = [urn for metadata, urn in metadata_value_pairs]
    urns_dict = dict((urn, metadata) for metadata, urn in metadata_value_pairs)

    results = []
    for hash_urn, client_files in filestore.HashFileStore.GetClientsForHashes(
        urns, token=token):
      for hit in client_files:
        metadata = ExportedMetadata(urns_dict[hash_urn])
        metadata.client_urn = rdfvalue.RDFURN(hit).Split(2)[0]

        result = ExportedFileStoreHash(
            metadata=metadata,
            hash=hash_urn.hash_value,
            fingerprint_type=hash_urn.fingerprint_type,
            hash_type=hash_urn.hash_type,
            target_urn=hit)
        results.append(result)

    return results
776627
def source_files(mongodb_path):
    """"""
    for root, dirs, files in os.walk(mongodb_path):
        for filename in files:
            # skip files in dbtests folder
            if 'dbtests' in root:
                continue
            if filename.endswith(('.cpp', '.c', '.h')):
                yield os.path.join(root, filename)
196925
def file_names(self) -> Iterable[str]:
        
        if self.is_sharded():
            yield from ShardedFile(self.filename_spec).get_filenames()
        elif self.filename_spec:
            yield self.filename_spec
1147244
def collect_outs(self):
        """"""
        # TODO: this should probably live in CacheManager.
        for outfile in self.rule.output_files or []:
            outfile_built = os.path.join(self.buildroot, outfile)
            if not os.path.exists(outfile_built):
                raise error.TargetBuildFailed(
                    self.address, 'Output file is missing: %s' % outfile)

        #git_sha = gitrepo.RepoState().GetRepo(self.address.repo).repo.commit()
        # git_sha is insufficient, and is actually not all that useful.
        # More factors to include in hash:
        # - commit/state of source repo of all dependencies
        #   (or all input files?)
        #   - Actually I like that idea: hash all the input files!
        # - versions of build tools used (?)
        metahash = self._metahash()
        log.debug('[%s]: Metahash: %s', self.address, metahash.hexdigest())
        # TODO: record git repo state and buildoptions in cachemgr
        # TODO: move cachemgr to outer controller(?)
        self.cachemgr.putfile(outfile_built, self.buildroot, metahash)
704904
def walk_tree(self):
        """"""
        shards = collections.deque([self])
        while shards:
            shard = shards.popleft()
            yield shard
            shards.extend(shard.children)