Return inputs and targets Tensors from a serialized tf.Example.
543492
def _parse_example(serialized_example):
  """"""
  data_fields = {
      "inputs": tf.VarLenFeature(tf.int64),
      "targets": tf.VarLenFeature(tf.int64)
  }
  parsed = tf.parse_single_example(serialized_example, data_fields)
  inputs = tf.sparse_tensor_to_dense(parsed["inputs"])
  targets = tf.sparse_tensor_to_dense(parsed["targets"])
  return inputs, targets
268633
def np2model_tensor(a):
    ""
    dtype = model_type(a.dtype)
    res = as_tensor(a)
    if not dtype: return res
    return res.type(dtype)
266268
def to_serialized_field(tensor_info):
  """"""
  # Select the type
  dtype = tensor_info.dtype

  # TODO(b/119937875): TF Examples proto only support int64, float32 and string
  # This create limitation like float64 downsampled to float32, bool converted
  # to int64 which is space ineficient, no support for complexes or quantized
  if tensor_info.dtype.is_integer or tensor_info.dtype.is_bool:
    dtype = tf.int64
  elif tensor_info.dtype.is_floating:
    dtype = tf.float32
  # It seems quite space inefficient to convert bool to int64
  # We may want to add support for complex, quantize dtype in the future

  # TFRecord only support 3 types
  if dtype not in (tf.int64, tf.float32, tf.string):
    raise NotImplementedError(
        'Serialization not implemented for {}'.format(dtype))

  # Select the feature proto type in function of the unknown shape
  if (tensor_info.shape is not None and  # Shape is a sequence (None, ...)
      tensor_info.shape.count(None) == 1 and
      tensor_info.shape[0] is None):
    return tf.io.FixedLenSequenceFeature(
        shape=tensor_info.shape[1:],
        dtype=dtype,
        allow_missing=True,
    )
  # At least one dimension is undefined
  elif tensor_info.shape is None or None in tensor_info.shape:
    return tf.io.VarLenFeature(dtype=dtype)
  else:
    return tf.io.FixedLenFeature(
        shape=tensor_info.shape,
        dtype=dtype,
    )
668012
def _tf_load_model(sess, model_dir):
  """"""

  meta_graph_pb = tf.saved_model.loader.load(
      sess=sess,
      tags=[tf.saved_model.tag_constants.SERVING],
      export_dir=model_dir)

  signature = meta_graph_pb.signature_def[signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY]
  input_alias_map = {friendly_name: tensor_info_proto.name
                     for (friendly_name, tensor_info_proto) in signature.inputs.items()}
  output_alias_map = {friendly_name: tensor_info_proto.name
                      for (friendly_name, tensor_info_proto) in signature.outputs.items()}

  return input_alias_map, output_alias_map
311912
def decode_transformer(encoder_output, encoder_decoder_attention_bias, targets,
                       hparams, name):
  """"""
  with tf.variable_scope(name):
    targets = common_layers.flatten4d3d(targets)

    decoder_input, decoder_self_bias = (
        transformer.transformer_prepare_decoder(targets, hparams))

    decoder_input = tf.nn.dropout(decoder_input,
                                  1.0 - hparams.layer_prepostprocess_dropout)

    decoder_output = transformer.transformer_decoder(
        decoder_input, encoder_output, decoder_self_bias,
        encoder_decoder_attention_bias, hparams)
    decoder_output = tf.expand_dims(decoder_output, axis=2)
    decoder_output_shape = common_layers.shape_list(decoder_output)
    decoder_output = tf.reshape(
        decoder_output, [decoder_output_shape[0], -1, 1, hparams.hidden_size])
    # Expand since t2t expects 4d tensors.
    return decoder_output
1292221
def validate(*tf_records):
    """"""
    if FLAGS.use_tpu:
        def _input_fn(params):
            return preprocessing.get_tpu_input_tensors(
                params['batch_size'], tf_records, filter_amount=1.0)
    else:
        def _input_fn():
            return preprocessing.get_input_tensors(
                FLAGS.train_batch_size, tf_records, filter_amount=1.0,
                shuffle_examples=False)

    steps = FLAGS.examples_to_validate // FLAGS.train_batch_size
    if FLAGS.use_tpu:
        steps //= FLAGS.num_tpu_cores

    estimator = dual_net.get_estimator()
    with utils.logged_timer("Validating"):
        estimator.evaluate(_input_fn, steps=steps, name=FLAGS.validate_name)
887837
def export_to_tf_tensor(self, x, laid_out_x):
    
    tensor_layout = self.tensor_layout(x.shape)
    if not tensor_layout.is_fully_replicated:
      raise NotImplementedError(
          "SimdMeshImpl only supports export_to_tf_tensor of fully-replicated "
          "Tensors.  Try reshaping to new dimension names. "
          " x.shape = %s tensor_layout=%s"
          % (x.shape, tensor_layout))
    return laid_out_x.one_slice
1339862
def make_preprocessing_fn(output_dir, features, keep_target):
  
  def preprocessing_fn(inputs):
    """Preprocessing function.

    Args:
      inputs: dictionary of raw input tensors

    Returns:
      A dictionary of transformed tensors
    """
    stats = json.loads(
      file_io.read_file_to_string(
          os.path.join(output_dir, STATS_FILE)).decode())

    result = {}
    for name, transform in six.iteritems(features):
      transform_name = transform['transform']
      source_column = transform['source_column']

      if transform_name == KEY_TRANSFORM:
        transform_name = 'identity'
      elif transform_name == TARGET_TRANSFORM:
        if not keep_target:
          continue
        if file_io.file_exists(os.path.join(output_dir, VOCAB_ANALYSIS_FILE % source_column)):
          transform_name = 'one_hot'
        else:
          transform_name = 'identity'

      if transform_name == 'identity':
        result[name] = inputs[source_column]
      elif transform_name == 'scale':
        result[name] = _scale(
            inputs[name],
            min_x_value=stats['column_stats'][source_column]['min'],
            max_x_value=stats['column_stats'][source_column]['max'],
            output_min=transform.get('value', 1) * (-1),
            output_max=transform.get('value', 1))
      elif transform_name in [ONE_HOT_TRANSFORM, EMBEDDING_TRANSFROM, MULTI_HOT_TRANSFORM,
                              TFIDF_TRANSFORM, BOW_TRANSFORM]:
        vocab, ex_count = read_vocab_file(
            os.path.join(output_dir, VOCAB_ANALYSIS_FILE % source_column))

        if transform_name == TFIDF_TRANSFORM:
          separator = transform.get('separator', ' ')
          tokens = tf.string_split(inputs[source_column], separator)
          ids = _string_to_int(tokens, vocab)
          weights = _tfidf(
              x=ids,
              reduced_term_freq=ex_count + [0],
              vocab_size=len(vocab) + 1,
              corpus_size=stats['num_examples'])

          result[name + '_ids'] = ids
          result[name + '_weights'] = weights
        elif transform_name == BOW_TRANSFORM:
          separator = transform.get('separator', ' ')
          tokens = tf.string_split(inputs[source_column], separator)
          ids = _string_to_int(tokens, vocab)
          weights = _bag_of_words(x=ids)

          result[name + '_ids'] = ids
          result[name + '_weights'] = weights
        elif transform_name == MULTI_HOT_TRANSFORM:
          separator = transform.get('separator', ' ')
          tokens = tf.string_split(inputs[source_column], separator)
          result[name] = _string_to_int(tokens, vocab)
        else:
          # ONE_HOT_TRANSFORM: making a dense vector is done at training
          # EMBEDDING_TRANSFROM: embedding vectors have to be done at training
          result[name] = _string_to_int(inputs[source_column], vocab)
      elif transform_name == IMAGE_TRANSFORM:
        make_image_to_vec_fn = _make_image_to_vec_tito(
            name, checkpoint=transform.get('checkpoint', None))
        result[name] = make_image_to_vec_fn(inputs[source_column])
      else:
        raise ValueError('unknown transform %s' % transform_name)
    return result

  return preprocessing_fn
728476
def normalize_example(self, example, hparams):
    """"""

    length = self.max_length(hparams)
    def _to_constant_shape(tensor):
      tensor = tensor[:length]
      tensor = tf.pad(tensor, [(0, length - tf.shape(tensor)[0])])
      return tf.reshape(tensor, [length])

    if self.has_inputs:
      example['inputs'] = _to_constant_shape(example['inputs'])
      example['targets'] = _to_constant_shape(example['targets'])
    elif 'inputs' in example:
      if self.packed_length:
        raise ValueError('cannot concatenate packed examples on the fly.')
      inputs = example.pop('inputs')[:-1]  # Remove EOS token.
      targets = tf.concat([inputs, example['targets']], 0)
      example['targets'] = _to_constant_shape(targets)
    else:
      example['targets'] = _to_constant_shape(example['targets'])
    if self.packed_length:
      if self.has_inputs:
        if 'inputs_segmentation' in example:
          example['inputs_segmentation'] = _to_constant_shape(
              example['inputs_segmentation'])
          example['inputs_position'] = _to_constant_shape(
              example['inputs_position'])
        else:
          example['inputs_segmentation'] = tf.to_int64(
              tf.not_equal(example['inputs'], 0))
          example['inputs_position'] = (
              example['inputs_segmentation'] * tf.range(length, dtype=tf.int64))
      if 'targets_segmentation' in example:
        example['targets_segmentation'] = _to_constant_shape(
            example['targets_segmentation'])
        example['targets_position'] = _to_constant_shape(
            example['targets_position'])
      else:
        example['targets_segmentation'] = tf.to_int64(
            tf.not_equal(example['targets'], 0))
        example['targets_position'] = (
            example['targets_segmentation'] * tf.range(length, dtype=tf.int64))
    return example
1092318
def encode_example(self, example_dict):
    """"""
    # Flatten dict matching the tf-example features
    # Use NonMutableDict to ensure there is no collision between features keys
    tfexample_dict = utils.NonMutableDict()

    # Iterate over example fields
    for feature_key, (feature, example_value) in utils.zip_dict(
        self._feature_dict, example_dict):
      # Encode the field with the associated encoder
      encoded_feature = feature.encode_example(example_value)

      # Singleton case
      if not feature.serialized_keys:
        tfexample_dict[feature_key] = encoded_feature
      # Feature contains sub features
      else:
        _assert_keys_match(encoded_feature.keys(), feature.serialized_keys)
        tfexample_dict.update({
            posixpath.join(feature_key, k): encoded_feature[k]
            for k in feature.serialized_keys
        })
    return tfexample_dict
126185
def make_prediction_output_tensors(args, features, input_ops, model_fn_ops,
                                   keep_target):
  """"""
  target_name = feature_transforms.get_target_name(features)
  key_names = get_key_names(features)

  outputs = {}
  outputs.update({key_name: tf.squeeze(input_ops.features[key_name])
                  for key_name in key_names})

  if is_classification_model(args.model):

    # build maps from ints to the origional categorical strings.
    class_names = read_vocab(args, target_name)
    table = tf.contrib.lookup.index_to_string_table_from_tensor(
        mapping=class_names,
        default_value='UNKNOWN')

    # Get the label of the input target.
    if keep_target:
      input_target_label = table.lookup(input_ops.features[target_name])
      outputs[PG_TARGET] = tf.squeeze(input_target_label)

    # TODO(brandondutra): get the score of the target label too.
    probabilities = model_fn_ops.predictions['probabilities']

    # if top_n == 0, this means use all the classes. We will use class names as
    # probabilities labels.
    if args.top_n == 0:
      predicted_index = tf.argmax(probabilities, axis=1)
      predicted = table.lookup(predicted_index)
      outputs.update({PG_CLASSIFICATION_FIRST_LABEL: predicted})
      probabilities_list = tf.unstack(probabilities, axis=1)
      for class_name, p in zip(class_names, probabilities_list):
        outputs[class_name] = p
    else:
      top_n = args.top_n

      # get top k labels and their scores.
      (top_k_values, top_k_indices) = tf.nn.top_k(probabilities, k=top_n)
      top_k_labels = table.lookup(tf.to_int64(top_k_indices))

      # Write the top_k values using 2*top_n columns.
      num_digits = int(math.ceil(math.log(top_n, 10)))
      if num_digits == 0:
        num_digits = 1
      for i in range(0, top_n):
        # Pad i based on the size of k. So if k = 100, i = 23 -> i = '023'. This
        # makes sorting the columns easy.
        padded_i = str(i + 1).zfill(num_digits)

        if i == 0:
          label_alias = PG_CLASSIFICATION_FIRST_LABEL
        else:
          label_alias = PG_CLASSIFICATION_LABEL_TEMPLATE % padded_i

        label_tensor_name = (tf.squeeze(
            tf.slice(top_k_labels, [0, i], [tf.shape(top_k_labels)[0], 1])))

        if i == 0:
          score_alias = PG_CLASSIFICATION_FIRST_SCORE
        else:
          score_alias = PG_CLASSIFICATION_SCORE_TEMPLATE % padded_i

        score_tensor_name = (tf.squeeze(
            tf.slice(top_k_values,
                     [0, i],
                     [tf.shape(top_k_values)[0], 1])))

        outputs.update({label_alias: label_tensor_name,
                        score_alias: score_tensor_name})

  else:
    if keep_target:
      outputs[PG_TARGET] = tf.squeeze(input_ops.features[target_name])

    scores = model_fn_ops.predictions['scores']
    outputs[PG_REGRESSION_PREDICTED_TARGET] = tf.squeeze(scores)

  return outputs
209636
def make_tf_example(features, pi, value):
    
    return tf.train.Example(features=tf.train.Features(feature={
        'x': tf.train.Feature(
            bytes_list=tf.train.BytesList(
                value=[features.tostring()])),
        'pi': tf.train.Feature(
            bytes_list=tf.train.BytesList(
                value=[pi.tostring()])),
        'outcome': tf.train.Feature(
            float_list=tf.train.FloatList(
                value=[value]))}))
1215396
def load_target_class(input_dir):
  """"""
  with tf.gfile.Open(os.path.join(input_dir, 'target_class.csv')) as f:
    return {row[0]: int(row[1]) for row in csv.reader(f) if len(row) >= 2}
941881
def find_signature_inputs_from_multivalued_ops(inputs):
  """"""
  dense_inputs = []  # List of (str, Tensor), with SparseTensors decomposed.
  for name, tensor in sorted(inputs.items()):
    if isinstance(tensor, tf.SparseTensor):
      dense_inputs.extend(("%s.%s" % (name, attr), getattr(tensor, attr))
                          for attr in ("indices", "values", "dense_shape"))
    else:
      dense_inputs.append((name, tensor))
  warnings = [(name, tensor.name) for name, tensor in dense_inputs
              if len(tensor.op.outputs) != 1]
  if warnings:
    return (
        "WARNING: The inputs declared in hub.add_signature() should be tensors "
        "from ops with a single output, or else uses of tf.colocate_with() on "
        "that op can trigger fatal errors when the module is applied and "
        "colocation constraints have to be rewritten.\nAffected inputs: %s" %
        ", ".join("%s='%s'" % pair for pair in warnings))
  return None
1075084
def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None):
    """"""
    del weight_collections, trainable  # Unused.
    m = module.Module(self.module_spec, trainable=False)
    images = inputs.get(self)
    return m({"images": images})
1163639
def load_batch(self, fn_batch):
        

        # TODO Assumes targets are available, which is how its distinct from
        # utils.load_batch_x(). These functions need to change names to be
        # clearer.

        inverse = list(zip(*fn_batch))
        feat_fn_batch = inverse[0]
        target_fn_batch = inverse[1]

        batch_inputs, batch_inputs_lens = utils.load_batch_x(feat_fn_batch,
                                                             flatten=False)
        batch_targets_list = []
        for targets_path in target_fn_batch:
            with open(targets_path, encoding=ENCODING) as targets_f:
                target_indices = self.corpus.labels_to_indices(targets_f.readline().split())
                batch_targets_list.append(target_indices)
        batch_targets = utils.target_list_to_sparse_tensor(batch_targets_list)

        return batch_inputs, batch_inputs_lens, batch_targets
487728
def produce_examples(shard_ids, wikis_dir, refs_dir, urls_dir, vocab_path,
                     out_filepaths):
  """"""
  # * Join the Wikipedia articles with their references
  # * Run Tf-idf to sort reference paragraphs
  # * Encode the Wikipedia and reference text with the vocabulary
  # * Write out TFRecords of tensorflow.Example
  tf.logging.info("Processing %d input shards into %d output files.",
                  len(shard_ids), len(out_filepaths))

  vocab = text_encoder.SubwordTextEncoder(vocab_path)
  eot_ids = vocab.encode(EOT)

  def example_generator():
    """Generate Example dicts."""
    stats = dict(total_original_wikis=0, total_original_refs=0,
                 total_found_refs=0, ref_lengths=[], wiki_original_refs=[],
                 wiki_found_refs=[], wikis_skipped_no_refs=0,
                 wikis_skipped_short_lead=0, num_wikis_written=0)
    ref_files_by_shard = _references_files_by_shard(refs_dir)
    for shard_id in shard_ids:
      tf.logging.info("Processing shard %d", shard_id)
      wiki_urls = _wiki_urls_for_shard(shard_id, urls_dir)
      tf.logging.info("Loaded wiki URLs for shard")
      refs_content = _references_content(ref_files_by_shard[shard_id])
      tf.logging.info("Loaded reference content for shard")
      for i, wiki in enumerate(_wiki_articles(shard_id, wikis_dir)):
        if not i % 1000:
          tf.logging.info("Processing wiki index %d for shard %d", i, shard_id)
        stats["total_original_wikis"] += 1

        # Get reference content
        wiki_ref_content = []
        ref_urls = wiki_urls[wiki.url]["refs"]
        stats["total_original_refs"] += len(ref_urls)
        stats_wiki_original_refs = len(ref_urls)
        stats_wiki_found_refs = 0
        for ref_url in ref_urls:
          ref_content = refs_content.get(ref_url)
          if not ref_content:
            continue
          stats["total_found_refs"] += 1
          stats["ref_lengths"].append(len(ref_content))
          stats_wiki_found_refs += 1
          wiki_ref_content.append(ref_content)

        stats["wiki_original_refs"].append(stats_wiki_original_refs)
        stats["wiki_found_refs"].append(stats_wiki_found_refs)
        if not wiki_ref_content or len(wiki_ref_content) < _MIN_REFS:
          # No/few refs were found
          stats["wikis_skipped_no_refs"] += 1
          continue

        # Rank reference paragraphs with TFIDF
        wiki_title = _normalize_text(wiki.title)
        ranked_paragraphs = rank_reference_paragraphs(wiki_title,
                                                      wiki_ref_content)

        # Construct inputs from Wiki title and references
        inputs = []
        inputs.extend(vocab.encode(wiki_title))
        inputs.extend(eot_ids)
        for paragraph in ranked_paragraphs:
          if len(inputs) >= 1e6:
            break
          paragraph += " "
          inputs.extend(vocab.encode(paragraph))

        # Construct targets from article sections
        targets, section_boundaries = _encode_wiki_sections(
            wiki.sections, vocab)

        # Skip if lead section is too short
        if (not section_boundaries or
            section_boundaries[0] < _MIN_LEADSECTION_TOKENS):
          stats["wikis_skipped_short_lead"] += 1
          continue

        inputs.append(text_encoder.EOS_ID)
        targets.append(text_encoder.EOS_ID)

        stats["num_wikis_written"] += 1
        yield {
            "inputs": inputs,
            "targets": targets,
            "section_boundaries": section_boundaries,
        }

    tf.logging.info("Total: %d, Skipped: %d",
                    stats["num_wikis_written"],
                    stats["total_original_wikis"] - stats["num_wikis_written"])
    tf.logging.info("Total refs: %d, Skipped refs: %d",
                    stats["total_found_refs"],
                    stats["total_original_refs"] - stats["total_found_refs"])
    stats_fname = os.path.join(os.path.split(out_filepaths[0])[0],
                               "stats.%d.json" % shard_ids[0])
    with tf.gfile.Open(stats_fname, "w") as f:
      f.write(json.dumps(stats))

  generator_utils.generate_files(example_generator(), out_filepaths)