Returns a ParsedTensorInfo instance from a TensorInfo proto.
197494
def _parse_tensor_info_proto(tensor_info):
  """"""
  encoding = tensor_info.WhichOneof("encoding")
  dtype = tf.DType(tensor_info.dtype)
  shape = tf.TensorShape(tensor_info.tensor_shape)
  if encoding == "name":
    return ParsedTensorInfo(dtype=dtype, shape=shape, is_sparse=False)
  elif encoding == "coo_sparse":
    return ParsedTensorInfo(dtype=dtype, shape=shape, is_sparse=True)
  else:
    raise ValueError("Unsupported TensorInfo encoding %r" % encoding)
1343601
@PublicEvolving
	public static <X extends Tuple> TupleTypeInfo<X> getBasicTupleTypeInfo(Class<?>... basicTypes) {
		if (basicTypes == null || basicTypes.length == 0) {
			throw new IllegalArgumentException();
		}
		
		TypeInformation<?>[] infos = new TypeInformation<?>[basicTypes.length];
		for (int i = 0; i < infos.length; i++) {
			Class<?> type = basicTypes[i];
			if (type == null) {
				throw new IllegalArgumentException("Type at position " + i + " is null.");
			}
			
			TypeInformation<?> info = BasicTypeInfo.getInfoFor(type);
			if (info == null) {
				throw new IllegalArgumentException("Type at position " + i + " is not a basic type.");
			}
			infos[i] = info;
		}

		@SuppressWarnings("unchecked")
		TupleTypeInfo<X> tupleInfo = (TupleTypeInfo<X>) new TupleTypeInfo<Tuple>(infos);
		return tupleInfo;
	}
333007
def get_input_shape(sym, proto_obj):
    """"""
    arg_params = proto_obj.arg_dict
    aux_params = proto_obj.aux_dict

    model_input_shape = [data[1] for data  in proto_obj.model_metadata.get('input_tensor_data')]
    data_names = [data[0] for data  in proto_obj.model_metadata.get('input_tensor_data')]

    # creating dummy inputs
    inputs = []
    for  in_shape in model_input_shape:
        inputs.append(nd.ones(shape=in_shape))

    data_shapes = []
    for idx, input_name in enumerate(data_names):
        data_shapes.append((input_name, inputs[idx].shape))

    ctx = context.cpu()
    # create a module
    mod = module.Module(symbol=sym, data_names=data_names, context=ctx, label_names=None)
    mod.bind(for_training=False, data_shapes=data_shapes, label_shapes=None)
    mod.set_params(arg_params=arg_params, aux_params=aux_params)

    data_forward = []
    for idx, input_name in enumerate(data_names):
        val = inputs[idx]
        data_forward.append(val)

    mod.forward(io.DataBatch(data_forward))
    result = mod.get_outputs()[0].asnumpy()

    return result.shape
87916
def Parse(self, stat, file_object, knowledge_base):
    """"""
    _, _ = stat, knowledge_base
    # TODO(user): Convert this to use the far more intelligent plaso parser.
    ie = IEParser(file_object)
    for dat in ie.Parse():
      yield rdf_webhistory.BrowserHistoryItem(
          url=dat["url"],
          domain=urlparse.urlparse(dat["url"]).netloc,
          access_time=dat.get("mtime"),
          program_name="Internet Explorer",
          source_urn=file_object.urn)
1015132
def _parse_device_information(device_info_string) -> dict:
    '''
    
    '''
    error_msg = 'Unexpected argument to _parse_device_information: {}'.format(
        device_info_string)
    if not device_info_string or \
            not isinstance(device_info_string, str):
        raise ParseError(error_msg)
    parsed_values = device_info_string.strip().split(' ')
    if len(parsed_values) < 3:
        log.error(error_msg)
        raise ParseError(error_msg)
    res = {
        _parse_key_from_substring(s): _parse_string_value_from_substring(s)
        for s in parsed_values[:3]
    }
    for key in ['model', 'version', 'serial']:
        if key not in res:
            raise ParseError(error_msg)
    return res
862993
def _load_graphdef_protobuf(handle, **kwargs):
    """"""
    # as_graph_def
    graph_def = tf.GraphDef.FromString(handle.read())

    # check if this is a lucid-saved model
    # metadata = modelzoo.util.extract_metadata(graph_def)
    # if metadata is not None:
    #   url = handle.name
    #   return modelzoo.vision_base.Model.load_from_metadata(url, metadata)

    # else return a normal graph_def
    return graph_def
784641
def serving_from_csv_input(train_config, args, keep_target):
  """"""
  examples = tf.placeholder(
      dtype=tf.string,
      shape=(None,),
      name='csv_input_string')

  features = parse_example_tensor(examples=examples,
                                  train_config=train_config,
                                  keep_target=keep_target)

  if keep_target:
    target = features.pop(train_config['target_column'])
  else:
    target = None
  features, target = preprocess_input(
      features=features,
      target=target,
      train_config=train_config,
      preprocess_output_dir=args.preprocess_output_dir,
      model_type=args.model_type)

  return input_fn_utils.InputFnOps(features,
                                   target,
                                   {'csv_line': examples}
                                   )
256332
public org.tensorflow.framework.TensorProto getTensor() {
    if (valueCase_ == 8) {
       return (org.tensorflow.framework.TensorProto) value_;
    }
    return org.tensorflow.framework.TensorProto.getDefaultInstance();
  }
1370998
def metainfo_to_protobuf(self) -> bytes:
        ''''''

        deck = deckspawnproto()
        deck.version = self.version
        deck.name = self.name
        deck.number_of_decimals = self.number_of_decimals
        deck.issue_mode = self.issue_mode
        if self.asset_specific_data:
            if not isinstance(self.asset_specific_data, bytes):
                deck.asset_specific_data = self.asset_specific_data.encode()
            else:
                deck.asset_specific_data = self.asset_specific_data

        if deck.ByteSize() > net_query(self.network).op_return_max_bytes:
            raise OverSizeOPReturn('''
                        Metainfo size exceeds maximum of {max} bytes supported by this network.'''
                                   .format(max=net_query(self.network)
                                           .op_return_max_bytes))

        return deck.SerializeToString()
705415
def _parse_proto(prototxt_fname):
    
    proto = caffe_parser.read_prototxt(prototxt_fname)

    # process data layer
    input_name, input_dim, layers = _get_input(proto)
    # only support single input, so always use `data` as the input data
    mapping = {input_name: 'data'}
    need_flatten = {input_name: False}
    symbol_string = "import mxnet as mx\ndata = mx.symbol.Variable(name='data')\n"

    flatten_count = 0
    output_name = ""
    prev_name = None

    # convert reset layers one by one
    for i, layer in enumerate(layers):
        type_string = ''
        param_string = ''
        skip_layer = False
        bottom_order = []
        name = re.sub('[-/]', '_', layer.name)
        if layer.type == 'Convolution' or layer.type == 4:
            type_string = 'mx.symbol.Convolution'
            param_string = _convert_conv_param(layer.convolution_param)
            need_flatten[name] = True
        if layer.type == 'Deconvolution' or layer.type == 39:
            type_string = 'mx.symbol.Deconvolution'
            param_string = _convert_conv_param(layer.convolution_param)
            need_flatten[name] = True
        if layer.type == 'Pooling' or layer.type == 17:
            type_string = 'mx.symbol.Pooling'
            param_string = _convert_pooling_param(layer.pooling_param)
            need_flatten[name] = True
        if layer.type == 'ReLU' or layer.type == 18:
            type_string = 'mx.symbol.Activation'
            param_string = "act_type='relu'"
            need_flatten[name] = need_flatten[mapping[layer.bottom[0]]]
        if layer.type == 'TanH' or layer.type == 23:
            type_string = 'mx.symbol.Activation'
            param_string = "act_type='tanh'"
            need_flatten[name] = need_flatten[mapping[layer.bottom[0]]]
        if layer.type == 'Sigmoid' or layer.type == 19:
            type_string = 'mx.symbol.Activation'
            param_string = "act_type='sigmoid'"
            need_flatten[name] = need_flatten[mapping[layer.bottom[0]]]
        if layer.type == 'LRN' or layer.type == 15:
            type_string = 'mx.symbol.LRN'
            param = layer.lrn_param
            param_string = "alpha=%f, beta=%f, knorm=%f, nsize=%d" % (
                param.alpha, param.beta, param.k, param.local_size)
            need_flatten[name] = True
        if layer.type == 'InnerProduct' or layer.type == 14:
            type_string = 'mx.symbol.FullyConnected'
            param = layer.inner_product_param
            param_string = "num_hidden=%d, no_bias=%s" % (
                param.num_output, not param.bias_term)
            need_flatten[name] = False
        if layer.type == 'Dropout' or layer.type == 6:
            type_string = 'mx.symbol.Dropout'
            param = layer.dropout_param
            param_string = "p=%f" % param.dropout_ratio
            need_flatten[name] = need_flatten[mapping[layer.bottom[0]]]
        if layer.type == 'Softmax' or layer.type == 20:
            if layer.softmax_param.axis == 2:
                symbol_string += "%s = mx.symbol.transpose(%s, axes=(0,2,1))\n" %\
                    (mapping[layer.bottom[0]], mapping[layer.bottom[0]])
                type_string = 'mx.symbol.SoftmaxActivation'
                param_string = "mode='channel'"
                need_flatten[name] = False
            else:
                type_string = 'mx.symbol.SoftmaxOutput'
        if layer.type == 'Flatten' or layer.type == 8:
            if 'softmax' in layer.bottom[0]:
                prev_name = re.sub('[-/]', '_', layers[i-1].name)
                skip_layer = True
            else:
                type_string = 'mx.symbol.Flatten'
            need_flatten[name] = False
        if layer.type == 'Split' or layer.type == 22:
            type_string = 'split'  # will process later
        if layer.type == 'Concat' or layer.type == 3:
            type_string = 'mx.symbol.Concat'
            need_flatten[name] = True
        if layer.type == 'Crop':
            type_string = 'mx.symbol.Crop'
            need_flatten[name] = True
            param_string = 'center_crop=True'
        if layer.type == 'BatchNorm':
            type_string = 'mx.symbol.BatchNorm'
            param = layer.batch_norm_param
            # CuDNN requires eps to be greater than 1e-05
            # We compensate for this change in convert_model
            epsilon = param.eps
            if (epsilon <= 1e-05):
                epsilon = 1e-04
            # if next layer is scale, don't fix gamma
            fix_gamma = layers[i+1].type != 'Scale'
            param_string = 'use_global_stats=%s, fix_gamma=%s, eps=%f' % (
                param.use_global_stats, fix_gamma, epsilon)
            need_flatten[name] = need_flatten[mapping[layer.bottom[0]]]
        if layer.type == 'Scale':
            assert layers[i-1].type == 'BatchNorm'
            need_flatten[name] = need_flatten[mapping[layer.bottom[0]]]
            skip_layer = True
            prev_name = re.sub('[-/]', '_', layers[i-1].name)
        if layer.type == 'PReLU':
            type_string = 'mx.symbol.LeakyReLU'
            param = layer.prelu_param
            param_string = "act_type='prelu', slope=%f" % param.filler.value
            need_flatten[name] = need_flatten[mapping[layer.bottom[0]]]
        if layer.type == 'Eltwise':
            type_string = 'mx.symbol.broadcast_add'
            param_string = ""
            need_flatten[name] = False
        if layer.type == 'Reshape':
            type_string = 'mx.symbol.Reshape'
            param = layer.reshape_param
            param_string = 'shape=(' + ','.join([str(x) for x in list(param.shape.dim)]) + ')'
            need_flatten[name] = True
        if layer.type == 'AbsVal':
            type_string = 'mx.symbol.abs'
            need_flatten[name] = need_flatten[mapping[layer.bottom[0]]]
        if layer.type == 'Normalize':
            bottom = re.sub('[-/]', '_', layer.bottom[0])
            conv_layer = _find_layer(layers, bottom)
            assert conv_layer is not None
            param = layer.norm_param
            assert not param.across_spatial and not param.channel_shared
            assert param.scale_filler.type == 'constant'
            if conv_layer.type == 'Convolution':
                scale_name = "%s_scale" % name
                symbol_string += "%s=mx.sym.Variable(name='%s', shape=(1, %d, 1, 1), init=mx.init.Constant(%f))\n" % \
                    (scale_name, scale_name, conv_layer.convolution_param.num_output,
                    param.scale_filler.value)
                symbol_string += "%s=mx.symbol.L2Normalization(name='%s', data=%s, mode='channel')\n" %\
                    (name, name, mapping[layer.bottom[0]])
                symbol_string += "%s=mx.symbol.broadcast_mul(lhs=%s, rhs=%s)\n" %\
                    (name, scale_name, name)
                type_string = 'split'
                need_flatten[name] = True
            else:
                raise ValueError('Unknown/Invalid normalize layer!')
        if layer.type == 'Permute':
            type_string = 'mx.symbol.transpose'
            param_string = "axes=(%s)" % (','.join([str(x) for x in layer.permute_param.order]))
            need_flatten[name] = True
            from_name = ''
        if layer.type == 'PriorBox':
            param = layer.prior_box_param
            if layer.bottom[0] == 'data':
                bottom_order = [1]
            else:
                bottom_order = [0]
            try:
                import math
                min_size = param.min_size[0] / input_dim[2]
                max_size = math.sqrt(param.min_size[0] * param.max_size[0]) / input_dim[2]
                sizes = '(%f, %f)' %(min_size, max_size)
            except AttributeError:
                min_size = param.min_size[0] / input_dim[2]
                sizes = '(%f)' %(min_size)
            ars = list(param.aspect_ratio)
            ratios = [1.]
            for ar in ars:
                ratios.append(ar)
                if param.flip:
                    ratios.append(1. / ar)
            ratios_string = '(' + ','.join(str(x) for x in ratios) + ')'
            clip = param.clip
            if (param.step_h > 0 or param.step_w > 0):
                step_h = param.step_h
                step_w = param.step_w
            elif param.step > 0:
                step_h = param.step
                step_w = param.step
            else:
                step_h = -1
                step_w = -1
            finput_dimh = float(input_dim[2])
            finput_dimw = float(input_dim[3])
            step = '(%f, %f)' % (step_h / finput_dimh, step_w / finput_dimw)
            assert param.offset == 0.5, "currently only support offset = 0.5"
            symbol_string += '%s = mx.contrib.symbol.MultiBoxPrior(%s, sizes=%s, ratios=%s, clip=%s, steps=%s, name="%s")\n' % \
                (name, mapping[layer.bottom[0]], sizes, ratios_string, clip, step, name)
            symbol_string += '%s = mx.symbol.Flatten(data=%s)\n' % (name, name)
            type_string = 'split'
            need_flatten[name] = False
        if layer.type == 'DetectionOutput':
            bottom_order = [1, 0, 2]
            param = layer.detection_output_param
            assert param.share_location == True
            assert param.background_label_id == 0
            nms_param = param.nms_param
            type_string = 'mx.contrib.symbol.MultiBoxDetection'
            param_string = "nms_threshold=%f, nms_topk=%d, clip=False" % \
                (nms_param.nms_threshold, nms_param.top_k)
        if skip_layer:
            assert len(layer.bottom) == 1
            symbol_string += "%s = %s\n" % (name, prev_name)
        elif type_string == '':
            raise ValueError('Unknown layer %s!' % layer.type)
        elif type_string != 'split':
            bottom = layer.bottom
            if param_string != "":
                param_string = ", " + param_string
            if len(bottom) == 1:
                # print(need_flatten)
                if need_flatten[mapping[bottom[0]]] and type_string == 'mx.symbol.FullyConnected':
                    flatten_name = "flatten_%d" % flatten_count
                    symbol_string += "%s=mx.symbol.Flatten(name='%s', data=%s)\n" % (
                        flatten_name, flatten_name, mapping[bottom[0]])
                    flatten_count += 1
                    need_flatten[flatten_name] = False
                    bottom[0] = flatten_name
                    mapping[bottom[0]] = bottom[0]
                symbol_string += "%s = %s(name='%s', data=%s %s)\n" % (
                    name, type_string, name, mapping[bottom[0]], param_string)
            else:
                if not bottom_order:
                    bottom_order = range(len(bottom))
                symbol_string += "%s = %s(name='%s', *[%s] %s)\n" % \
                                 (name, type_string, name, ','.join([mapping[bottom[x]] for x in bottom_order]), param_string)
                if layer.type == 'Concat' and layer.concat_param.axis == 2:
                    symbol_string += "%s = mx.symbol.Reshape(data=%s, shape=(0, -1, 4), name='%s')\n" %\
                        (name, name, name)
        for j in range(len(layer.top)):
            mapping[layer.top[j]] = name
        output_name = name
    return symbol_string, output_name, input_dim
327331
def Parse(self, stat, file_object, knowledge_base):
    """"""
    _, _ = stat, knowledge_base
    # TODO(user): Convert this to use the far more intelligent plaso parser.
    ff = Firefox3History(file_object)
    for timestamp, unused_entry_type, url, title in ff.Parse():
      yield rdf_webhistory.BrowserHistoryItem(
          url=url,
          domain=urlparse.urlparse(url).netloc,
          access_time=timestamp,
          program_name="Firefox",
          source_path=file_object.Path(),
          title=title)
163295
def decode_example(self, tfexample_dict):
    """"""
    tensor_dict = {}
    # Iterate over the Tensor dict keys
    for feature_key, feature in six.iteritems(self._feature_dict):
      decoded_feature = decode_single_feature_from_dict(
          feature_k=feature_key,
          feature=feature,
          tfexample_dict=tfexample_dict,
      )
      tensor_dict[feature_key] = decoded_feature
    return tensor_dict
965160
def GetPresetsInformation(cls):
    
    parser_presets_information = []
    for preset_definition in ParsersManager.GetPresets():
      preset_information_tuple = (
          preset_definition.name, ', '.join(preset_definition.parsers))
      # TODO: refactor to pass PresetDefinition.
      parser_presets_information.append(preset_information_tuple)

    return parser_presets_information
607526
def read_from_json(json_filename):
  """"""
  with tf.io.gfile.GFile(json_filename) as f:
    dataset_info_json_str = f.read()
  # Parse it back into a proto.
  parsed_proto = json_format.Parse(dataset_info_json_str,
                                   dataset_info_pb2.DatasetInfo())
  return parsed_proto
543492
def _parse_example(serialized_example):
  """"""
  data_fields = {
      "inputs": tf.VarLenFeature(tf.int64),
      "targets": tf.VarLenFeature(tf.int64)
  }
  parsed = tf.parse_single_example(serialized_example, data_fields)
  inputs = tf.sparse_tensor_to_dense(parsed["inputs"])
  targets = tf.sparse_tensor_to_dense(parsed["targets"])
  return inputs, targets
359763
def _from_string(cls, serialized):
        
        parse = cls.parse_url(serialized)

        if parse['version_guid']:
            parse['version_guid'] = cls.as_object_id(parse['version_guid'])

        return cls(**{key: parse.get(key) for key in cls.KEY_FIELDS})
865066
def from_protobuf(cls, msg):
        """"""
        if not isinstance(msg, cls._protobuf_cls):
            raise TypeError("Expected message of type "
                            "%r" % cls._protobuf_cls.__name__)
        kwargs = {k: getattr(msg, k) for k in cls._get_params()}
        return cls(**kwargs)