Convert tfp data into an InferenceData object.
332847
def from_tfp(
    posterior=None,
    *,
    var_names=None,
    model_fn=None,
    feed_dict=None,
    posterior_predictive_samples=100,
    posterior_predictive_size=1,
    observed=None,
    coords=None,
    dims=None
):
    """"""
    return TfpConverter(
        posterior=posterior,
        var_names=var_names,
        model_fn=model_fn,
        feed_dict=feed_dict,
        posterior_predictive_samples=posterior_predictive_samples,
        posterior_predictive_size=posterior_predictive_size,
        observed=observed,
        coords=coords,
        dims=dims,
    ).to_inference_data()
1206957
def get_data_feed(self, train_mode=True, qname_in='input', qname_out='output', input_mapping=None):
    """"""
    return TFNode.DataFeed(self.mgr, train_mode, qname_in, qname_out, input_mapping)
894642
def _fromdata(self, code, dtype, count, value, name=None):
        """"""
        self.code = int(code)
        self.name = name if name else str(code)
        self.dtype = TIFF_DATA_TYPES[dtype]
        self.count = int(count)
        self.value = value
1081049
def igetattr(self, name, context=None):
        """"""
        if not context:
            context = contextmod.InferenceContext()
        try:
            # avoid recursively inferring the same attr on the same class
            if context.push((self._proxied, name)):
                raise exceptions.InferenceError(
                    message="Cannot infer the same attribute again",
                    node=self,
                    context=context,
                )

            # XXX frame should be self._proxied, or not ?
            get_attr = self.getattr(name, context, lookupclass=False)
            yield from _infer_stmts(
                self._wrap_attr(get_attr, context), context, frame=self
            )
        except exceptions.AttributeInferenceError as error:
            try:
                # fallback to class.igetattr since it has some logic to handle
                # descriptors
                # But only if the _proxied is the Class.
                if self._proxied.__class__.__name__ != "ClassDef":
                    raise exceptions.InferenceError(**vars(error)) from error
                attrs = self._proxied.igetattr(name, context, class_context=False)
                yield from self._wrap_attr(attrs, context)
            except exceptions.AttributeInferenceError as error:
                raise exceptions.InferenceError(**vars(error)) from error
112809
def wrap_inference_results(inference_result_proto):
  
  inference_proto = inference_pb2.InferenceResult()
  if isinstance(inference_result_proto,
                classification_pb2.ClassificationResponse):
    inference_proto.classification_result.CopyFrom(
        inference_result_proto.result)
  elif isinstance(inference_result_proto, regression_pb2.RegressionResponse):
    inference_proto.regression_result.CopyFrom(inference_result_proto.result)
  return inference_proto
61553
def compile_data(tmp_dir, datasets, filename, datatypes_to_clean=None):
  """"""
  datatypes_to_clean = datatypes_to_clean or []
  filename = os.path.join(tmp_dir, filename)
  lang1_fname = filename + ".lang1"
  lang2_fname = filename + ".lang2"
  if tf.gfile.Exists(lang1_fname) and tf.gfile.Exists(lang2_fname):
    tf.logging.info("Skipping compile data, found files:\n%s\n%s", lang1_fname,
                    lang2_fname)
    return filename
  with tf.gfile.GFile(lang1_fname, mode="w") as lang1_resfile:
    with tf.gfile.GFile(lang2_fname, mode="w") as lang2_resfile:
      for dataset in datasets:
        url = dataset[0]
        compressed_filename = os.path.basename(url)
        compressed_filepath = os.path.join(tmp_dir, compressed_filename)
        if url.startswith("http"):
          generator_utils.maybe_download(tmp_dir, compressed_filename, url)

        if dataset[1][0] == "tmx":
          cleaning_requested = "tmx" in datatypes_to_clean
          tmx_filename = os.path.join(tmp_dir, dataset[1][1])
          if tmx_filename.endswith(".gz"):
            with gzip.open(tmx_filename, "rb") as tmx_file:
              _tmx_to_source_target(tmx_file, lang1_resfile, lang2_resfile,
                                    do_cleaning=cleaning_requested)
          else:
            with tf.gfile.Open(tmx_filename) as tmx_file:
              _tmx_to_source_target(tmx_file, lang1_resfile, lang2_resfile,
                                    do_cleaning=cleaning_requested)

        elif dataset[1][0] == "tsv":
          _, src_column, trg_column, glob_pattern = dataset[1]
          filenames = tf.gfile.Glob(os.path.join(tmp_dir, glob_pattern))
          if not filenames:
            # Capture *.tgz and *.tar.gz too.
            mode = "r:gz" if compressed_filepath.endswith("gz") else "r"
            with tarfile.open(compressed_filepath, mode) as corpus_tar:
              corpus_tar.extractall(tmp_dir)
            filenames = tf.gfile.Glob(os.path.join(tmp_dir, glob_pattern))
          for tsv_filename in filenames:
            if tsv_filename.endswith(".gz"):
              new_filename = tsv_filename.strip(".gz")
              generator_utils.gunzip_file(tsv_filename, new_filename)
              tsv_filename = new_filename
            with tf.gfile.Open(tsv_filename) as tsv_file:
              for line in tsv_file:
                if line and "\t" in line:
                  parts = line.split("\t")
                  source, target = parts[src_column], parts[trg_column]
                  source, target = source.strip(), target.strip()
                  clean_pairs = [(source, target)]
                  if "tsv" in datatypes_to_clean:
                    clean_pairs = cleaner_en_xx.clean_en_xx_pairs(clean_pairs)
                  for source, target in clean_pairs:
                    if source and target:
                      lang1_resfile.write(source)
                      lang1_resfile.write("\n")
                      lang2_resfile.write(target)
                      lang2_resfile.write("\n")

        else:
          lang1_filename, lang2_filename = dataset[1]
          lang1_filepath = os.path.join(tmp_dir, lang1_filename)
          lang2_filepath = os.path.join(tmp_dir, lang2_filename)
          is_sgm = (
              lang1_filename.endswith("sgm") and lang2_filename.endswith("sgm"))

          if not (tf.gfile.Exists(lang1_filepath) and
                  tf.gfile.Exists(lang2_filepath)):
            # For .tar.gz and .tgz files, we read compressed.
            mode = "r:gz" if compressed_filepath.endswith("gz") else "r"
            with tarfile.open(compressed_filepath, mode) as corpus_tar:
              corpus_tar.extractall(tmp_dir)
          if lang1_filepath.endswith(".gz"):
            new_filepath = lang1_filepath.strip(".gz")
            generator_utils.gunzip_file(lang1_filepath, new_filepath)
            lang1_filepath = new_filepath
          if lang2_filepath.endswith(".gz"):
            new_filepath = lang2_filepath.strip(".gz")
            generator_utils.gunzip_file(lang2_filepath, new_filepath)
            lang2_filepath = new_filepath

          for example in text_problems.text2text_txt_iterator(
              lang1_filepath, lang2_filepath):
            line1res = _preprocess_sgm(example["inputs"], is_sgm)
            line2res = _preprocess_sgm(example["targets"], is_sgm)
            clean_pairs = [(line1res, line2res)]
            if "txt" in datatypes_to_clean:
              clean_pairs = cleaner_en_xx.clean_en_xx_pairs(clean_pairs)
            for line1res, line2res in clean_pairs:
              if line1res and line2res:
                lang1_resfile.write(line1res)
                lang1_resfile.write("\n")
                lang2_resfile.write(line2res)
                lang2_resfile.write("\n")

  return filename
656751
def _infer_map(node, context):
    """"""
    values = {}
    for name, value in node.items:
        if isinstance(name, nodes.DictUnpack):
            double_starred = helpers.safe_infer(value, context)
            if not double_starred:
                raise exceptions.InferenceError
            if not isinstance(double_starred, nodes.Dict):
                raise exceptions.InferenceError(node=node, context=context)
            unpack_items = _infer_map(double_starred, context)
            values = _update_with_replacement(values, unpack_items)
        else:
            key = helpers.safe_infer(name, context=context)
            value = helpers.safe_infer(value, context=context)
            if any(not elem for elem in (key, value)):
                raise exceptions.InferenceError(node=node, context=context)
            values = _update_with_replacement(values, {key: value})
    return values
327626
def to_table_data(self):
        

        self._validate_source_data()
        self._loader.inc_table_count()

        yield TableData(
            self._make_table_name(),
            ["key", "value"],
            [record for record in self._buffer.items()],
            dp_extractor=self._loader.dp_extractor,
            type_hints=self._extract_type_hints(),
        )
341088
private Data[] convertToData(T[] items) {
        if (items == null || items.length == 0) {
            return new Data[0];
        }
        if (items[0] instanceof Data) {
            return (Data[]) items;
        }
        final Data[] ret = new Data[items.length];
        for (int i = 0; i < items.length; i++) {
            ret[i] = convertToData(items[i]);
        }
        return ret;
    }
1102030
def text_to_data(self, text, elt, ps):
        '''
        '''
        v = text
        if v == 'false': 
            if self.pyclass is None:
                return False
            return self.pyclass(False)

        if v == 'true': 
            if self.pyclass is None:
                return True
            return self.pyclass(True)

        try:
            v = int(v)
        except:
            try:
                v = long(v)
            except:
                raise EvaluateException('Unparseable boolean', 
                        ps.Backtrace(elt))

        if v:
            if self.pyclass is None:
                return True
            return self.pyclass(True)

        if self.pyclass is None:
             return False
        return self.pyclass(False)
461514
def partition_inference(tctc_mat, comp, tau, sigma, kappa):
     
    """
    communityinfo = {}
    communityinfo['community'] = []
    communityinfo['start'] = np.empty(0)
    communityinfo['end'] = np.empty(0)
    communityinfo['size'] = np.empty(0)
    for i, tcomp in enumerate(comp):
        # This can go in parallel loop
        if len(tcomp) > 0:
            for traj in tcomp:
                # Check it does not already exist.
                ignore = 0
                preexisting = 0
                if i != 0:
                    cutoff = i-1-kappa
                    if cutoff < 0:
                        cutoff = 0
                    
                    if np.any(np.sum(np.sum(tctc_mat[traj, :, cutoff:i][:, traj], axis=0), axis=0) == np.power(len(traj), 2)):
                        # Make sure that a small trajectory could exist 
                        for checknode in np.where(communityinfo['end']>=cutoff)[0]:
                            if traj == communityinfo['community'][checknode]:
                                ignore = 1
                        if ignore == 0: 
                            for checknode in np.where(communityinfo['end']>=cutoff)[0]:
                                if set(communityinfo['community'][checknode]).issuperset(traj):
                                    preexisting = 1                              
                if ignore == 0:
                    # Check how long it continues
                    # For efficiency, increase in blocks
                    approxmaxlength = tau*2
                    a = np.sum(
                        np.sum(tctc_mat[traj, :, i:i+approxmaxlength][:, traj], axis=0), axis=0)
                    if len(traj)*len(traj)*approxmaxlength == a.sum():
                        ok = 0
                        ii = 1
                        while ok == 0:
                            b = np.sum(np.sum(
                                tctc_mat[traj, :, i+(approxmaxlength*ii):i+(approxmaxlength*(ii+1))][:, traj], axis=0), axis=0)
                            a = np.append(a, b)
                            if len(traj)*len(traj)*approxmaxlength != b.sum():
                                ok = 1
                            if i+(approxmaxlength*(ii+1)) > tctc_mat.shape[-1]:
                                ok = 1
                            ii += 1
                    a = np.where(a == np.power(len(traj), 2))[0]
                    # Add an additional value that is false in case end of time series
                    if len(a) == 1:
                        stopind = i + 1
                    else:
                        a = np.append(a, a.max()+kappa+2)
                        # Find the stop index (if stopind = 4 and start = 0, then tctc_mat[:,:,start:stopind]==1)
                        stopind = i + np.split(a, np.where(
                                np.diff(a) > kappa+1)[0]+1)[0][-1] + 1
                    # Add trajectory to dictionary
                    if ((stopind - i) >= tau or preexisting == 1) and len(traj) >= sigma:
                        communityinfo['community'].append(sorted(traj))
                        communityinfo['start'] = np.append(communityinfo['start'], int(i))
                        communityinfo['end'] = np.append(
                            communityinfo['end'], int(stopind))
                        communityinfo['size'] = np.append(communityinfo['size'], len(traj))

    communityinfo = pd.DataFrame(communityinfo)

    communityinfo['start'] = communityinfo['start'].astype(int)
    communityinfo['end'] = communityinfo['end'].astype(int)
    # First check that there is not already a trajectory that is ongoing
    badrows = []
    for v in communityinfo.iterrows():
        skipselrule = (communityinfo['end'] == v[1]['end'])
        for u in communityinfo[skipselrule].iterrows():
            a = 1
            if u[1]['start'] > v[1]['start'] and sorted(u[1]['community']) == sorted(v[1]['community']):
                badrows.append(u[0])
    communityinfo = communityinfo.drop(badrows)

    # Then see if any subset trajectory can be placed earlier in time.
    for v in communityinfo.iterrows():
        skipselrule = (communityinfo['end'] <= v[1]['start']) & (
            communityinfo['end']+kappa >= v[1]['start'])
        for u in communityinfo[skipselrule].iterrows():
            a = 1
            if set(u[1]['community']).issuperset(v[1]['community']):
                communityinfo.loc[v[0], 'start'] = u[1]['start']

    # It is possible to make the condition below effective_length
    communityinfo['length'] = np.array(communityinfo['end']) - np.array(communityinfo['start'])
    communityinfo = communityinfo[communityinfo['length'] >= tau]
    communityinfo = communityinfo[communityinfo['size'] >= sigma]

    # Make sure that the traj is not completely enguled by another
    badrows = []
    if kappa > 0:
        for v in communityinfo.iterrows():
            skipselrule = (communityinfo['end'] == v[1]['end']) & (
                communityinfo['start'] < v[1]['start'])
            for u in communityinfo[skipselrule].iterrows():
                if set(v[1]['community']).issubset(u[1]['community']):
                    badrows.append(v[0])
        communityinfo = communityinfo.drop(badrows)

    return communityinfo
818036
def run_inference_on_image(image):
  
  if not tf.gfile.Exists(image):
    tf.logging.fatal('File does not exist %s', image)
  image_data = tf.gfile.FastGFile(image, 'rb').read()

  # Creates graph from saved GraphDef.
  create_graph()

  with tf.Session() as sess:
    # Some useful tensors:
    # 'softmax:0': A tensor containing the normalized prediction across
    #   1000 labels.
    # 'pool_3:0': A tensor containing the next-to-last layer containing 2048
    #   float description of the image.
    # 'DecodeJpeg/contents:0': A tensor containing a string providing JPEG
    #   encoding of the image.
    # Runs the softmax tensor by feeding the image_data as input to the graph.
    softmax_tensor = sess.graph.get_tensor_by_name('softmax:0')
    predictions = sess.run(softmax_tensor,
                           {'DecodeJpeg/contents:0': image_data})
    predictions = np.squeeze(predictions)

    # Creates node ID --> English string lookup.
    node_lookup = NodeLookup()

    top_k = predictions.argsort()[-FLAGS.num_top_predictions:][::-1]
    for node_id in top_k:
      human_string = node_lookup.id_to_string(node_id)
      score = predictions[node_id]
      print('%s (score = %.5f)' % (human_string, score))
549392
public void inferenceAll(List data) {
	System.out.println("Starting inference ...");

	long start, stop, elapsed;
	start = System.currentTimeMillis();
	
	for (int i = 0; i < data.size(); i++) {
	    System.out.println("sequence " + Integer.toString(i + 1));
	    List seq = (List)data.get(i);
	    inference(seq);	    
	}
	
	stop = System.currentTimeMillis();
	elapsed = stop - start;
	
	System.out.println("Inference " + Integer.toString(data.size()) + " sequences completed!");
	System.out.println("Inference time: " + Double.toString((double)elapsed / 1000) + " seconds");
    }
808222
def _load_data(path, labels_number=1):
  """"""
  with tf.io.gfile.GFile(path, "rb") as f:
    data = f.read()
  offset = 0
  max_offset = len(data) - 1
  while offset < max_offset:
    labels = np.frombuffer(data, dtype=np.uint8, count=labels_number,
                           offset=offset).reshape((labels_number,))
    # 1 byte per label, 1024 * 3 = 3072 bytes for the image.
    offset += labels_number
    img = (np.frombuffer(data, dtype=np.uint8, count=3072, offset=offset)
           .reshape((3, _CIFAR_IMAGE_SIZE, _CIFAR_IMAGE_SIZE))
           .transpose((1, 2, 0))
          )
    offset += 3072
    yield labels, img
1107496
private Object convertToMaps(TabularData pTd, Stack<String> pExtraArgs, ObjectToJsonConverter pConverter) throws AttributeNotFoundException {
        JSONObject ret = new JSONObject();
        TabularType type = pTd.getTabularType();
        List<String> indexNames = type.getIndexNames();

        boolean found = false;
        for (CompositeData cd : (Collection<CompositeData>) pTd.values()) {
            Stack<String> path = (Stack<String>) pExtraArgs.clone();
            try {
                JSONObject targetJSONObject = ret;
                // TODO: Check whether all keys can be represented as simple types. If not, well
                // we dont do any magic and return the tabular data as an array.
                for (int i = 0; i < indexNames.size() - 1; i++) {
                    Object indexValue = pConverter.extractObject(cd.get(indexNames.get(i)), null, true);
                    targetJSONObject = getNextMap(targetJSONObject, indexValue);
                }
                Object row = pConverter.extractObject(cd, path, true);
                String finalIndex = indexNames.get(indexNames.size() - 1);
                Object finalIndexValue = pConverter.extractObject(cd.get(finalIndex), null, true);
                targetJSONObject.put(finalIndexValue, row);
                found = true;
            } catch (ValueFaultHandler.AttributeFilteredException exp) {
                // Ignoring filtered attributes
            }
        }
        if (!pTd.isEmpty() && !found) {
            throw new ValueFaultHandler.AttributeFilteredException();
        }
        return ret;
    }
539457
def make_dataset_from_selfplay(data_extracts):
    
    tf_examples = (make_tf_example(features_lib.extract_features(pos), pi, result)
                   for pos, pi, result in data_extracts)
    return tf_examples
305879
def posterior_to_xarray(self):
        """"""
        data = {}
        for i, var_name in enumerate(self.var_names):
            data[var_name] = np.expand_dims(self.posterior[i], axis=0)
        return dict_to_dataset(data, library=self.tfp, coords=self.coords, dims=self.dims)