input: N x n_dim image of of raw data under lens function, as a dataframe
    output: (undirected graph, list of node contents, dictionary of patches)
485365
def mapper_graph(df, lens_data=None, lens='pca', resolution=10, gain=0.5, equalize=True, clust='kmeans', stat='db',
                 max_K=5):
    
    if lens_data is None:
        lens_data = apply_lens(df, lens=lens)

    patch_clusterings = {}
    counter = 0
    patches = covering_patches(lens_data, resolution=resolution, gain=gain, equalize=equalize)
    for key, patch in patches.items():
        if len(patch) > 0:
            patch_clusterings[key] = optimal_clustering(df, patch, method=clust, statistic=stat, max_K=max_K)
            counter += 1
    print 'total of {} patches required clustering'.format(counter)

    all_clusters = []
    for key in patch_clusterings:
        all_clusters += patch_clusterings[key]
    num_nodes = len(all_clusters)
    print 'this implies {} nodes in the mapper graph'.format(num_nodes)

    A = np.zeros((num_nodes, num_nodes))
    for i in range(num_nodes):
        for j in range(i):
            overlap = set(all_clusters[i]).intersection(set(all_clusters[j]))
            if len(overlap) > 0:
                A[i, j] = 1
                A[j, i] = 1

    G = nx.from_numpy_matrix(A)
    total = []
    all_clusters_new = []
    mapping = {}
    cont = 0
    for m in all_clusters:
        total += m
    for n, m in enumerate(all_clusters):
        if len(m) == 1 and total.count(m) > 1:
            G.remove_node(n)
        else:
            all_clusters_new.append(m)
            mapping[n] = cont
            cont += 1
    H = nx.relabel_nodes(G, mapping)
    return H, all_clusters_new, patches
366614
def reconstruct(self, D=None, X=None):
        """"""

        if D is None:
            D = self.getdict(crop=False)
        if X is None:
            X = self.getcoef()
        Df = sl.rfftn(D, self.xstep.cri.Nv, self.xstep.cri.axisN)
        Xf = sl.rfftn(X, self.xstep.cri.Nv, self.xstep.cri.axisN)
        DXf = sl.inner(Df, Xf, axis=self.xstep.cri.axisM)
        return sl.irfftn(DXf, self.xstep.cri.Nv, self.xstep.cri.axisN)
581367
def data_nodes(self):
        

        return {k: v for k, v in self.nodes.items() if v['type'] == 'data'}
259705
def reconstruct(self, D=None):
        """"""

        if D is None:
            Df = self.Xf
        else:
            Df = sl.rfftn(D, None, self.cri.axisN)

        Sf = np.sum(self.Zf * Df, axis=self.cri.axisM)
        return sl.irfftn(Sf, self.cri.Nv, self.cri.axisN)
983401
def slice_hidden(self, x):
    
    x_sliced = tf.reshape(
        x, shape=[-1, self.hparams.num_blocks, self.hparams.block_dim])
    return x_sliced
142475
def pick_slices(img, num_slices_per_view):
    

    slices = list()
    for view in range(len(img.shape)):
        dim_size = img.shape[view]
        non_empty_slices = np.array(
            [sl for sl in range(dim_size) if np.count_nonzero(get_axis(img, view, sl)) > 0])
        num_non_empty = len(non_empty_slices)

        # trying to 5% slices at the tails (bottom clipping at 0)
        skip_count = max(0, np.around(num_non_empty * 0.05).astype('int16'))
        # only when possible
        if skip_count > 0 and (num_non_empty - 2 * skip_count >= num_slices_per_view):
            non_empty_slices = non_empty_slices[skip_count: -skip_count]
            num_non_empty = len(non_empty_slices)

        # sampling non-empty slices only
        sampled_indices = np.linspace(0, num_non_empty, num=min(num_non_empty, num_slices_per_view),
                                      endpoint=False)
        slices_in_dim = non_empty_slices[np.around(sampled_indices).astype('int64')]

        # ensure you do not overshoot
        slices_in_dim = [sn for sn in slices_in_dim if sn >= 0 or sn <= num_non_empty]

        slices.append(slices_in_dim)

    return slices
1372357
def convert_reshape(node, **kwargs):
    
    name, input_nodes, attrs = get_inputs(node, kwargs)

    output_shape_list = convert_string_to_list(attrs["shape"])

    initializer = kwargs["initializer"]
    output_shape_np = np.array(output_shape_list, dtype='int64')
    data_type = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[output_shape_np.dtype]
    dims = np.shape(output_shape_np)

    output_shape_name = "reshape_attr_tensor" + str(kwargs["idx"])
    tensor_node = onnx.helper.make_tensor_value_info(output_shape_name, data_type, dims)

    initializer.append(
        onnx.helper.make_tensor(
            name=output_shape_name,
            data_type=data_type,
            dims=dims,
            vals=output_shape_list,
            raw=False,
        )
    )

    input_nodes.append(output_shape_name)

    not_supported_shape = [-2, -3, -4]

    for val in output_shape_list:
        if val in not_supported_shape:
            raise AttributeError("Reshape: Shape value not supported in ONNX", val)

    reshape_node = onnx.helper.make_node(
        "Reshape",
        input_nodes,
        [name],
        name=name
    )

    return [tensor_node, reshape_node]
178823
def dataset(directory, images_file, labels_file):
  """"""

  images_file = download(directory, images_file)
  labels_file = download(directory, labels_file)

  check_image_file_header(images_file)
  check_labels_file_header(labels_file)

  def decode_image(image):
    # Normalize from [0, 255] to [0.0, 1.0]
    image = tf.decode_raw(image, tf.uint8)
    image = tf.cast(image, tf.float32)
    image = tf.reshape(image, [784])
    return image / 255.0

  def decode_label(label):
    label = tf.decode_raw(label, tf.uint8)  # tf.string -> [tf.uint8]
    label = tf.reshape(label, [])  # label is a scalar
    return tf.to_int32(label)

  images = tf.data.FixedLengthRecordDataset(
      images_file, 28 * 28, header_bytes=16).map(decode_image)
  labels = tf.data.FixedLengthRecordDataset(
      labels_file, 1, header_bytes=8).map(decode_label)
  return tf.data.Dataset.zip((images, labels))
1070057
def _all_dims(x, default_dims=None):
  """"""
  if x.get_shape().ndims is not None:
    return list(xrange(x.get_shape().ndims))
  else:
    return default_dims
1278560
def numpyview(arr, datatype, shape, raw=False):
    

    if raw:
        return n.frombuffer(arr, dtype=n.dtype(datatype)).view(n.dtype(datatype)).reshape(shape)   # for shared mps.RawArray
    else:
        return n.frombuffer(arr.get_obj(), dtype=n.dtype(datatype)).view(n.dtype(datatype)).reshape(shape)
266312
def observed_data_to_xarray(self):
        """"""
        observed_data_raw = _read_data(self.observed_data)
        variables = self.observed_data_var
        if isinstance(variables, str):
            variables = [variables]
        observed_data = {}
        for key, vals in observed_data_raw.items():
            if variables is not None and key not in variables:
                continue
            vals = np.atleast_1d(vals)
            val_dims = self.dims.get(key)
            val_dims, coords = generate_dims_coords(
                vals.shape, key, dims=val_dims, coords=self.coords
            )
            observed_data[key] = xr.DataArray(vals, dims=val_dims, coords=coords)
        return xr.Dataset(data_vars=observed_data)
626086
def __dfsify_branch_uv(u, v, dfs_data):
    """"""
    buv = B(u, v, dfs_data)
    new_list = []
    for edge_id in buv:
        edge = dfs_data['graph'].get_edge(edge_id)
        j, k = edge['vertices']
        d_x = D(j, dfs_data)
        d_y = D(k, dfs_data)
        if d_x < d_y:
            smaller = d_x
            larger = d_y
        else:
            smaller = d_y
            larger = d_x
        frond = (smaller, larger)
        new_list.append(frond)
    return new_list
772945
def pretty_view(graph, oriented=False, construction=False,
                graphviz_prog=DEFAULT_GRAPHVIZ_PROG):
    """"""
    layout = create_layout(graph, graphviz_prog=graphviz_prog)
    matrix_view = defaultdict(lambda: ' ')

    # Add the edge to the view

    # print('GRAPH EDGES:', tuple(graph.edges()))
    # print('LAYOUT:', layout)

    edges = ((layout[source], layout[target])
             for source, target in graph.edges())
    # print('EDGES:', tuple(edges))

    for source, target in edges:
        previous_edge_char = None
        previous_position = source
        while source != target:
            for is_dir, transform, edge_char in DIRECTIONS:
                first_loop = previous_edge_char is None
                if is_dir(source, target):
                    previous_source = source
                    source = transform(source)
                    if not first_loop:  # first loop: no previous char
                        char = CHARACTER[previous_edge_char, edge_char]
                        if not isinstance(char, str):
                            char = char.value
                        matrix_view[previous_source] = char
                        if construction:
                            old = defaultdict(lambda: ' ', matrix_view)
                            yield view.build(matrix_view)
                        assert isinstance(matrix_view[previous_source], str)
                    if source != target:
                        previous_edge_char = edge_char
                    previous_position = previous_source
                    break  # for loop ; don't test the remain directions

        if oriented:
            matrix_view[previous_position] = ORIENTATION[previous_edge_char, edge_char]
        if construction:
            yield view.build(matrix_view)

    # mark the place where nodes labels will be added
    # for node, coords in layout.items():
        # matrix_view[coords] = node[0]
    # Add the node labels to the view
    # matrix_view = view.clean(matrix_view)
    for node, (x, y) in layout.items():
        if len(node) == 1:
            matrix_view[x, y] = node
            continue
        row_min, row_max = (view.previous_unwrittable_on_row(matrix_view, (x, y)),
                            view.next_unwrittable_on_row(matrix_view, (x, y)))
        col_min, col_max = (view.previous_unwrittable_on_col(matrix_view, (x, y)),
                              view.next_unwrittable_on_col(matrix_view, (x, y)))
        # print('NODE ' + node + ':',
              # '→row: [{};{}]'.format(row_min, row_max).ljust(20),
              # '↓col: [{};{}]'.format(col_min, col_max))
        print_coords = [itertools.count(x), itertools.cycle((y,))]
        if row_min is None:  # write left to right, end at (x, y)
            if row_max is None or row_max > (x + len(node) / 2):  # enough space at the right
                factor = 2
            else:
                factor = 1
            print_coords[0] = tuple(
                x - (len(node) // factor) + offset + 1
                for offset in range(len(node))
            )
            # print('DEBUG 1:', y, len(node), print_coords[0])
        elif row_max is None:  # write left to right, beginning at (x, y)
            if row_min < (x - len(node) / 2):  # enough space at the left
                factor = 1
            else:
                factor = 0
            print_coords[0] = tuple(
                x + offset - (len(node) // 2) * factor
                for offset in range(len(node))
            )
            # print('DEBUG 2:', print_coords[0])
        elif (row_max - row_min) > len(node) + 1:  # write left to right, if enough place
            print_coords[0] = tuple(
                x + offset
                for offset in range(len(node))
            )
            # print('DEBUG 3:', print_coords[0])

        elif col_min is None:  # write up to down, end at (x, y)
            if col_max is None or col_max > (x + len(node) / 2):  # enough space at the right
                factor = 2
            else:
                factor = 1
            print_coords = (itertools.cycle((x,)), tuple(
                y - (len(node) // factor) + offset + 1
                for offset in range(len(node))
            ))
            # print('DEBUG 4:', y, len(node), print_coords[1])
        elif col_max is None:  # write up to down, beginning at (x, y)
            if col_min < (x - len(node) / 2):  # enough space at the left
                factor = 1
            else:
                factor = 0
            print_coords = (itertools.cycle((x,)), tuple(
                y + offset - (len(node) // 2) * factor
                for offset in range(len(node))
            ))
            # print('DEBUG 5:', print_coords[1])
        elif (col_max - col_min) > len(node) + 1:  # write up to down, if enough place
            print_coords = (itertools.cycle((x,)), tuple(
                y + offset
                for offset in range(len(node))
            ))
            # print('DEBUG 6:', print_coords[1])
        else:  # not enough space
            if (row_max - row_min) > (col_max - col_min):
                # more space on Y axis
                node = node[:row_max - row_min]  # cut the node
                print_coords = (itertools.cycle((x,)), tuple(
                    x + offset
                    for offset in range(len(node))
                ))
                # print('DEBUG 7:', print_coords[1])
            else:
                # more space on X axis
                node = node[:col_max - col_min]  # cut the node
                print_coords[0] = tuple(
                    x + offset
                    for offset in range(len(node))
                )
                # print('DEBUG 8:', print_coords[0])

        for letter, i, j in zip(node, *print_coords):
            matrix_view[i, j] = letter




    if construction:
        yield view.build(matrix_view)
    else:
        yield from view.build(matrix_view)
276161
def preprocess(self, raw_inputs):
        
        image_arrays = []
        for raw_im in raw_inputs:
            im = raw_im.resize(VGG16_DIM[:2], Image.ANTIALIAS)
            im = im.convert('RGB')
            arr = np.array(im).astype('float32')
            image_arrays.append(arr)

        all_raw_inputs = np.array(image_arrays)
        return imagenet_utils.preprocess_input(all_raw_inputs)
299108
def get_wireframe(viewer, x, y, z, **kwargs):
    
    # TODO: something like this would make a great utility function
    # for ginga
    n, m = x.shape
    objs = []
    for i in range(n):
        pts = np.asarray([(x[i][j], y[i][j], z[i][j])
                          for j in range(m)])
        objs.append(viewer.dc.Path(pts, **kwargs))

    for j in range(m):
        pts = np.asarray([(x[i][j], y[i][j], z[i][j])
                          for i in range(n)])
        objs.append(viewer.dc.Path(pts, **kwargs))

    return viewer.dc.CompoundObject(*objs)
246021
def build_graph(self, data_paths, batch_size, graph_mod):
    """"""
    tensors = GraphReferences()
    is_training = graph_mod == GraphMod.TRAIN
    if data_paths:
      _, tensors.examples = _util.read_examples(
          data_paths,
          batch_size,
          shuffle=is_training,
          num_epochs=None if is_training else 2)
    else:
      tensors.examples = tf.placeholder(tf.string, name='input', shape=(None,))

    if graph_mod == GraphMod.PREDICT:
      inception_input, inception_embeddings = self.build_inception_graph()
      # Build the Inception graph. We later add final training layers
      # to this graph. This is currently used only for prediction.
      # For training, we use pre-processed data, so it is not needed.
      embeddings = inception_embeddings
      tensors.input_jpeg = inception_input
    else:
      # For training and evaluation we assume data is preprocessed, so the
      # inputs are tf-examples.
      # Generate placeholders for examples.
      with tf.name_scope('inputs'):
        feature_map = {
            'image_uri':
                tf.FixedLenFeature(
                    shape=[], dtype=tf.string, default_value=['']),
            # Some images may have no labels. For those, we assume a default
            # label. So the number of labels is label_count+1 for the default
            # label.
            'label':
                tf.FixedLenFeature(
                    shape=[1], dtype=tf.int64,
                    default_value=[len(self.labels)]),
            'embedding':
                tf.FixedLenFeature(
                    shape=[BOTTLENECK_TENSOR_SIZE], dtype=tf.float32)
        }
        parsed = tf.parse_example(tensors.examples, features=feature_map)
        labels = tf.squeeze(parsed['label'])
        uris = tf.squeeze(parsed['image_uri'])
        embeddings = parsed['embedding']

    # We assume a default label, so the total number of labels is equal to
    # label_count+1.
    all_labels_count = len(self.labels) + 1
    with tf.name_scope('final_ops'):
      softmax, logits = self.add_final_training_ops(
          embeddings,
          all_labels_count,
          BOTTLENECK_TENSOR_SIZE,
          dropout_keep_prob=self.dropout if is_training else None)

    # Prediction is the index of the label with the highest score. We are
    # interested only in the top score.
    prediction = tf.argmax(softmax, 1)
    tensors.predictions = [prediction, softmax, embeddings]

    if graph_mod == GraphMod.PREDICT:
      return tensors

    with tf.name_scope('evaluate'):
      loss_value = loss(logits, labels)

    # Add to the Graph the Ops that calculate and apply gradients.
    if is_training:
      tensors.train, tensors.global_step = training(loss_value)
    else:
      tensors.global_step = tf.Variable(0, name='global_step', trainable=False)
      tensors.uris = uris

    # Add means across all batches.
    loss_updates, loss_op = _util.loss(loss_value)
    accuracy_updates, accuracy_op = _util.accuracy(logits, labels)

    if not is_training:
      tf.summary.scalar('accuracy', accuracy_op)
      tf.summary.scalar('loss', loss_op)

    tensors.metric_updates = loss_updates + accuracy_updates
    tensors.metric_values = [loss_op, accuracy_op]
    return tensors
309935
def create_background_map(data, bsx, bsy):
    """"""
    sx, sy = data.shape
    mx = sx // bsx
    my = sy // bsy
    comp = []
    rms = []
    # Rows
    sp = numpy.split(data, numpy.arange(bsx, sx, bsx), axis=0)
    for s in sp:
        # Columns
        rp = numpy.split(s, numpy.arange(bsy, sy, bsy), axis=1)
        for r in rp:
            b, r = background_estimator(r)
            comp.append(b)
            rms.append(r)

    # Reconstructed image
    z = numpy.array(comp)
    z.shape = (mx, my)
    # median filter
    ndfilter.median_filter(z, size=(3, 3), output=z)

    # Interpolate to the original size
    new = _interpolation(z, sx, sy, mx, my)

    # Interpolate the rms
    z = numpy.array(rms)
    z.shape = (mx, my)
    nrms = _interpolation(z, sx, sy, mx, my)

    return new, nrms