Helper function for memwarp of multiple input GDAL Datasets
496138
def memwarp_multi(src_ds_list, res='first', extent='intersection', t_srs='first', r='cubic', verbose=True, dst_ndv=0):
    
    return warp_multi(src_ds_list, res, extent, t_srs, r, warptype=memwarp, verbose=verbose, dst_ndv=dst_ndv)
47968
def gdaldem_mem_ds(ds, processing='hillshade', returnma=False, computeEdges=False):
    
    choices = ["hillshade", "slope", "aspect", "color-relief", "TRI", "TPI", "Roughness"]
    out = None
    scale=1.0
    if not get_ds_srs(ds).IsProjected():
        scale=111120
    if processing in choices:
        out = gdal.DEMProcessing('', ds, processing, format='MEM', computeEdges=computeEdges, scale=scale)
    else:
        print("Invalid processing choice")
        print(choices)
    #This should be a separate function
    if returnma:
        from pygeotools.lib import iolib
        out = iolib.ds_getma(out)
    return out
840826
def get_data_iters(dataset, batch_size, opt):
    """"""
    if dataset == 'mnist':
        train_data, val_data = get_mnist_iterator(batch_size, (1, 28, 28),
                                                  num_parts=kv.num_workers, part_index=kv.rank)
    elif dataset == 'cifar10':
        train_data, val_data = get_cifar10_iterator(batch_size, (3, 32, 32),
                                                    num_parts=kv.num_workers, part_index=kv.rank)
    elif dataset == 'imagenet':
        shape_dim = 299 if model_name == 'inceptionv3' else 224

        if not opt.data_dir:
            raise ValueError('Dir containing raw images in train/val is required for imagenet.'
                             'Please specify "--data-dir"')

        train_data, val_data = get_imagenet_iterator(opt.data_dir, batch_size,
                                                                opt.num_workers, shape_dim, opt.dtype)
    elif dataset == 'caltech101':
        train_data, val_data = get_caltech101_iterator(batch_size, opt.num_workers, opt.dtype)
    elif dataset == 'dummy':
        shape_dim = 299 if model_name == 'inceptionv3' else 224
        train_data, val_data = dummy_iterator(batch_size, (3, shape_dim, shape_dim))
    return train_data, val_data
189268
def pad_batch(features, batch_multiple):
  """"""
  feature = list(features.items())[0][1]
  batch_size = tf.shape(feature)[0]
  mod = batch_size % batch_multiple
  has_mod = tf.cast(tf.cast(mod, tf.bool), tf.int32)
  batch_padding = batch_multiple * has_mod - mod

  padded_features = {}
  for k, feature in features.items():
    rank = len(feature.shape)
    paddings = [[0, 0] for _ in range(rank)]
    paddings[0][1] = batch_padding
    padded_feature = tf.pad(feature, paddings)
    padded_features[k] = padded_feature
  return padded_features
957486
def export_fun(args):
  """"""
  # Input placeholder for inferencing
  x = tf.placeholder(tf.float32, [None, IMAGE_PIXELS * IMAGE_PIXELS], name="x")

  # Variables of the hidden layer
  hid_w = tf.Variable(tf.truncated_normal([IMAGE_PIXELS * IMAGE_PIXELS, hidden_units],
                      stddev=1.0 / IMAGE_PIXELS), name="hid_w")
  hid_b = tf.Variable(tf.zeros([hidden_units]), name="hid_b")
  tf.summary.histogram("hidden_weights", hid_w)

  # Variables of the softmax layer
  sm_w = tf.Variable(tf.truncated_normal([hidden_units, 10],
                     stddev=1.0 / math.sqrt(hidden_units)), name="sm_w")
  sm_b = tf.Variable(tf.zeros([10]), name="sm_b")

  hid_lin = tf.nn.xw_plus_b(x, hid_w, hid_b)
  hid = tf.nn.relu(hid_lin)
  y = tf.nn.softmax(tf.nn.xw_plus_b(hid, sm_w, sm_b))
  prediction = tf.argmax(y, 1, name="prediction")

  saver = tf.train.Saver()

  with tf.Session() as sess:
    # load graph from a checkpoint
    logging.info("model path: {}".format(args.model_dir))
    ckpt = tf.train.get_checkpoint_state(args.model_dir)
    logging.info("ckpt: {}".format(ckpt))
    assert ckpt and ckpt.model_checkpoint_path, "Invalid model checkpoint path: {}".format(args.model_dir)
    saver.restore(sess, ckpt.model_checkpoint_path)

    logging.info("Exporting saved_model to: {}".format(args.export_dir))
    # exported signatures defined in code
    signatures = {
      tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: {
        'inputs': {'image': x},
        'outputs': {'prediction': prediction},
        'method_name': tf.saved_model.signature_constants.PREDICT_METHOD_NAME
      },
      'featurize': {
        'inputs': {'image': x},
        'outputs': {'features': hid},
        'method_name': 'featurize'
      }
    }
    TFNode.export_saved_model(sess,
                              args.export_dir,
                              tf.saved_model.tag_constants.SERVING,
                              signatures)
    logging.info("Exported saved_model")
83352
def gpu_mem_restore(func):
    ""
    @functools.wraps(func)
    def wrapper(*args, **kwargs):
        tb_clear_frames = os.environ.get('FASTAI_TB_CLEAR_FRAMES', None)
        if not IS_IN_IPYTHON or tb_clear_frames=="0":
            return func(*args, **kwargs)

        try:
            return func(*args, **kwargs)
        except Exception as e:
            if ("CUDA out of memory" in str(e) or
                "device-side assert triggered" in str(e) or
                tb_clear_frames == "1"):
                type, val, tb = get_ref_free_exc_info() # must!
                gc.collect()
                if "device-side assert triggered" in str(e):
                    warn("""When 'device-side assert triggered' error happens, it's not possible to recover and you must restart the kernel to continue. Use os.environ['CUDA_LAUNCH_BLOCKING']="1" before restarting to debug""")
                raise type(val).with_traceback(tb) from None
            else: raise # re-raises the exact last exception
    return wrapper
216042
def subseparable_conv_block(inputs, filters, dilation_rates_and_kernel_sizes,
                            **kwargs):
  """"""
  return conv_block_internal(subseparable_conv, inputs, filters,
                             dilation_rates_and_kernel_sizes, **kwargs)
869159
def basic_generator(in_size:int, n_channels:int, noise_sz:int=100, n_features:int=64, n_extra_layers=0, **conv_kwargs):
    ""
    cur_size, cur_ftrs = 4, n_features//2
    while cur_size < in_size:  cur_size *= 2; cur_ftrs *= 2
    layers = [conv_layer(noise_sz, cur_ftrs, 4, 1, transpose=True, **conv_kwargs)]
    cur_size = 4
    while cur_size < in_size // 2:
        layers.append(conv_layer(cur_ftrs, cur_ftrs//2, 4, 2, 1, transpose=True, **conv_kwargs))
        cur_ftrs //= 2; cur_size *= 2
    layers += [conv_layer(cur_ftrs, cur_ftrs, 3, 1, 1, transpose=True, **conv_kwargs) for _ in range(n_extra_layers)]
    layers += [conv2d_trans(cur_ftrs, n_channels, 4, 2, 1, bias=False), nn.Tanh()]
    return nn.Sequential(*layers)
494489
def generator(ngf, nc, no_bias=True, fix_gamma=True, eps=1e-5 + 1e-12, z_dim=100, activation='sigmoid'):
    '''
    '''
    BatchNorm = mx.sym.BatchNorm
    rand = mx.sym.Variable('rand')

    rand = mx.sym.Reshape(rand, shape=(-1, z_dim, 1, 1))

    g1 = mx.sym.Deconvolution(rand, name='gen1', kernel=(5,5), stride=(2,2),target_shape=(2,2), num_filter=ngf*8, no_bias=no_bias)
    gbn1 = BatchNorm(g1, name='genbn1', fix_gamma=fix_gamma, eps=eps)
    gact1 = mx.sym.Activation(gbn1, name="genact1", act_type="relu")

    g2 = mx.sym.Deconvolution(gact1, name='gen2', kernel=(5,5), stride=(2,2),target_shape=(4,4), num_filter=ngf*4, no_bias=no_bias)
    gbn2 = BatchNorm(g2, name='genbn2', fix_gamma=fix_gamma, eps=eps)
    gact2 = mx.sym.Activation(gbn2, name='genact2', act_type='relu')

    g3 = mx.sym.Deconvolution(gact2, name='gen3', kernel=(5,5), stride=(2,2), target_shape=(8,8), num_filter=ngf*2, no_bias=no_bias)
    gbn3 = BatchNorm(g3, name='genbn3', fix_gamma=fix_gamma, eps=eps)
    gact3 = mx.sym.Activation(gbn3, name='genact3', act_type='relu')

    g4 = mx.sym.Deconvolution(gact3, name='gen4', kernel=(5,5), stride=(2,2), target_shape=(16,16), num_filter=ngf, no_bias=no_bias)
    gbn4 = BatchNorm(g4, name='genbn4', fix_gamma=fix_gamma, eps=eps)
    gact4 = mx.sym.Activation(gbn4, name='genact4', act_type='relu')

    g5 = mx.sym.Deconvolution(gact4, name='gen5', kernel=(5,5), stride=(2,2), target_shape=(32,32), num_filter=nc, no_bias=no_bias)
    gout = mx.sym.Activation(g5, name='genact5', act_type=activation)

    return gout
1075904
def gdaldem_mem_ma(ma, ds=None, res=None, extent=None, srs=None, processing='hillshade', returnma=False, computeEdges=False):
    
    if ds is None:
        ds = mem_ds(res, extent, srs=None, dtype=gdal.GDT_Float32)
    else:
        ds = mem_ds_copy(ds)
    b = ds.GetRasterBand(1)
    b.WriteArray(ma)
    out = gdaldem_mem_ds(ds, processing=processing, returnma=returnma)
    return out
306412
def diskwarp(src_ds, res=None, extent=None, t_srs=None, r='cubic', outdir=None, dst_fn=None, dst_ndv=None, verbose=True):
    
    if dst_fn is None:
        dst_fn = os.path.splitext(src_ds.GetFileList()[0])[0]+'_warp.tif'
    if outdir is not None:
        dst_fn = os.path.join(outdir, os.path.basename(dst_fn))  
    driver = iolib.gtif_drv
    dst_ds = warp(src_ds, res, extent, t_srs, r, driver, dst_fn, dst_ndv=dst_ndv, verbose=verbose)
    #Write out
    dst_ds = None
    #Now reopen ds from disk
    dst_ds = gdal.Open(dst_fn)
    return dst_ds
211451
def gdaOnes(shape, dtype, numGhosts=1):
    
    res = GhostedDistArray(shape, dtype)
    res.setNumberOfGhosts(numGhosts)
    res[:] = 1
    return res
1225092
def convolve_sep2(data, hx, hy, res_g=None, sub_blocks=None):
    

    if isinstance(data, np.ndarray):
        data = np.ascontiguousarray(data)

        if sub_blocks == (1, 1) or sub_blocks is None:
            return _convolve_sep2_numpy(data, hx, hy)
        else:
            # cut the image into tile and operate on every of them
            N_sub = [int(np.ceil(1. * n / s)) for n, s in zip(data.shape, sub_blocks)]
            Npads = [int(len(_h) / 2) for _h in [hy, hx]]
            res = np.empty(data.shape, np.float32)
            for data_tile, data_s_src, data_s_dest \
                    in tile_iterator(data, blocksize=N_sub,
                                     padsize=Npads,
                                     mode="constant"):
                res_tile = _convolve_sep2_numpy(data_tile.copy(),
                                                hx, hy)
                res[data_s_src] = res_tile[data_s_dest]
            return res
    elif isinstance(data, OCLArray):
        if not sub_blocks is None:
            raise NotImplementedError()
        return _convolve_sep2_gpu(data, hx, hy, res_g=res_g)
    else:
        raise TypeError("array argument (1) has bad type: %s" % type(data))
208639
def convolve(data, h, res_g=None, sub_blocks=None):
    

    if not len(data.shape) in [1, 2, 3]:
        raise ValueError("dim = %s not supported" % (len(data.shape)))

    if len(data.shape) != len(h.shape):
        raise ValueError("dimemnsion of data (%s) and h (%s) are different" % (len(data.shape), len(h.shape)))

    if isinstance(data, OCLArray) and isinstance(h, OCLArray):
        return _convolve_buf(data, h, res_g)
    elif isinstance(data, np.ndarray) and isinstance(h, np.ndarray):
        if sub_blocks == (1,) * len(data.shape) or sub_blocks is None:
            return _convolve_np(data, h)
        else:
            # cut the image into tile and operate on every of them
            N_sub = [int(np.ceil(1. * n / s)) for n, s in zip(data.shape, sub_blocks)]
            Npads = [int(s / 2) for s in h.shape]
            res = np.empty(data.shape, np.float32)
            for data_tile, data_s_src, data_s_dest \
                    in tile_iterator(data, blocksize=N_sub,
                                     padsize=Npads,
                                     mode="constant"):
                res_tile = _convolve_np(data_tile.copy(),
                                        h)
                res[data_s_src] = res_tile[data_s_dest]
            return res
    else:
        raise TypeError("unknown types (%s, %s)" % (type(data), type(h)))
285154
def memmap_array(self, dtype, shape, offset=0, mode='r', order='C'):
        """"""
        if not self.is_file:
            raise ValueError('Cannot memory-map file without fileno')
        return numpy.memmap(self._fh, dtype=dtype, mode=mode,
                            offset=self._offset + offset,
                            shape=shape, order=order)
1256354
def save_datasets(self, writer="geotiff", datasets=None, compute=True, **kwargs):
        """"""
        if datasets is not None:
            datasets = [self[ds] for ds in datasets]
        else:
            datasets = [self.datasets.get(ds) for ds in self.wishlist]
            datasets = [ds for ds in datasets if ds is not None]
        if not datasets:
            raise RuntimeError("None of the requested datasets have been "
                               "generated or could not be loaded. Requested "
                               "composite inputs may need to have matching "
                               "dimensions (eg. through resampling).")
        writer, save_kwargs = load_writer(writer, ppp_config_dir=self.ppp_config_dir, **kwargs)
        return writer.save_datasets(datasets, compute=compute, **save_kwargs)
722172
def getMeanInpCurrents(params, numunits=100,
                       filepattern=os.path.join('simulation_output_default',
                                                'population_input_spikes*')):
    '''
    
    '''
    #convolution kernels
    x = np.arange(100) * params.dt
    kernel = np.exp(-x / params.model_params['tau_syn_ex'])


    #number of external inputs:
    K_bg = np.array(sum(params.K_bg, []))

    #compensate for DC CC connections if we're using that
    iDC =  K_bg * params.dc_amplitude * 1E-3 # unit ????
    
    data = {}
    
    #loop over network-populations
    for i, Y in enumerate(params.Y):
        if i % SIZE == RANK:
            #file to open
            fname = glob.glob(filepattern+'*' + Y + '*')[0]
            print fname
            #read in read data and assess units, up to numunits
            rawdata = np.array(helpers.read_gdf(fname))
            units = np.unique(rawdata[:, 0])
            if numunits > units.size:
                numcells = units.size
            else:
                numcells = numunits
            units = units[:numcells]
            
            #churn through data and extract the input currents per cell
            for j, unit in enumerate(units):
                slc = rawdata[:, 0] == unit
                #just the spikes:
                if j == 0:
                    dataslc = rawdata[slc, 2:]
                else:
                    dataslc = np.r_['0,3', dataslc, rawdata[slc, 2:]]
            
            #fix the datatype, it may be object
            dataslc = dataslc.astype(float)
                   
            #fill in data-structure
            data.update({
                Y : {
                    'E' : np.convolve(dataslc[:, :, 0].mean(axis=0),
                                      kernel, 'same')*1E-3 + float(iDC[i]),
                    'I' : np.convolve(dataslc[:, :, 1].mean(axis=0),
                                      kernel, 'same')*1E-3,
                    'tvec' : rawdata[slc, 1],
                    'numunits' : numunits,
                }
            })
    
    
    data = COMM.allgather(data)
    
    return {k: v for d in data for k, v in d.items()}