Drop analysis for optimal threshold for extracting stream.
482218
def dropanalysis(np, fel, p, ad8, ssa, outlet, minthresh, maxthresh, numthresh,
                     logspace, drp, workingdir=None,
                     mpiexedir=None, exedir=None, log_file=None, runtime_file=None, hostfile=None):
        """"""
        parstr = '%f %f %f' % (minthresh, maxthresh, numthresh)
        if logspace == 'false':
            parstr += ' 1'
        else:
            parstr += ' 0'
        fname = TauDEM.func_name('dropanalysis')
        return TauDEM.run(FileClass.get_executable_fullpath(fname, exedir),
                          {'-fel': fel, '-p': p, '-ad8': ad8, '-ssa': ssa, '-o': outlet},
                          workingdir,
                          {'-par': parstr},
                          {'-drp': drp},
                          {'mpipath': mpiexedir, 'hostfile': hostfile, 'n': np},
                          {'logfile': log_file, 'runtimefile': runtime_file})
1202054
def streamReachAndWatershed(self,
                                delineate,
                                out_stream_order_grid,
                                out_network_connectivity_tree,
                                out_network_coordinates,
                                out_stream_reach_file,
                                out_watershed_grid,
                                pit_filled_elevation_grid=None,
                                flow_dir_grid=None,
                                contributing_area_grid=None,
                                stream_raster_grid=None,
                                outlet_shapefile=None
                                ):
        
        log("PROCESS: StreamReachAndWatershed")
        if pit_filled_elevation_grid:
            self.pit_filled_elevation_grid = pit_filled_elevation_grid
        if flow_dir_grid:
            self.flow_dir_grid = flow_dir_grid
        if contributing_area_grid:
            self.contributing_area_grid = contributing_area_grid
        if stream_raster_grid:
            self.stream_raster_grid = stream_raster_grid

        # Construct the taudem command line.
        cmd = [os.path.join(self.taudem_exe_path, 'streamnet'),
               '-fel', self.pit_filled_elevation_grid,
               '-p', self.flow_dir_grid,
               '-ad8', self.contributing_area_grid,
               '-src', self.stream_raster_grid,
               '-ord', out_stream_order_grid,
               '-tree', out_network_connectivity_tree,
               '-coord', out_network_coordinates,
               '-net', out_stream_reach_file,
               '-w', out_watershed_grid,
               ]

        if outlet_shapefile:
            cmd += ['-o', outlet_shapefile]
        if delineate:
            cmd += ['-sw']

        self._run_mpi_cmd(cmd)

        # create projection file
        self._add_prj_file(self.pit_filled_elevation_grid,
                           out_stream_order_grid)
        self._add_prj_file(self.pit_filled_elevation_grid,
                           out_stream_reach_file)
        self._add_prj_file(self.pit_filled_elevation_grid,
                           out_watershed_grid)
1103451
private static FixedBucketsHistogram fromBytesSparse(ByteBuffer buf)
  {
    double lowerLimit = buf.getDouble();
    double upperLimit = buf.getDouble();
    int numBuckets = buf.getInt();
    OutlierHandlingMode outlierHandlingMode = OutlierHandlingMode.values()[buf.get()];

    long count = buf.getLong();
    long lowerOutlierCount = buf.getLong();
    long upperOutlierCount = buf.getLong();
    long missingValueCount = buf.getLong();

    double max = buf.getDouble();
    double min = buf.getDouble();

    int nonEmptyBuckets = buf.getInt();
    long histogram[] = new long[numBuckets];
    for (int i = 0; i < nonEmptyBuckets; i++) {
      int bucket = buf.getInt();
      long bucketCount = buf.getLong();
      histogram[bucket] = bucketCount;
    }

    return new FixedBucketsHistogram(
        lowerLimit,
        upperLimit,
        numBuckets,
        outlierHandlingMode,
        histogram,
        count,
        max,
        min,
        lowerOutlierCount,
        upperOutlierCount,
        missingValueCount
    );
  }
529861
def dechunk(stream):
    

    # TODO(vovan): Add support for chunk extensions:
    # TODO(vovan): http://tools.ietf.org/html/rfc2616#section-3.6.1

    while True:
        chunk_len = read_until(stream, b'\r\n')

        if chunk_len is None:
            raise DechunkError(
                'Could not extract chunk size: unexpected end of data.')

        try:
            chunk_len = int(chunk_len.strip(), 16)
        except (ValueError, TypeError) as err:
            raise DechunkError('Could not parse chunk size: %s' % (err,))

        if chunk_len == 0:
            break

        bytes_to_read = chunk_len
        while bytes_to_read:
            chunk = stream.read(bytes_to_read)
            bytes_to_read -= len(chunk)
            yield chunk

        # chunk ends with \r\n
        crlf = stream.read(2)
        if crlf != b'\r\n':
            raise DechunkError('No CR+LF at the end of chunk!')
54959
def susvd(x, x_obs, rho, penalties):
    

    assert type(x) == Tensor, "Input array must be a Tensor"

    while True:

        # proximal operator for the Fro. norm
        x = squared_error(x, rho, x_obs)

        # sequential singular value thresholding
        for ix, penalty in enumerate(penalties):
            x = x.unfold(ix).svt(penalty / rho).fold()

        yield x
1177814
def _read_stream_for_size(stream, buf_size=65536):
    """"""
    size = 0
    while True:
        buf = stream.read(buf_size)
        size += len(buf)
        if not buf:
            break
    return size
29466
def reservoir_sample(stream, num_items, item_parser=lambda x: x):
    
    kept = []
    for index, item in enumerate(stream):
        if index < num_items:
            kept.append(item_parser(item))
        else:
            r = random.randint(0, index)
            if r < num_items:
                kept[r] = item_parser(item)
    return kept
1073077
def StreamIDNextDownIDToConnectivity(stream_id_array,
                                     next_down_id_array,
                                     out_csv_file):
    
    list_all = []
    max_count_upstream = 0

    for hydroid in np.sort(stream_id_array):
        # find the HydroID of the upstreams
        list_upstreamID = stream_id_array[next_down_id_array == hydroid]
        # count the total number of the upstreams
        count_upstream = len(list_upstreamID)
        if count_upstream > max_count_upstream:
            max_count_upstream = count_upstream
        nextDownID = next_down_id_array[stream_id_array == hydroid][0]
        # append the list of Stream HydroID, NextDownID, Count of Upstream ID,
        # and  HydroID of each Upstream into a larger list
        list_all.append(
            np.concatenate(
                [np.array([hydroid, nextDownID, count_upstream]),
                 list_upstreamID]
            ).astype(int))

    with open_csv(out_csv_file, 'w') as csvfile:
        connectwriter = csv_writer(csvfile)
        for row_list in list_all:
            out = np.concatenate([
                row_list,
                np.array([0 for _ in xrange(max_count_upstream - row_list[2])])
            ])
            connectwriter.writerow(out.astype(int))
457982
def _scatter_obs(
        adata,
        x=None,
        y=None,
        color=None,
        use_raw=None,
        layers='X',
        sort_order=True,
        alpha=None,
        basis=None,
        groups=None,
        components=None,
        projection='2d',
        legend_loc='right margin',
        legend_fontsize=None,
        legend_fontweight=None,
        color_map=None,
        palette=None,
        frameon=None,
        right_margin=None,
        left_margin=None,
        size=None,
        title=None,
        show=None,
        save=None,
        ax=None):
    """"""
    sanitize_anndata(adata)
    from scipy.sparse import issparse
    if use_raw is None and adata.raw is not None: use_raw = True

    # process layers
    if layers is None:
        layers = 'X'
    if isinstance(layers, str) and (layers == 'X' or layers in adata.layers.keys()):
        layers = (layers, layers, layers)
    elif isinstance(layers, (tuple, list)) and len(layers) == 3:
        for layer in layers:
            if layer not in adata.layers.keys() and layer != 'X':
                raise ValueError(
                    '`layers` should have elements that are either \'X\' or in adata.layers.keys().')
    else:
        raise ValueError('`layers` should be a string or a list/tuple of length 3.')
    if use_raw and (layers != ('X', 'X', 'X') or layers != ['X', 'X', 'X']):
        ValueError('`use_raw` must be `False` if layers other than \'X\' are used.')

    if legend_loc not in VALID_LEGENDLOCS:
        raise ValueError(
            'Invalid `legend_loc`, need to be one of: {}.'.format(VALID_LEGENDLOCS))
    if components is None: components = '1,2' if '2d' in projection else '1,2,3'
    if isinstance(components, str): components = components.split(',')
    components = np.array(components).astype(int) - 1
    keys = ['grey'] if color is None else [color] if isinstance(color, str) else color
    if title is not None and isinstance(title, str):
        title = [title]
    highlights = adata.uns['highlights'] if 'highlights' in adata.uns else []
    if basis is not None:
        try:
            # ignore the '0th' diffusion component
            if basis == 'diffmap': components += 1
            Y = adata.obsm['X_' + basis][:, components]
            # correct the component vector for use in labeling etc.
            if basis == 'diffmap': components -= 1
        except KeyError:
            raise KeyError('compute coordinates using visualization tool {} first'
                           .format(basis))
    elif x is not None and y is not None:
        x_arr = adata._get_obs_array(x, use_raw=use_raw, layer=layers[0])
        y_arr = adata._get_obs_array(y, use_raw=use_raw, layer=layers[1])

        x_arr = x_arr.toarray().flatten() if issparse(x_arr) else x_arr
        y_arr = y_arr.toarray().flatten() if issparse(y_arr) else y_arr

        Y = np.c_[x_arr[:, None], y_arr[:, None]]
    else:
        raise ValueError('Either provide a `basis` or `x` and `y`.')

    if size is None:
        n = Y.shape[0]
        size = 120000 / n

    if legend_loc.startswith('on data') and legend_fontsize is None:
        legend_fontsize = rcParams['legend.fontsize']
    elif legend_fontsize is None:
        legend_fontsize = rcParams['legend.fontsize']

    palette_was_none = False
    if palette is None: palette_was_none = True
    if isinstance(palette, list):
        if not is_color_like(palette[0]):
            palettes = palette
        else:
            palettes = [palette]
    else:
        palettes = [palette for i in range(len(keys))]
    for i, palette in enumerate(palettes):
        palettes[i] = utils.default_palette(palette)

    if basis is not None:
        component_name = (
            'DC' if basis == 'diffmap'
            else 'tSNE' if basis == 'tsne'
            else 'UMAP' if basis == 'umap'
            else 'PC' if basis == 'pca'
            else basis.replace('draw_graph_', '').upper() if 'draw_graph' in basis
            else basis)
    else:
        component_name = None
    axis_labels = (x, y) if component_name is None else None
    show_ticks = True if component_name is None else False

    # generate the colors
    color_ids = []
    categoricals = []
    colorbars = []
    for ikey, key in enumerate(keys):
        c = 'white'
        categorical = False  # by default, assume continuous or flat color
        colorbar = None
        # test whether we have categorial or continuous annotation
        if key in adata.obs_keys():
            if is_categorical_dtype(adata.obs[key]):
                categorical = True
            else:
                c = adata.obs[key]
        # coloring according to gene expression
        elif (use_raw
              and adata.raw is not None
              and key in adata.raw.var_names):
            c = adata.raw[:, key].X
        elif key in adata.var_names:
            c = adata[:, key].X if layers[2] == 'X' else adata[:, key].layers[layers[2]]
            c = c.toarray().flatten() if issparse(c) else c
        elif is_color_like(key):  # a flat color
            c = key
            colorbar = False
        else:
            raise ValueError(
                'key \'{}\' is invalid! pass valid observation annotation, '
                'one of {} or a gene name {}'
                .format(key, adata.obs_keys(), adata.var_names))
        if colorbar is None:
            colorbar = not categorical
        colorbars.append(colorbar)
        if categorical: categoricals.append(ikey)
        color_ids.append(c)

    if right_margin is None and len(categoricals) > 0:
        if legend_loc == 'right margin': right_margin = 0.5
    if title is None and keys[0] is not None:
        title = [key.replace('_', ' ') if not is_color_like(key) else '' for key in keys]

    axs = scatter_base(Y,
                       title=title,
                       alpha=alpha,
                       component_name=component_name,
                       axis_labels=axis_labels,
                       component_indexnames=components + 1,
                       projection=projection,
                       colors=color_ids,
                       highlights=highlights,
                       colorbars=colorbars,
                       right_margin=right_margin,
                       left_margin=left_margin,
                       sizes=[size for c in keys],
                       color_map=color_map,
                       show_ticks=show_ticks,
                       ax=ax)

    def add_centroid(centroids, name, Y, mask):
        Y_mask = Y[mask]
        if Y_mask.shape[0] == 0: return
        median = np.median(Y_mask, axis=0)
        i = np.argmin(np.sum(np.abs(Y_mask - median), axis=1))
        centroids[name] = Y_mask[i]

    # loop over all categorical annotation and plot it
    for i, ikey in enumerate(categoricals):
        palette = palettes[i]
        key = keys[ikey]
        utils.add_colors_for_categorical_sample_annotation(
            adata, key, palette, force_update_colors=not palette_was_none)
        # actually plot the groups
        mask_remaining = np.ones(Y.shape[0], dtype=bool)
        centroids = {}
        if groups is None:
            for iname, name in enumerate(adata.obs[key].cat.categories):
                if name not in settings.categories_to_ignore:
                    mask = scatter_group(axs[ikey], key, iname,
                                         adata, Y, projection, size=size, alpha=alpha)
                    mask_remaining[mask] = False
                    if legend_loc.startswith('on data'): add_centroid(centroids, name, Y, mask)
        else:
            groups = [groups] if isinstance(groups, str) else groups
            for name in groups:
                if name not in set(adata.obs[key].cat.categories):
                    raise ValueError('"' + name + '" is invalid!'
                                     + ' specify valid name, one of '
                                     + str(adata.obs[key].cat.categories))
                else:
                    iname = np.flatnonzero(adata.obs[key].cat.categories.values == name)[0]
                    mask = scatter_group(axs[ikey], key, iname,
                                         adata, Y, projection, size=size, alpha=alpha)
                    if legend_loc.startswith('on data'): add_centroid(centroids, name, Y, mask)
                    mask_remaining[mask] = False
        if mask_remaining.sum() > 0:
            data = [Y[mask_remaining, 0], Y[mask_remaining, 1]]
            if projection == '3d': data.append(Y[mask_remaining, 2])
            axs[ikey].scatter(*data, marker='.', c='lightgrey', s=size,
                                    edgecolors='none', zorder=-1)
        legend = None
        if legend_loc.startswith('on data'):
            if legend_fontweight is None:
                legend_fontweight = 'bold'
            for name, pos in centroids.items():
                axs[ikey].text(pos[0], pos[1], name,
                               weight=legend_fontweight,
                               verticalalignment='center',
                               horizontalalignment='center',
                               fontsize=legend_fontsize)
            all_pos = np.zeros((len(adata.obs[key].cat.categories), 2))
            for iname, name in enumerate(adata.obs[key].cat.categories):
                if name in centroids:
                    all_pos[iname] = centroids[name]
                else:
                    all_pos[iname] = [np.nan, np.nan]
            utils._tmp_cluster_pos = all_pos
            if legend_loc == 'on data export':
                filename = settings.writedir + 'pos.csv'
                logg.msg('exporting label positions to {}'.format(filename), v=1)
                if settings.writedir != '' and not os.path.exists(settings.writedir):
                    os.makedirs(settings.writedir)
                np.savetxt(filename, all_pos, delimiter=',')
        elif legend_loc == 'right margin':
            legend = axs[ikey].legend(
                frameon=False, loc='center left',
                bbox_to_anchor=(1, 0.5),
                ncol=(1 if len(adata.obs[key].cat.categories) <= 14
                      else 2 if len(adata.obs[key].cat.categories) <= 30 else 3),
                fontsize=legend_fontsize)
        elif legend_loc != 'none':
            legend = axs[ikey].legend(
                frameon=False, loc=legend_loc, fontsize=legend_fontsize)
        if legend is not None:
            for handle in legend.legendHandles: handle.set_sizes([300.0])

    # draw a frame around the scatter
    frameon = settings._frameon if frameon is None else frameon
    if not frameon and x is None and y is None:
        for ax in axs:
            ax.set_xlabel('')
            ax.set_ylabel('')
            ax.set_frame_on(False)

    utils.savefig_or_show('scatter' if basis is None else basis, show=show, save=save)
    if show == False: return axs if len(keys) > 1 else axs[0]
1285303
def query_streamer(self, index):
        """"""

        if index >= len(self.graph.streamers):
            return None

        info = self.streamer_status[index]
        highest_ack = self.streamer_acks.get(index, 0)

        return [info.last_attempt_time, info.last_success_time, info.last_error, highest_ack, info.last_status, info.attempt_number, info.comm_status]
387593
def _get_optimal_thresholds(nd_dict, quantized_dtype, num_bins=8001, num_quantized_bins=255, logger=None):
    """"""
    if stats is None:
        raise ImportError('scipy.stats is required for running entropy mode of calculating'
                          ' the optimal thresholds for quantizing FP32 ndarrays into int8.'
                          ' Please check if the scipy python bindings are installed.')
    assert isinstance(nd_dict, dict)
    if logger is not None:
        logger.info('Calculating optimal thresholds for quantization using KL divergence'
                    ' with num_bins=%d and num_quantized_bins=%d' % (num_bins, num_quantized_bins))
    th_dict = {}
    # copy nd_dict keys since the keys() only returns a view in python3
    layer_names = list(nd_dict.keys())
    for name in layer_names:
        assert name in nd_dict
        min_val, max_val, min_divergence, opt_th = \
            _get_optimal_threshold(nd_dict[name], quantized_dtype, num_bins=num_bins,
                                   num_quantized_bins=num_quantized_bins)
        del nd_dict[name]  # release the memory of ndarray
        if min_val < 0:
            th_dict[name] = (-opt_th, opt_th)
        else:
            th_dict[name] = (0, opt_th)
        if logger is not None:
            logger.info('layer=%s, min_val=%f, max_val=%f, min_divergence=%f, optimal_threshold=%f'
                        % (name, min_val, max_val, min_divergence, opt_th))
    return th_dict
816821
void validate(DeepWater dl, boolean expensive) {
    boolean classification = expensive || dl.nclasses() != 0 ? dl.isClassifier() : _distribution == DistributionFamily.bernoulli || _distribution == DistributionFamily.bernoulli;
    if (_mini_batch_size < 1)
      dl.error("_mini_batch_size", "Mini-batch size must be >= 1");

    if (_weights_column!=null && expensive) {
      Vec w = (train().vec(_weights_column));
      if (!w.isInt() || w.max() > 1 || w.min() < 0) {
        dl.error("_weights_column", "only supporting weights of 0 or 1 right now");
      }
    }

    if (_clip_gradient<=0)
      dl.error("_clip_gradient", "Clip gradient must be >= 0");

    if (_hidden != null && _network_definition_file != null && !_network_definition_file.isEmpty())
      dl.error("_hidden", "Cannot provide hidden layers and a network definition file at the same time.");

    if (_activation != null && _network_definition_file != null && !_network_definition_file.isEmpty())
      dl.error("_activation", "Cannot provide activation functions and a network definition file at the same time.");

    if (_problem_type == ProblemType.image) {
      if (_image_shape.length != 2)
        dl.error("_image_shape", "image_shape must have 2 dimensions (width, height)");
      if (_image_shape[0] < 0)
        dl.error("_image_shape", "image_shape[0] must be >=1 or automatic (0).");
      if (_image_shape[1] < 0)
        dl.error("_image_shape", "image_shape[1] must be >=1 or automatic (0).");
      if (_channels != 1 && _channels != 3)
        dl.error("_channels", "channels must be either 1 or 3.");
    } else if (_problem_type != auto) {
      dl.warn("_image_shape", "image shape is ignored, only used for image_classification");
      dl.warn("_channels", "channels shape is ignored, only used for image_classification");
      dl.warn("_mean_image_file", "mean_image_file shape is ignored, only used for image_classification");
    }
    if (_categorical_encoding==CategoricalEncodingScheme.Enum) {
      dl.error("_categorical_encoding", "categorical encoding scheme cannot be Enum: the neural network must have numeric columns as input.");
    }

    if (_autoencoder)
      dl.error("_autoencoder", "Autoencoder is not supported right now.");

    if (_network == Network.user) {
      if (_network_definition_file == null || _network_definition_file.isEmpty())
        dl.error("_network_definition_file", "network_definition_file must be provided if the network is user-specified.");
      else if (!new File(_network_definition_file).exists())
        dl.error("_network_definition_file", "network_definition_file " + _network_definition_file + " not found.");
    } else {
      if (_network_definition_file != null && !_network_definition_file.isEmpty() && _network != Network.auto)
        dl.error("_network_definition_file", "network_definition_file cannot be provided if a pre-defined network is chosen.");
    }
    if (_network_parameters_file != null && !_network_parameters_file.isEmpty()) {
      if (!DeepWaterModelInfo.paramFilesExist(_network_parameters_file)) {
        dl.error("_network_parameters_file", "network_parameters_file " + _network_parameters_file + " not found.");
      }
    }
    if (_checkpoint!=null) {
      DeepWaterModel other = (DeepWaterModel) _checkpoint.get();
      if (other == null)
        dl.error("_width", "Invalid checkpoint provided: width mismatch.");
      if (!Arrays.equals(_image_shape, other.get_params()._image_shape))
        dl.error("_width", "Invalid checkpoint provided: width mismatch.");
    }

    if (!_autoencoder) {
      if (classification) {
        dl.hide("_regression_stop", "regression_stop is used only with regression.");
      } else {
        dl.hide("_classification_stop", "classification_stop is used only with classification.");
        //          dl.hide("_max_hit_ratio_k", "max_hit_ratio_k is used only with classification.");
        //          dl.hide("_balance_classes", "balance_classes is used only with classification.");
      }
      //        if( !classification || !_balance_classes )
      //          dl.hide("_class_sampling_factors", "class_sampling_factors requires both classification and balance_classes.");
      if (!classification && _valid != null || _valid == null)
        dl.hide("_score_validation_sampling", "score_validation_sampling requires classification and a validation frame.");
    } else {
      if (_nfolds > 1) {
        dl.error("_nfolds", "N-fold cross-validation is not supported for Autoencoder.");
      }
    }
    if (H2O.CLOUD.size() == 1 && _replicate_training_data)
      dl.hide("_replicate_training_data", "replicate_training_data is only valid with cloud size greater than 1.");
    if (_single_node_mode && (H2O.CLOUD.size() == 1 || !_replicate_training_data))
      dl.hide("_single_node_mode", "single_node_mode is only used with multi-node operation with replicated training data.");
    if (H2O.ARGS.client && _single_node_mode)
      dl.error("_single_node_mode", "Cannot run on a single node in client mode");
    if (_autoencoder)
      dl.hide("_use_all_factor_levels", "use_all_factor_levels is mandatory in combination with autoencoder.");
    if (_nfolds != 0)
      dl.hide("_overwrite_with_best_model", "overwrite_with_best_model is unsupported in combination with n-fold cross-validation.");
    if (expensive) dl.checkDistributions();

    if (_score_training_samples < 0)
      dl.error("_score_training_samples", "Number of training samples for scoring must be >= 0 (0 for all).");
    if (_score_validation_samples < 0)
      dl.error("_score_validation_samples", "Number of training samples for scoring must be >= 0 (0 for all).");
    if (classification && dl.hasOffsetCol())
      dl.error("_offset_column", "Offset is only supported for regression.");

    // reason for the error message below is that validation might not have the same horizontalized features as the training data (or different order)
    if (expensive) {
      if (!classification && _balance_classes) {
        dl.error("_balance_classes", "balance_classes requires classification.");
      }
      if (_class_sampling_factors != null && !_balance_classes) {
        dl.error("_class_sampling_factors", "class_sampling_factors requires balance_classes to be enabled.");
      }
      if (_replicate_training_data && null != train() && train().byteSize() > 0.9*H2O.CLOUD.free_mem()/H2O.CLOUD.size() && H2O.CLOUD.size() > 1) {
        dl.error("_replicate_training_data", "Compressed training dataset takes more than 90% of avg. free available memory per node (" + 0.9*H2O.CLOUD.free_mem()/H2O.CLOUD.size() + "), cannot run with replicate_training_data.");
      }
    }
    if (_autoencoder && _stopping_metric != ScoreKeeper.StoppingMetric.AUTO && _stopping_metric != ScoreKeeper.StoppingMetric.MSE) {
      dl.error("_stopping_metric", "Stopping metric must either be AUTO or MSE for autoencoder.");
    }
  }
1332464
function decompressDeltas18Bit (buffer) {
  let D = new Array(k.OBCIGanglionSamplesPerPacket); // 2
  D[0] = [0, 0, 0, 0];
  D[1] = [0, 0, 0, 0];

  let receivedDeltas = [];
  for (let i = 0; i < k.OBCIGanglionSamplesPerPacket; i++) {
    receivedDeltas.push([0, 0, 0, 0]);
  }

  let miniBuf;

  // Sample 1 - Channel 1
  miniBuf = new Buffer(
    [
      (buffer[0] >> 6),
      ((buffer[0] & 0x3F) << 2) | (buffer[1] >> 6),
      ((buffer[1] & 0x3F) << 2) | (buffer[2] >> 6)
    ]
  );
  receivedDeltas[0][0] = convert18bitAsInt32(miniBuf);

  // Sample 1 - Channel 2
  miniBuf = new Buffer(
    [
      (buffer[2] & 0x3F) >> 4,
      (buffer[2] << 4) | (buffer[3] >> 4),
      (buffer[3] << 4) | (buffer[4] >> 4)
    ]);
  // miniBuf = new Buffer([(buffer[2] & 0x1F), buffer[3], buffer[4] >> 2]);
  receivedDeltas[0][1] = convert18bitAsInt32(miniBuf);

  // Sample 1 - Channel 3
  miniBuf = new Buffer(
    [
      (buffer[4] & 0x0F) >> 2,
      (buffer[4] << 6) | (buffer[5] >> 2),
      (buffer[5] << 6) | (buffer[6] >> 2)
    ]);
  receivedDeltas[0][2] = convert18bitAsInt32(miniBuf);

  // Sample 1 - Channel 4
  miniBuf = new Buffer(
    [
      (buffer[6] & 0x03),
      buffer[7],
      buffer[8]
    ]);
  receivedDeltas[0][3] = convert18bitAsInt32(miniBuf);

  // Sample 2 - Channel 1
  miniBuf = new Buffer(
    [
      (buffer[9] >> 6),
      ((buffer[9] & 0x3F) << 2) | (buffer[10] >> 6),
      ((buffer[10] & 0x3F) << 2) | (buffer[11] >> 6)
    ]);
  receivedDeltas[1][0] = convert18bitAsInt32(miniBuf);

  // Sample 2 - Channel 2
  miniBuf = new Buffer(
    [
      (buffer[11] & 0x3F) >> 4,
      (buffer[11] << 4) | (buffer[12] >> 4),
      (buffer[12] << 4) | (buffer[13] >> 4)
    ]);
  receivedDeltas[1][1] = convert18bitAsInt32(miniBuf);

  // Sample 2 - Channel 3
  miniBuf = new Buffer(
    [
      (buffer[13] & 0x0F) >> 2,
      (buffer[13] << 6) | (buffer[14] >> 2),
      (buffer[14] << 6) | (buffer[15] >> 2)
    ]);
  receivedDeltas[1][2] = convert18bitAsInt32(miniBuf);

  // Sample 2 - Channel 4
  miniBuf = new Buffer([(buffer[15] & 0x03), buffer[16], buffer[17]]);
  receivedDeltas[1][3] = convert18bitAsInt32(miniBuf);

  return receivedDeltas;
}
738565
def threshold(np, acc, stream_raster, threshold=100., workingdir=None,
                  mpiexedir=None, exedir=None, log_file=None, runtime_file=None, hostfile=None):
        """"""
        fname = TauDEM.func_name('threshold')
        return TauDEM.run(FileClass.get_executable_fullpath(fname, exedir),
                          {'-ssa': acc}, workingdir,
                          {'-thresh': threshold},
                          {'-src': stream_raster},
                          {'mpipath': mpiexedir, 'hostfile': hostfile, 'n': np},
                          {'logfile': log_file, 'runtimefile': runtime_file})
889456
function(fs, globs, opt) {
    if (!opt) opt = {};
    if (typeof opt.cwd !== 'string') opt.cwd = fs.resolve('.');
    if (typeof opt.dot !== 'boolean') opt.dot = false;
    if (typeof opt.silent !== 'boolean') opt.silent = true;
    if (typeof opt.nonull !== 'boolean') opt.nonull = false;
    if (typeof opt.cwdbase !== 'boolean') opt.cwdbase = false;
    if (opt.cwdbase) opt.base = opt.cwd;

    // only one glob no need to aggregate
    if (!Array.isArray(globs)) globs = [globs];

    var positives = [];
    var negatives = [];

    var ourOpt = extend({}, opt);
    delete ourOpt.root;
    
    globs.forEach(function(glob, index) {
      if (typeof glob !== 'string' && !(glob instanceof RegExp)) {
        throw new Error('Invalid glob at index ' + index);
      }

      var globArray = isNegative(glob) ? negatives : positives;

      // create Minimatch instances for negative glob patterns
      if (globArray === negatives && typeof glob === 'string') {
        var ourGlob = resolveGlob(glob, opt);
        glob = new Minimatch(ourGlob, ourOpt);
      }

      globArray.push({
        index: index,
        glob: glob
      });
    });

    if (positives.length === 0) throw new Error('Missing positive glob');

    // only one positive glob no need to aggregate
    if (positives.length === 1) return streamFromPositive(positives[0]);

    // create all individual streams
    var streams = positives.map(streamFromPositive);

    // then just pipe them to a single unique stream and return it
    var aggregate = new Combine(streams);
    var uniqueStream = unique('path');
    var returnStream = aggregate.pipe(uniqueStream);

    aggregate.on('error', function (err) {
      returnStream.emit('error', err);
    });

    return returnStream;

    function streamFromPositive(positive) {
      var negativeGlobs = negatives.filter(indexGreaterThan(positive.index)).map(toGlob);
      return gs.createStream(fs, positive.glob, negativeGlobs, opt);
    }
  }
252184
def _read_stream_for_size(stream):
    """"""
    size = 0
    while True:
        buf = stream.read(_BUFFER_SIZE)
        size += len(buf)
        if not buf:
            break
    return size
1045169
def extract_data(data_path, feature):
    '''
    '''
    population = nm.load_neurons(data_path)

    feature_data = [nm.get(feature, n) for n in population]
    feature_data = list(chain(*feature_data))

    return stats.optimal_distribution(feature_data)