Open dataset using dask and use pattern fields to set new columns
495559
def _open_dataset(self, urlpath):
        
        import dask.dataframe

        if self.pattern is None:
            self._dataframe = dask.dataframe.read_csv(
                urlpath, storage_options=self._storage_options,
                **self._csv_kwargs)
            return

        if not (DASK_VERSION >= '0.19.0'):
            raise ValueError("Your version of dask is '{}'. "
                "The ability to include filenames in read_csv output "
                "(``include_path_column``) was added in 0.19.0, so "
                "pattern urlpaths are not supported.".format(DASK_VERSION))

        drop_path_column = 'include_path_column' not in self._csv_kwargs
        path_column = self._path_column()

        self._dataframe = dask.dataframe.read_csv(
            urlpath, storage_options=self._storage_options, **self._csv_kwargs)

        # add the new columns to the dataframe
        self._set_pattern_columns(path_column)

        if drop_path_column:
            self._dataframe = self._dataframe.drop([path_column], axis=1)
452501
function (newDatasetMetadataNames,
                          currentDatasetMetadataNames, heatMap, callback) {
    var tool = {};
    tool.execute = function (options) {
      return options.input;
    };
    tool.toString = function () {
      return 'Select Fields';
    };
    tool.gui = function () {
      var items = [
        {
          name: 'current_dataset_annotation_name',
          options: currentDatasetMetadataNames,
          type: 'select',
          value: 'id',
          required: true
        }];
      items.push({
        name: 'new_dataset_annotation_name',
        type: 'select',
        value: 'id',
        options: newDatasetMetadataNames,
        required: true
      });
      return items;
    };
    morpheus.HeatMap.showTool(tool, heatMap, callback);
  }
810822
def load_ds_ids_from_config(self):
        """"""
        ids = []
        for dataset in self.datasets.values():
            # xarray doesn't like concatenating attributes that are lists
            # https://github.com/pydata/xarray/issues/2060
            if 'coordinates' in dataset and \
                    isinstance(dataset['coordinates'], list):
                dataset['coordinates'] = tuple(dataset['coordinates'])
            # Build each permutation/product of the dataset
            id_kwargs = []
            for key in DATASET_KEYS:
                val = dataset.get(key)
                if key in ["wavelength", "modifiers"] and isinstance(val,
                                                                     list):
                    # special case: wavelength can be [min, nominal, max]
                    # but is still considered 1 option
                    # it also needs to be a tuple so it can be used in
                    # a dictionary key (DatasetID)
                    id_kwargs.append((tuple(val), ))
                elif key == "modifiers" and val is None:
                    # empty modifiers means no modifiers applied
                    id_kwargs.append((tuple(), ))
                elif isinstance(val, (list, tuple, set)):
                    # this key has multiple choices
                    # (ex. 250 meter, 500 meter, 1000 meter resolutions)
                    id_kwargs.append(val)
                elif isinstance(val, dict):
                    id_kwargs.append(val.keys())
                else:
                    # this key only has one choice so make it a one
                    # item iterable
                    id_kwargs.append((val, ))
            for id_params in itertools.product(*id_kwargs):
                dsid = DatasetID(*id_params)
                ids.append(dsid)

                # create dataset infos specifically for this permutation
                ds_info = dataset.copy()
                for key in DATASET_KEYS:
                    if isinstance(ds_info.get(key), dict):
                        ds_info.update(ds_info[key][getattr(dsid, key)])
                    # this is important for wavelength which was converted
                    # to a tuple
                    ds_info[key] = getattr(dsid, key)
                self.ids[dsid] = ds_info

        return ids
774034
def main(net, df=None):
  '''
  
  '''
  if df is None:
    df = net.export_df()

  # rows
  #############
  rows = df.index.tolist()
  if type(rows[0]) is str:

    if len(rows) != len(list(set(rows))):
      new_rows = add_index_list(rows)
      df.index = new_rows

  elif type(rows[0]) is tuple:

    row_names = []
    for inst_row in rows:
      row_names.append(inst_row[0])

    if len(row_names) != len(list(set(row_names))):
      row_names = add_index_list(row_names)

      # add back to tuple
      new_rows = []
      for inst_index in range(len(rows)):
        inst_row = rows[inst_index]
        new_row = list(inst_row)
        new_row[0] = row_names[inst_index]
        new_row = tuple(new_row)
        new_rows.append(new_row)

      df.index = new_rows

  # cols
  #############
  cols = df.columns.tolist()
  if type(cols[0]) is str:

    # list column names
    if len(cols) != len(list(set(cols))):
      new_cols = add_index_list(cols)
      df.columns = new_cols

  elif type(cols[0]) is tuple:

    col_names = []
    for inst_col in cols:
      col_names.append(inst_col[0])

    if len(col_names) != len(list(set(col_names))):
      col_names = add_index_list(col_names)

      # add back to tuple
      new_cols = []
      for inst_index in range(len(cols)):
        inst_col = cols[inst_index]
        new_col = list(inst_col)
        new_col[0] = col_names[inst_index]
        new_col = tuple(new_col)
        new_cols.append(new_col)

      df.columns = new_cols

  # return dataframe with unique names
  return df
1120733
def loadInputPatterns(self, filename, cols = None, everyNrows = 1,
                          delim = ' ', checkEven = 1):
        
        self.loadInputPatternsFromFile(filename, cols, everyNrows,
                                       delim, checkEven)
1273187
def _dataset_concat(datasets, dim, data_vars, coords, compat, positions):
    
    from .dataset import Dataset

    if compat not in ['equals', 'identical']:
        raise ValueError("compat=%r invalid: must be 'equals' "
                         "or 'identical'" % compat)

    dim, coord = _calc_concat_dim_coord(dim)
    # Make sure we're working on a copy (we'll be loading variables)
    datasets = [ds.copy() for ds in datasets]
    datasets = align(*datasets, join='outer', copy=False, exclude=[dim])

    concat_over, equals = _calc_concat_over(datasets, dim, data_vars, coords)

    def insert_result_variable(k, v):
        assert isinstance(v, Variable)
        if k in datasets[0].coords:
            result_coord_names.add(k)
        result_vars[k] = v

    # create the new dataset and add constant variables
    result_vars = OrderedDict()
    result_coord_names = set(datasets[0].coords)
    result_attrs = datasets[0].attrs
    result_encoding = datasets[0].encoding

    for k, v in datasets[0].variables.items():
        if k not in concat_over:
            insert_result_variable(k, v)

    # check that global attributes and non-concatenated variables are fixed
    # across all datasets
    for ds in datasets[1:]:
        if (compat == 'identical' and
                not utils.dict_equiv(ds.attrs, result_attrs)):
            raise ValueError('dataset global attributes not equal')
        for k, v in ds.variables.items():
            if k not in result_vars and k not in concat_over:
                raise ValueError('encountered unexpected variable %r' % k)
            elif (k in result_coord_names) != (k in ds.coords):
                raise ValueError('%r is a coordinate in some datasets but not '
                                 'others' % k)
            elif k in result_vars and k != dim:
                # Don't use Variable.identical as it internally invokes
                # Variable.equals, and we may already know the answer
                if compat == 'identical' and not utils.dict_equiv(
                        v.attrs, result_vars[k].attrs):
                    raise ValueError(
                        'variable %s not identical across datasets' % k)

                # Proceed with equals()
                try:
                    # May be populated when using the "different" method
                    is_equal = equals[k]
                except KeyError:
                    result_vars[k].load()
                    is_equal = v.equals(result_vars[k])
                if not is_equal:
                    raise ValueError(
                        'variable %s not equal across datasets' % k)

    # we've already verified everything is consistent; now, calculate
    # shared dimension sizes so we can expand the necessary variables
    dim_lengths = [ds.dims.get(dim, 1) for ds in datasets]
    non_concat_dims = {}
    for ds in datasets:
        non_concat_dims.update(ds.dims)
    non_concat_dims.pop(dim, None)

    def ensure_common_dims(vars):
        # ensure each variable with the given name shares the same
        # dimensions and the same shape for all of them except along the
        # concat dimension
        common_dims = tuple(pd.unique([d for v in vars for d in v.dims]))
        if dim not in common_dims:
            common_dims = (dim,) + common_dims
        for var, dim_len in zip(vars, dim_lengths):
            if var.dims != common_dims:
                common_shape = tuple(non_concat_dims.get(d, dim_len)
                                     for d in common_dims)
                var = var.set_dims(common_dims, common_shape)
            yield var

    # stack up each variable to fill-out the dataset (in order)
    for k in datasets[0].variables:
        if k in concat_over:
            vars = ensure_common_dims([ds.variables[k] for ds in datasets])
            combined = concat_vars(vars, dim, positions)
            insert_result_variable(k, combined)

    result = Dataset(result_vars, attrs=result_attrs)
    result = result.set_coords(result_coord_names)
    result.encoding = result_encoding

    if coord is not None:
        # add concat dimension last to ensure that its in the final Dataset
        result[coord.name] = coord

    return result
34671
def concatenate(cls, datasets, datatype=None, new_type=None):
        
        from . import Dataset, default_datatype
        new_type = new_type or Dataset
        if isinstance(datasets, NdMapping):
            dimensions = datasets.kdims
            keys, datasets = zip(*datasets.data.items())
        elif isinstance(datasets, list) and all(not isinstance(v, tuple) for v in datasets):
            # Allow concatenating list of datasets (by declaring no dimensions and keys)
            dimensions, keys = [], [()]*len(datasets)
        else:
            raise DataError('Concatenation only supported for NdMappings '
                            'and lists of Datasets, found %s.' % type(datasets).__name__)

        template = datasets[0]
        datatype = datatype or template.interface.datatype

        # Handle non-general datatypes by casting to general type
        if datatype == 'array':
            datatype = default_datatype
        elif datatype == 'image':
            datatype = 'grid'

        if len(datasets) > 1 and not dimensions and cls.interfaces[datatype].gridded:
            raise DataError('Datasets with %s datatype cannot be concatenated '
                            'without defining the dimensions to concatenate along. '
                            'Ensure you pass in a NdMapping (e.g. a HoloMap) '
                            'of Dataset types, not a list.' % datatype)

        datasets = template.interface.cast(datasets, datatype)
        template = datasets[0]
        data = list(zip(keys, datasets)) if keys else datasets
        concat_data = template.interface.concat(data, dimensions, vdims=template.vdims)
        return template.clone(concat_data, kdims=dimensions+template.kdims, new_type=new_type)
1169707
def _open_xarray_dataset(self, val, chunks=CHUNK_SIZE):
        """"""
        dask_arr = from_sds(val, chunks=chunks)
        attrs = val.attributes()
        return xr.DataArray(dask_arr, dims=('y', 'x'),
                            attrs=attrs)
1020213
function buildValueColumns(
  item,
  loadedDimensions,
  columnCombinations,
  dataSets
) {
  var thisColumnOptions = clone(item._columnOptions);
  thisColumnOptions.tableStructure = item._tableStructure;
  var columns = [];
  var hasNoConcepts = item._concepts.length === 0;
  var regionDimension = getDimensionById(
    loadedDimensions,
    item.regionDimensionId
  );
  var regionDimensionIndex = getDimensionIndexById(
    loadedDimensions,
    item.regionDimensionId
  );
  var timePeriodDimension = getDimensionById(
    loadedDimensions,
    item.timePeriodDimensionId
  );
  var timePeriodDimensionIndex = getDimensionIndexById(
    loadedDimensions,
    item.timePeriodDimensionId
  );

  var regionCount = defined(regionDimension)
    ? regionDimension.values.length
    : 1;
  var timePeriodCount = defined(timePeriodDimension)
    ? timePeriodDimension.values.length
    : 1;

  for (
    var combinationIndex = 0;
    combinationIndex < columnCombinations.ids.length;
    combinationIndex++
  ) {
    var ids = columnCombinations.ids[combinationIndex];
    var dimensionIndices = mapDimensionValueIdsToKeyValues(
      item,
      loadedDimensions,
      ids
    );
    // The name is just the joined names of all the columns involved, or 'value' if no columns still have names.
    var combinationName =
      columnCombinations.names[combinationIndex]
        .filter(function(name) {
          return !!name;
        })
        .join(" ") || "Value";
    var combinationId = ids.join(" ") || "Value";
    var values = [];
    for (
      var timePeriodIndex = 0;
      timePeriodIndex < timePeriodCount;
      timePeriodIndex++
    ) {
      for (var regionIndex = 0; regionIndex < regionCount; regionIndex++) {
        if (defined(regionDimensionIndex)) {
          dimensionIndices[regionDimensionIndex] =
            regionDimension.values[regionIndex].id;
        }
        if (defined(timePeriodDimensionIndex)) {
          dimensionIndices[timePeriodDimensionIndex] =
            timePeriodDimension.values[timePeriodIndex].id;
        }
        values.push(
          getObsValue(item, loadedDimensions, dimensionIndices, dataSets)
        );
      }
    }
    thisColumnOptions.id = combinationId; // So we can refer to the dimension in a template by a sequence of ids or names.
    var column = new TableColumn(combinationName, values, thisColumnOptions);
    if (hasNoConcepts) {
      // If there are no concepts displayed to the user, there is only one value column, and we won't add a "total" column.
      // So make this column active.
      column.isActive = true;
    }
    columns.push(column);
  }
  return columns;
}
10054
def open_datasets(path, backend_kwargs={}, no_warn=False, **kwargs):
    # type: (str, T.Dict[str, T.Any], bool, T.Any) -> T.List[xr.Dataset]
    
    if not no_warn:
        warnings.warn("open_datasets is an experimental API, DO NOT RELY ON IT!", FutureWarning)

    fbks = []
    datasets = []
    try:
        datasets.append(open_dataset(path, backend_kwargs=backend_kwargs, **kwargs))
    except DatasetBuildError as ex:
        fbks.extend(ex.args[2])
    # NOTE: the recursive call needs to stay out of the exception handler to avoid showing
    #   to the user a confusing error message due to exception chaining
    for fbk in fbks:
        bks = backend_kwargs.copy()
        bks['filter_by_keys'] = fbk
        datasets.extend(open_datasets(path, backend_kwargs=bks, no_warn=True, **kwargs))
    return datasets
627822
function makeDataset(arr, colstats) {

    var labelix = parseInt($("#labelix").val());
    if(labelix < 0) labelix = D + labelix; // -1 should turn to D-1

    var data = [];
    var labels = [];
    for(var i=0;i<N;i++) {
      var arri = arr[i];
      
      // create the input datapoint Vol()
      var p = arri.slice(0, D-1);
      var xarr = [];
      for(var j=0;j<D;j++) {
        if(j===labelix) continue; // skip!

        if(colstats[j].numeric) {
          xarr.push(parseFloat(arri[j]));
        } else {
          var u = colstats[j].uniques;
          var ix = u.indexOf(arri[j]); // turn into 1ofk encoding
          for(var q=0;q<u.length;q++) {
            if(q === ix) { xarr.push(1.0); }
            else { xarr.push(0.0); }
          }
        }
      }
      var x = new convnetjs.Vol(xarr);
      
      // process the label (last column)
      if(colstats[labelix].numeric) {
        var L = parseFloat(arri[labelix]); // regression
      } else {
        var L = colstats[labelix].uniques.indexOf(arri[labelix]); // classification
        if(L==-1) {
          console.log('whoa label not found! CRITICAL ERROR, very fishy.');
        }
      }
      data.push(x);
      labels.push(L);
    }
    
    var dataset = {};
    dataset.data = data;
    dataset.labels = labels;
    return dataset;
  }
105678
def dataset_prepare(self):
        ''''''

        # Initialize the prepare subcommand's argparser
        parser = argparse.ArgumentParser(description='Preprocess a raw dialogue corpus into a dsrt dataset')
        self.init_dataset_prepare_args(parser)

        # Parse the args we got
        args = parser.parse_args(sys.argv[3:])
        args.config = ConfigurationLoader(args.config).load().data_config

        print(CLI_DIVIDER  + '\n')
        Preprocessor(**vars(args)).run()
618141
def _hstack(self, Xs):
        
        types = set(type(X) for X in Xs)

        if self.sparse_output_:
            return sparse.hstack(Xs).tocsr()
        elif dd.Series in types or dd.DataFrame in types:
            with warnings.catch_warnings():
                warnings.filterwarnings("ignore", "Concatenating", UserWarning)
                return dd.concat(Xs, axis="columns")
        elif da.Array in types:
            # To allow compatibility with dask core 1.0.0, this is the `else`
            # part of the definition of the dask.array.hstack inlined.
            # The `then` branch is removed because _validate_output in
            # sklearn.compose.ColumnTransformer ensures ndim == 2, so the
            # check `all(x.ndim == 1 for x in Xs)` should always fail.
            #
            # Once dask.array.hstack supports allow_unknown_chunksizes,
            # changed this to da.hstack(Xs, allow_unknown_chunksizes=True)
            return da.concatenate(Xs, axis=1, allow_unknown_chunksizes=True)
        elif self.preserve_dataframe and (pd.Series in types or pd.DataFrame in types):
            return pd.concat(Xs, axis="columns")
        else:
            return np.hstack(Xs)
162352
def create_preprocessed_dataset(path_to_data, outputpath, preprocessing_queue):
    """"""
    # Log everything
    logging.info("Data soure %s", path_to_data)
    logging.info("Output will be stored in %s", outputpath)
    tmp = "Preprocessing Queue:\n"
    for preprocessing_class in preprocessing_queue:
        tmp += str(preprocessing_class) + "\n"
    logging.info(tmp)
    # Load from pickled file
    if not os.path.isfile(path_to_data):
        logging.info(("'%s' does not exist. Please either abort this script "
                      "or update the data location."), path_to_data)
        raw_dataset_path = utils.choose_raw_dataset()
        # Get project-relative path
        raw_dataset_path = "raw-datasets" + \
                           raw_dataset_path.split("raw-datasets")[1]
        print(raw_dataset_path)
        sys.exit()  # TODO: Update model!
    logging.info("Start loading data...")
    loaded = pickle.load(open(path_to_data, "rb"))
    raw_datasets = loaded['handwriting_datasets']
    logging.info("Start applying preprocessing methods")
    start_time = time.time()
    for i, raw_dataset in enumerate(raw_datasets):
        if i % 10 == 0 and i > 0:
            utils.print_status(len(raw_datasets), i, start_time)
        # Do the work
        raw_dataset['handwriting'].preprocessing(preprocessing_queue)
    sys.stdout.write("\r%0.2f%% (done)\033[K\n" % (100))
    print("")
    pickle.dump({'handwriting_datasets': raw_datasets,
                 'formula_id2latex': loaded['formula_id2latex'],
                 'preprocessing_queue': preprocessing_queue},
                open(outputpath, "wb"),
                2)
190025
function () {
      var rowCounts = new Uint32Array(nrows);
      var columnCounts = new Uint32Array(ncols);
      for (var i = 0; i < nrows; i++) {
        var obj = matrix[i];
        if (obj.values != null) {
          // trim to size
          var values = new Uint16Array(obj.length);
          var indices = new Uint32Array(obj.length);
          values.set(obj.values.slice(0, obj.length));
          indices.set(obj.indices.slice(0, obj.length));
          matrix[i] = {values: values, indices: indices};
          rowCounts[i] = values.length;
          for (var j = 0; j < obj.length; j++) {
            columnCounts[indices[j]] += 1;
          }
        }
      }
      dataset = new morpheus.Dataset({
        rows: nrows, columns: ncols, name: morpheus.Util.getBaseFileName(morpheus.Util
          .getFileName(fileOrUrl)), array: matrix, dataType: dataType, defaultValue: 0
      });

      dataset.getRowMetadata().add('count>0').array = rowCounts;
      dataset.getColumnMetadata().add('count>0').array = columnCounts;
    }
1107285
@Override
    public DataSet reshape(int rows, int cols) {
        DataSet ret = new DataSet(getFeatures().reshape(new long[] {rows, cols}), getLabels());
        return ret;
    }
596459
def load_adjusted_array(self, domain, columns, dates, sids, mask):
        
        if len(columns) != 1:
            raise ValueError(
                "Can't load multiple columns with DataFrameLoader"
            )

        column = columns[0]
        self._validate_input_column(column)

        date_indexer = self.dates.get_indexer(dates)
        assets_indexer = self.assets.get_indexer(sids)

        # Boolean arrays with True on matched entries
        good_dates = (date_indexer != -1)
        good_assets = (assets_indexer != -1)

        data = self.baseline[ix_(date_indexer, assets_indexer)]
        mask = (good_assets & as_column(good_dates)) & mask

        # Mask out requested columns/rows that didn't match.
        data[~mask] = column.missing_value

        return {
            column: AdjustedArray(
                # Pull out requested columns/rows from our baseline data.
                data=data,
                adjustments=self.format_adjustments(dates, sids),
                missing_value=column.missing_value,
            ),
        }