Generate the columns and the whole dataset.
977653
def columns_dataset(self):
        
        data = {}
        words_total = {}

        for instance, words in self.raw_dataset.items():
            words_item_total = {}
            for word in words:
                words_total.setdefault(word, 0)
                words_item_total.setdefault(word, 0)
                words_total[word] += 1
                words_item_total[word] += 1
            data[instance] = words_item_total

        columns = sorted(words_total.keys(),
                         key=lambda w: words_total[w],
                         reverse=True)[:250]
        columns = sorted(columns)
        dataset = {}
        for instance in data.keys():
            dataset[instance] = [data[instance].get(word, 0)
                                 for word in columns]
        return columns, dataset
190025
function () {
      var rowCounts = new Uint32Array(nrows);
      var columnCounts = new Uint32Array(ncols);
      for (var i = 0; i < nrows; i++) {
        var obj = matrix[i];
        if (obj.values != null) {
          // trim to size
          var values = new Uint16Array(obj.length);
          var indices = new Uint32Array(obj.length);
          values.set(obj.values.slice(0, obj.length));
          indices.set(obj.indices.slice(0, obj.length));
          matrix[i] = {values: values, indices: indices};
          rowCounts[i] = values.length;
          for (var j = 0; j < obj.length; j++) {
            columnCounts[indices[j]] += 1;
          }
        }
      }
      dataset = new morpheus.Dataset({
        rows: nrows, columns: ncols, name: morpheus.Util.getBaseFileName(morpheus.Util
          .getFileName(fileOrUrl)), array: matrix, dataType: dataType, defaultValue: 0
      });

      dataset.getRowMetadata().add('count>0').array = rowCounts;
      dataset.getColumnMetadata().add('count>0').array = columnCounts;
    }
326999
def generate_data(self, data_dir, tmp_dir, task_id=-1):
    """"""
    filepath_fns = {
        problem.DatasetSplit.TRAIN: self.training_filepaths,
        problem.DatasetSplit.EVAL: self.dev_filepaths,
        problem.DatasetSplit.TEST: self.test_filepaths,
    }

    # We set shuffled=True as we don't want to shuffle on disk later.
    split_paths = [(split["split"], filepath_fns[split["split"]](
        data_dir, split["shards"], shuffled=True))
                   for split in self.dataset_splits]
    all_paths = []
    for _, paths in split_paths:
      all_paths.extend(paths)

    if self.is_generate_per_split:
      for split, paths in split_paths:
        generator_utils.generate_files(
            self.generate_encoded_samples(data_dir, tmp_dir, split),
            paths,
            cycle_every_n=self.total_number_of_frames // len(paths))
    else:
      generator_utils.generate_files(
          self.generate_encoded_samples(data_dir, tmp_dir,
                                        problem.DatasetSplit.TRAIN),
          all_paths,
          cycle_every_n=self.total_number_of_frames // len(all_paths))
795951
def dump_to_store(dataset, store, writer=None, encoder=None,
                  encoding=None, unlimited_dims=None):
    """"""
    if writer is None:
        writer = ArrayWriter()

    if encoding is None:
        encoding = {}

    variables, attrs = conventions.encode_dataset_coordinates(dataset)

    check_encoding = set()
    for k, enc in encoding.items():
        # no need to shallow copy the variable again; that already happened
        # in encode_dataset_coordinates
        variables[k].encoding = enc
        check_encoding.add(k)

    if encoder:
        variables, attrs = encoder(variables, attrs)

    store.store(variables, attrs, check_encoding, writer,
                unlimited_dims=unlimited_dims)
989082
def _make_data(data) -> Tuple[List[Dict], List[Dict]]:
        """"""
        jsdata = []
        for idx, row in data.iterrows():
            row.index = row.index.astype(str)
            rdict = row.to_dict()
            rdict.update(dict(key=str(idx)))
            jsdata.append(rdict)

        return jsdata, Table._make_columns(data.columns)
1154967
def load(self, dataset_keys, previous_datasets=None):
        """"""
        all_datasets = previous_datasets or DatasetDict()
        datasets = DatasetDict()

        # Include coordinates in the list of datasets to load
        dsids = [self.get_dataset_key(ds_key) for ds_key in dataset_keys]
        coordinates = self._get_coordinates_for_dataset_keys(dsids)
        all_dsids = list(set().union(*coordinates.values())) + dsids

        for dsid in all_dsids:
            if dsid in all_datasets:
                continue
            coords = [all_datasets.get(cid, None)
                      for cid in coordinates.get(dsid, [])]
            ds = self._load_dataset_with_area(dsid, coords)
            if ds is not None:
                all_datasets[dsid] = ds
                if dsid in dsids:
                    datasets[dsid] = ds
        self._load_ancillary_variables(all_datasets)

        return datasets
852039
def calculate_columns(self):
        """"""

        self.cols = int(math.ceil(len(self.elements) / float(self.rows)))
1167262
def dataset(self, ref, load_all=False, exception=True):
        """"""
        return self.database.dataset(ref, load_all=load_all, exception=exception)
648249
def build_dataset(filename, max_lines=-1):
    """"""
    encodings = dict(list(map(reversed, enumerate(string.printable))))
    digitize = lambda char: encodings[char] if char in encodings else len(encodings)
    encode_line = lambda line: np.array(list(map(digitize, line)))
    nonblank_line = lambda line: len(line) > 2

    with open(filename) as f:
        lines = f.readlines()

    encoded_lines = list(map(encode_line, list(filter(nonblank_line, lines))[:max_lines]))
    num_outputs = len(encodings) + 1

    return encoded_lines, num_outputs
308724
def _with_columns(self, columns):
        """"""
        table = type(self)()
        for label, column in zip(self.labels, columns):
            self._add_column_and_format(table, label, column)
        return table
281916
def to_dataframe(self):
        
        keys = self.data[0].keys()
        column_list =[]
        for k in keys:
            key_list = []
            for i in xrange(0,len(self.data)):
                key_list.append(self.data[i][k])
            column_list.append(key_list)
        df = DataFrame(np.asarray(column_list).transpose(), columns=keys)
        for i in xrange(0,df.shape[1]):
            if is_number(df.iloc[:,i]):
                df.iloc[:,i] = df.iloc[:,i].astype(float)
        return df
81986
def columns(self, model=None):
        
        column = self.column(model=model)
        if column:
            yield column

        check = self.__value
        if not isinstance(check, (list, set, tuple)):
            check = (check,)

        for val in check:
            if isinstance(val, (Query, QueryCompound)):
                for col in val.columns(model):
                    yield col
448590
def generate_field_spec(row):
    
    names = set()
    fields = []
    for cell in row:
        name = column_alias(cell, names)
        field = {
            'name': name,
            'title': cell.column,
            'type': unicode(cell.type).lower(),
            'has_nulls': False,
            'has_empty': False,
            'samples': []
        }
        if hasattr(cell.type, 'format'):
            field['type'] = 'date'
            field['format'] = cell.type.format
        fields.append(field)
    return fields
161360
def transform(self, df):
        """"""

        # Make sure the dataframe columns are the same as the ones passed to fit
        assert df.columns.equals(self.columns_)

        # Convert the dataframe with get_dummies
        df_with_dummies = pd.get_dummies(df)

        # Make sure our columns exactly match what we stored during the fit operation
        my_columns = set(df_with_dummies.columns.tolist())
        output_column_set = set(self.columns_in_order)
        for column in my_columns.difference(output_column_set):
            print('Deleting extra column {:s}'.format(column))
            del df_with_dummies[column]
        for column in output_column_set.difference(my_columns):
            print('Adding column {:s}'.format(column))
            df_with_dummies[column] = 0

        # Returns the same columns in the same order every time
        return np.asarray(df_with_dummies[self.columns_in_order])
1325996
def gen_csv(sc, filename, field_list, source, filters):
    '''
    '''

    # First thing we need to do is initialize the csvfile and build the header
    # for the file.
    datafile = open(filename, 'wb')
    csvfile = csv.writer(datafile)
    header = []
    for field in field_list:
        header.append(fields.fields[field]['name'])
    csvfile.writerow(header)

    debug.write('Generating %s: ' % filename)
    # Next we will run the Security Center query.  because this could be a
    # potentially very large dataset that is returned, we don't want to run out
    # of memory.  To get around this, we will pass the query function the writer
    # function with the appropriate fields so that it is parsed inline.
    fparams = {'fobj': csvfile, 'flist': field_list}
    sc.query('vulndetails', source=source, 
             func=writer, func_params=fparams, **filters)
    debug.write('\n')

    # Lastly we need to close the datafile.
    datafile.close()
1113087
private void createJobPropsForDatasets() {
    final Set<Dataset> datasetsWithProps = Sets.newHashSet();
    for (Dataset dataset : this.datasets) {
      datasetsWithProps.addAll(createJobPropsForDataset(dataset));
    }

    this.datasets.clear();
    this.datasets.addAll(datasetsWithProps);
  }
67058
def generate_dataset(number_items=1000):
    
    data = []
    names = get_names()
    totalnames = len(names)
    #init random seeder
    random.seed()
    #calculate items
#    names = random.sample(names, number_items)
    for i in range(number_items):
        data.append({"name":names[random.randint(0,totalnames-1)],
                     "age":random.randint(1,100),
                     "description":li_words(50, False)})
    return data