creates example data set 2
972300
def make_inst2():
    """"""
    I,d = multidict({1:45, 2:20, 3:30 , 4:30}) # demand
    J,M = multidict({1:35, 2:50, 3:40})        # capacity
    c = {(1,1):8,    (1,2):9,    (1,3):14  ,   # {(customer,factory) : cost<float>}
         (2,1):6,    (2,2):12,   (2,3):9   ,
         (3,1):10,   (3,2):13,   (3,3):16  ,
         (4,1):9,    (4,2):7,    (4,3):5   ,
         }
    return I,J,c,d,M
1017446
public SimpleDataSet generateData(int samples)
    {
        int totalClasses = 1;
        for(int d : dimensions)
            totalClasses *= d;
        catDataInfo = new CategoricalData[] { new CategoricalData(totalClasses) } ;
        List<DataPoint> dataPoints = new ArrayList<DataPoint>(totalClasses*samples);
        int[] curClassPointer = new int[1];
                
        for(int i = 0; i < dimensions[0]; i++)
        {
            int[] curDim = new int[dimensions.length];
            curDim[0] = i;
            addSamples(curClassPointer, 0, samples, dataPoints, curDim);
        }
        
        return new SimpleDataSet(dataPoints);
    }
1191594
private File saveExampleSet(ExampleSet outputSet, String outputDir) throws IOException
	{
		logger.info("");

		//
		//File tmp = File.createTempFile("train", null);
		//tmp.deleteOnExit();
		File tmp = new File(outputDir + "train");

		BufferedWriter out = new BufferedWriter(new FileWriter(tmp));
		outputSet.write(out);
		out.close();

		// add the example set to the model
		//model.add(tmp, "train");

		return tmp;
	}
1086042
public static void main(String[] args) {
    SparkSession spark = SparkSession
      .builder()
      .appName("Java Spark SQL user-defined Datasets aggregation example")
      .getOrCreate();

    // $example on:typed_custom_aggregation$
    Encoder<Employee> employeeEncoder = Encoders.bean(Employee.class);
    String path = "examples/src/main/resources/employees.json";
    Dataset<Employee> ds = spark.read().json(path).as(employeeEncoder);
    ds.show();
    // +-------+------+
    // |   name|salary|
    // +-------+------+
    // |Michael|  3000|
    // |   Andy|  4500|
    // | Justin|  3500|
    // |  Berta|  4000|
    // +-------+------+

    MyAverage myAverage = new MyAverage();
    // Convert the function to a `TypedColumn` and give it a name
    TypedColumn<Employee, Double> averageSalary = myAverage.toColumn().name("average_salary");
    Dataset<Double> result = ds.select(averageSalary);
    result.show();
    // +--------------+
    // |average_salary|
    // +--------------+
    // |        3750.0|
    // +--------------+
    // 
    spark.stop();
  }
185592
function createWordIndex(exampleData) {
  const index = {};
  const keys = ['shortdesc', 'title', 'tags'];
  exampleData.forEach((data, i) => {
    keys.forEach(key => {
      let text = data[key];
      if (Array.isArray(text)) {
        text = text.join(' ');
      }
      const words = text ? text.split(/\W+/) : [];
      words.forEach(word => {
        if (word) {
          word = word.toLowerCase();
          let counts = index[word];
          if (counts) {
            if (index in counts) {
              counts[i] += 1;
            } else {
              counts[i] = 1;
            }
          } else {
            counts = {};
            counts[i] = 1;
            index[word] = counts;
          }
        }
      });
    });
  });
  return index;
}
372023
def save_sampleset(self, f, name):
        ''''''
        f.create_dataset(name + '_encoder_x', data=self.encoder_x)
        f.create_dataset(name + '_decoder_x', data=self.decoder_x)
        f.create_dataset(name + '_decoder_y', data=self.decoder_y)
386863
@Override
    public DataSet sample(int numSamples, org.nd4j.linalg.api.rng.Random rng, boolean withReplacement) {
        Set<Integer> added = new HashSet<>();
        List<DataSet> toMerge = new ArrayList<>();
        boolean terminate = false;
        for (int i = 0; i < numSamples && !terminate; i++) {
            int picked = rng.nextInt(numExamples());
            if (!withReplacement) {
                while (added.contains(picked)) {
                    picked = rng.nextInt(numExamples());
                    if(added.size() == numExamples()){
                        terminate = true;
                        break;
                    }
                }
            }
            added.add(picked);
            toMerge.add(get(picked));
        }
        return DataSet.merge(toMerge);
    }
872366
def prepare_data_loader(args, dataset, vocab, test=False):
    
    # Preprocess
    dataset = dataset.transform(lambda s1, s2, label: (vocab(s1), vocab(s2), label),
                                lazy=False)

    # Batching
    batchify_fn = btf.Tuple(btf.Pad(), btf.Pad(), btf.Stack(dtype='int32'))
    data_lengths = [max(len(d[0]), len(d[1])) for d in dataset]
    batch_sampler = nlp.data.FixedBucketSampler(lengths=data_lengths,
                                                batch_size=args.batch_size,
                                                shuffle=(not test))
    data_loader = gluon.data.DataLoader(dataset=dataset,
                                        batch_sampler=batch_sampler,
                                        batchify_fn=batchify_fn)
    return data_loader
1365521
def _generate_examples(self, filepath):
    
    with tf.io.gfile.GFile(filepath, "rb") as f:
      data = tfds.core.lazy_imports.scipy.io.loadmat(f)

    # Maybe should shuffle ?

    assert np.max(data["y"]) <= 10  # Sanity check
    assert np.min(data["y"]) > 0

    for image, label in zip(np.rollaxis(data["X"], -1), data["y"]):
      yield {
          "image": image,
          "label": label % 10,  # digit 0 is saved as 0 (instead of 10)
      }
683056
@Override
    public DataSet sample(int numSamples, org.nd4j.linalg.api.rng.Random rng) {
        return sample(numSamples, rng, false);
    }
75074
def create_dataset(self, in_stream):
        
        assert XLS_IMPORT
        xls_book = xlrd.open_workbook(file_contents=in_stream)
        dataset = tablib.Dataset()
        sheet = xls_book.sheets()[0]

        dataset.headers = sheet.row_values(0)
        for i in range(1, sheet.nrows):
            dataset.append(sheet.row_values(i))
        return dataset
1013670
def _generate_examples(self, filepath):
    """"""
    rows_per_pair_id = collections.defaultdict(list)

    with tf.io.gfile.GFile(filepath) as f:
      reader = csv.DictReader(f, delimiter='\t', quoting=csv.QUOTE_NONE)
      for row in reader:
        rows_per_pair_id[row['pairID']].append(row)

    for rows in six.itervalues(rows_per_pair_id):
      premise = {row['language']: row['sentence1'] for row in rows}
      hypothesis = {row['language']: row['sentence2'] for row in rows}
      yield {
          'premise': premise,
          'hypothesis': hypothesis,
          'label': rows[0]['gold_label'],
      }
167216
function createSamples(src) {
  var result = [];

  src
    .replace(/…/, '')
    .trim()
    .replace(/,$/, '')
    .split(',')
    .map(function (val) { return val.trim(); })
    .forEach(function (val) {
      if (val.indexOf('~') !== -1) {
        result = result.concat(fillRange(val));
      } else {
        // push test data as String
        result.push(val);
        // push test data as Number if the same
        if (String(+val) === val) { result.push(+val); }
      }
    });

  return result;
}
7835
def dataset_generator(filepath,
                      dataset,
                      chunk_size=1,
                      start_idx=None,
                      end_idx=None):
  """"""
  encoder = dna_encoder.DNAEncoder(chunk_size=chunk_size)
  with h5py.File(filepath, "r") as h5_file:
    # Get input keys from h5_file
    src_keys = [s % dataset for s in ["%s_in", "%s_na", "%s_out"]]
    src_values = [h5_file[k] for k in src_keys]
    inp_data, mask_data, out_data = src_values
    assert len(set([v.len() for v in src_values])) == 1

    if start_idx is None:
      start_idx = 0
    if end_idx is None:
      end_idx = inp_data.len()

    for i in range(start_idx, end_idx):
      if i % 100 == 0:
        print("Generating example %d for %s" % (i, dataset))
      inputs, mask, outputs = inp_data[i], mask_data[i], out_data[i]
      ex_dict = to_example_dict(encoder, inputs, mask, outputs)
      # Original data has one output for every 128 input bases. Ensure that the
      # ratio has been maintained given the chunk size and removing EOS.
      assert (len(ex_dict["inputs"]) - 1) == ((
          128 // chunk_size) * ex_dict["targets_shape"][0])
      yield ex_dict
97801
private File saveExampleSet(ExampleSet outputSet, ZipModel model) throws IOException
	{
		logger.info("");

		//
		File tmp = File.createTempFile("train", null);
		tmp.deleteOnExit();
		//File tmp = new File("examples/train");

		BufferedWriter out = new BufferedWriter(new FileWriter(tmp));
		outputSet.write(out);
		out.close();

		// add the example set to the model
		model.add(tmp, "train");

		return tmp;
	}
1344306
def sample(cls, dataset, samples=[]):
        
        ndims = dataset.ndims
        dimensions = dataset.dimensions(label='name')
        arrays = [dataset.data[vdim.name] for vdim in dataset.vdims]
        data = defaultdict(list)

        for sample in samples:
            if np.isscalar(sample): sample = [sample]
            if len(sample) != ndims:
                sample = [sample[i] if i < len(sample) else None
                          for i in range(ndims)]
            sampled, int_inds = [], []
            for d, ind in zip(dimensions, sample):
                cdata = dataset.data[d]
                mask = cls.key_select_mask(dataset, cdata, ind)
                inds = np.arange(len(cdata)) if mask is None else np.argwhere(mask)
                int_inds.append(inds)
                sampled.append(cdata[mask])
            for d, arr in zip(dimensions, np.meshgrid(*sampled)):
                data[d].append(arr)
            for vdim, array in zip(dataset.vdims, arrays):
                da = dask_array_module()
                flat_index = np.ravel_multi_index(tuple(int_inds)[::-1], array.shape)
                if da and isinstance(array, da.Array):
                    data[vdim.name].append(array.flatten().vindex[tuple(flat_index)])
                else:
                    data[vdim.name].append(array.flat[flat_index])
        concatenated = {d: np.concatenate(arrays).flatten() for d, arrays in data.items()}
        return concatenated
936300
def fill_with_sample_data(self):
        """"""
        # replace these sample data with another dataset later
        # this function is deprecated as soon as a common file format for this
        # type of data will be available
        self.p01 = np.array([[0.576724636119866,  0.238722774405744,  0.166532122130638,  0.393474644666218],
                             [0.303345245644811,  0.0490956843857575, 0.0392403031072856, 0.228441890034704]])

        self.p10 = np.array([[0.158217002255554,  0.256581140990052,  0.557852226779526,  0.422638238585814],
                             [0.0439831163244427, 0.0474928027621488, 0.303675296728195,  0.217512052135178]])

        self.pxx = np.array([[0.265058361624580,  0.504696084604205,  0.275615651089836,  0.183887116747968],
                             [0.652671638030746,  0.903411512852094,  0.657084400164519,  0.554046057830118]])

        self.wxx = np.array([[[0.188389148850583, 0.0806836453984190, 0.0698113025807722, 0.0621499191745602],
                              [0.240993281622128, 0.0831019646519721, 0.0415130545715575, 0.155284541403192]],
                             [[0.190128959522795, 0.129220679033862,  0.0932213021787505, 0.193080698516532],
                              [0.196379692358065, 0.108549414860949,  0.0592714297292217, 0.0421945385836429]],
                             [[0.163043672107111, 0.152063537378127,  0.102823783410167,  0.0906028835221283],
                              [0.186579466868095, 0.189705690316132,  0.0990207345993082, 0.107831389238912]],
                             [[0.197765724699431, 0.220046257566978,  0.177876233348082,  0.261288786454262],
                              [0.123823472714948, 0.220514673922285,  0.102486496386323,  0.101975538893918]],
                             [[0.114435243444815, 0.170857634762767,  0.177327072603662,  0.135362730582518],
                              [0.0939211776723413,0.174291820501902,  0.125275822078525,  0.150842841725936]],
                             [[0.0988683809545079, 0.152323481100248, 0.185606883566286, 0.167242856061538],
                              [0.0760275616817939, 0.127275603247149,  0.202466168603738,  0.186580243138018]],
                             [[0.0473688704207573, 0.0948047647595988, 0.193333422312280,  0.0902721256884624],
                              [0.0822753470826286, 0.0965608324996108, 0.369966294031327,  0.255290907016382]]])