Constructs a Dataset by reading in a file in SVM light format.
The lines parameter is filled with the lines of the file for further processing
(if lines is null, it is assumed no line information is desired)
495834
public static Dataset<String, String> readSVMLightFormat(String filename, List<String> lines) {
    return readSVMLightFormat(filename, new HashIndex<String>(), new HashIndex<String>(), lines);
  }
859065
def import_libsvm_sparse(filename):
    """"""
    from sklearn.datasets import load_svmlight_file
    X, y = load_svmlight_file(filename)
    return Dataset(X.toarray().tolist(), y.tolist())
11322
def data(self, data, part=False, dataset=''):
        
        #if self.parser is None:
        #    raise ValueError('no parser')
        imgs = self.imgtype.convert(data)
        for channel, data in zip(self.imgtype.channels, imgs):
            key = dataset + channel
            data = self.scanner(data, part)
            if isinstance(self.parser, LevelParser):
                self.storage.add_links(self.parser(data, part, key))
            else:
                for level, level_data in enumerate(data):
                    level_key = key + level_dataset(level)
                    level_data = self.parser(level_data, part, level_key)
                    self.storage.add_links(level_data)
208061
def prepare_dataset(dataset,
                    formula_id2index,
                    feature_list,
                    is_traindata,
                    do_normalization=False):
    """"""
    prepared = []
    start_time = time.time()
    translation = []
    for i, data in enumerate(dataset):
        x = []
        handwriting = data['handwriting']
        x = handwriting.feature_extraction(feature_list)  # Feature selection
        y = formula_id2index[data['formula_id']]  # Get label
        translation.append((handwriting.raw_data_id,
                            handwriting.formula_in_latex,
                            handwriting.formula_id))
        prepared.append((numpy.array(x), y))
        if i % 100 == 0 and i > 0:
            utils.print_status(len(dataset), i, start_time)
    sys.stdout.write("\r100%" + " "*80 + "\n")
    sys.stdout.flush()

    # Feature normalization
    if do_normalization:
        _normalize_features(feature_list, prepared, is_traindata)
    return (prepared, translation)
104596
public static Dataset<String, String> readSVMLightFormat(String filename) {
    return readSVMLightFormat(filename, new HashIndex<String>(), new HashIndex<String>());
  }
594673
def dataset_to_stream(dataset, input_name, num_chunks=0, append_targets=False):
  """"""
  for example in tfds.as_numpy(dataset):
    inp, out = example[0][input_name], example[1]
    if len(out.shape) > 1 and out.shape[-1] == 1:
      out = np.squeeze(out, axis=-1)
    if num_chunks > 0:
      inp = np.split(inp, num_chunks, axis=1)
      out = np.split(out, num_chunks, axis=1)
    if append_targets:
      inp = (inp, out)
    yield inp, out
178823
def dataset(directory, images_file, labels_file):
  """"""

  images_file = download(directory, images_file)
  labels_file = download(directory, labels_file)

  check_image_file_header(images_file)
  check_labels_file_header(labels_file)

  def decode_image(image):
    # Normalize from [0, 255] to [0.0, 1.0]
    image = tf.decode_raw(image, tf.uint8)
    image = tf.cast(image, tf.float32)
    image = tf.reshape(image, [784])
    return image / 255.0

  def decode_label(label):
    label = tf.decode_raw(label, tf.uint8)  # tf.string -> [tf.uint8]
    label = tf.reshape(label, [])  # label is a scalar
    return tf.to_int32(label)

  images = tf.data.FixedLengthRecordDataset(
      images_file, 28 * 28, header_bytes=16).map(decode_image)
  labels = tf.data.FixedLengthRecordDataset(
      labels_file, 1, header_bytes=8).map(decode_label)
  return tf.data.Dataset.zip((images, labels))
78754
def build_dataset(filename, sequence_length, alphabet_size, max_lines=-1):
    """"""
    with open(filename) as f:
        content = f.readlines()
    content = content[:max_lines]
    content = [line for line in content if len(line) > 2]   # Remove blank lines
    seqs = np.zeros((sequence_length, len(content), alphabet_size))
    for ix, line in enumerate(content):
        padded_line = (line + " " * sequence_length)[:sequence_length]
        seqs[:, ix, :] = string_to_one_hot(padded_line, alphabet_size)
    return seqs
1003029
def create_dataset(self, in_stream):
        
        assert XLSX_IMPORT
        from io import BytesIO
        xlsx_book = openpyxl.load_workbook(BytesIO(in_stream), read_only=True)

        dataset = tablib.Dataset()
        sheet = xlsx_book.active

        # obtain generator
        rows = sheet.rows
        dataset.headers = [cell.value for cell in next(rows)]

        for row in rows:
            row_values = [cell.value for cell in row]
            dataset.append(row_values)
        return dataset
274387
private ExampleSet readDataSet(File in) throws IOException {
        logger.info("read the example set");

        //
        ExampleSet inputSet1 = new SentenceSetCopy();
        inputSet1.read(new BufferedReader(new FileReader(in)));

        return inputSet1;
    }
1019139
def get_dataset_split(tmp_dir, split, use_control_set):
  
  if not use_control_set:
    dataset_split = {
        problem.DatasetSplit.TRAIN: [
            f for f in tf.gfile.Glob(
                os.path.join(tmp_dir, "train-novels/*/*.txt"))
        ],
        problem.DatasetSplit.EVAL: [
            os.path.join(tmp_dir, "lambada_development_plain_text.txt")
        ],
        problem.DatasetSplit.TEST: [
            os.path.join(tmp_dir, "lambada_test_plain_text.txt")
        ]
    }

  else:
    dataset_split = {
        problem.DatasetSplit.TRAIN: [
            f for f in tf.gfile.Glob(
                os.path.join(tmp_dir, "train-novels/*/*.txt"))
        ],
        problem.DatasetSplit.EVAL: [
            os.path.join(tmp_dir, "lambada_control_test_data_plain_text.txt")
        ],
    }

  return dataset_split[split]
920871
def get_dataset(self, key, info):
        """"""
        logger.debug("Reading %s.", key)
        return self.file_content[key.name]
1374456
def parse_values_from_lines(self,lines):
        
        assert len(lines) == 3,"SvdData.parse_values_from_lines: expected " + \
                               "3 lines, not {0}".format(len(lines))
        try:
            self.svdmode = int(lines[0].strip().split()[0])
        except Exception as e:
            raise Exception("SvdData.parse_values_from_lines: error parsing" + \
                            " svdmode from line {0}: {1} \n".format(lines[0],str(e)))
        try:
            raw = lines[1].strip().split()
            self.maxsing = int(raw[0])
            self.eigthresh = float(raw[1])
        except Exception as e:
            raise Exception("SvdData.parse_values_from_lines: error parsing" + \
                            " maxsing and eigthresh from line {0}: {1} \n"\
                            .format(lines[1],str(e)))
        # try:
        #     self.eigwrite = int(lines[2].strip())
        # except Exception as e:
        #     raise Exception("SvdData.parse_values_from_lines: error parsing" + \
        #                     " eigwrite from line {0}: {1} \n".format(lines[2],str(e)))
        self.eigwrite = lines[2].strip()
338419
def build (features = nil)
      File.open(@directory + "/train", "w") do |f| 
        f.puts @fv_train.to_libSVM(features) 
      end
      
      #Execute the LibSVM model train process without much fanfare
      output = `#{@svm_train_path} #{@directory}/train #{@directory}/model`
      
      puts output if (@verbose)
  
      #To determine the 
      @fv_test  = Eluka::FeatureVectors.new(@features, false)
      return output
    end
1209811
def load_data_split(proc_data_dir):
    
    ds_train = Dataset.load(path.join(proc_data_dir, 'train.bin'))
    ds_val = Dataset.load(path.join(proc_data_dir, 'val.bin'))
    ds_test = Dataset.load(path.join(proc_data_dir, 'test.bin'))
    return ds_train, ds_val, ds_test
1305133
private void read_problem() throws IOException
	{
		BufferedReader fp = new BufferedReader(new FileReader(input_file_name));
		java.util.Vector vy = new java.util.Vector();
		java.util.Vector vx = new java.util.Vector();
		int max_index = 0;

		while(true)
		{
			String line = fp.readLine();
			if(line == null) break;

			StringTokenizer st = new StringTokenizer(line," \t\n\r\f:");

			vy.addElement(st.nextToken());
			int m = st.countTokens()/2;
			svm_node[] x = new svm_node[m];
			for(int j=0;j<m;j++)
			{
				x[j] = new svm_node();
				x[j].index = atoi(st.nextToken());
				x[j].value = atof(st.nextToken());
			}
			if(m>0) max_index = Math.max(max_index, x[m-1].index);
			vx.addElement(x);
		}

		prob = new svm_problem();
		prob.l = vy.size();
		prob.x = new svm_node[prob.l][];
		for(int i=0;i<prob.l;i++)
			prob.x[i] = (svm_node[])vx.elementAt(i);
		prob.y = new double[prob.l];
		for(int i=0;i<prob.l;i++)
			prob.y[i] = atof((String)vy.elementAt(i));

		if(param.gamma == 0)
			param.gamma = 1.0/max_index;

		fp.close();
	}
1123350
def _read(self, directory, filename, session, path, name, extension, spatial, spatialReferenceID, maskMap):
        
        # Assign file extension attribute to file object
        self.fileExtension = extension

        if isinstance(maskMap, RasterMapFile) and maskMap.fileExtension == 'msk':
            # Vars from mask map
            columns = maskMap.columns
            rows = maskMap.rows
            upperLeftX = maskMap.west
            upperLeftY = maskMap.north

            # Derive the cell size (GSSHA cells are square, so it is the same in both directions)
            cellSizeX = int(abs(maskMap.west - maskMap.east) / columns)
            cellSizeY = -1 * cellSizeX

            # Dictionary of keywords/cards and parse function names
            KEYWORDS = {'DATASET': wdc.datasetHeaderChunk,
                        'TS': wdc.datasetScalarTimeStepChunk}

            # Open file and read plain text into text field
            with open(path, 'r') as f:
                chunks = pt.chunk(KEYWORDS, f)

            # Parse header chunk first
            header = wdc.datasetHeaderChunk('DATASET', chunks['DATASET'][0])

            # Parse each time step chunk and aggregate
            timeStepRasters = []

            for chunk in chunks['TS']:
                timeStepRasters.append(wdc.datasetScalarTimeStepChunk(chunk, columns, header['numberCells']))

            # Set WMS dataset file properties
            self.name = header['name']
            self.numberCells = header['numberCells']
            self.numberData = header['numberData']
            self.objectID = header['objectID']

            if header['type'] == 'BEGSCL':
                self.objectType = header['objectType']
                self.type = self.SCALAR_TYPE

            elif header['type'] == 'BEGVEC':
                self.vectorType = header['objectType']
                self.type = self.VECTOR_TYPE

            # Create WMS raster dataset files for each raster
            for timeStep, timeStepRaster in enumerate(timeStepRasters):
                # Create new WMS raster dataset file object
                wmsRasterDatasetFile = WMSDatasetRaster()

                # Set the wms dataset for this WMS raster dataset file
                wmsRasterDatasetFile.wmsDataset = self

                # Set the time step and timestamp and other properties
                wmsRasterDatasetFile.iStatus = timeStepRaster['iStatus']
                wmsRasterDatasetFile.timestamp = timeStepRaster['timestamp']
                wmsRasterDatasetFile.timeStep = timeStep + 1

                # If spatial is enabled create PostGIS rasters
                if spatial:
                    # Process the values/cell array
                    wmsRasterDatasetFile.raster = RasterLoader.makeSingleBandWKBRaster(session,
                                                                                       columns, rows,
                                                                                       upperLeftX, upperLeftY,
                                                                                       cellSizeX, cellSizeY,
                                                                                       0, 0,
                                                                                       spatialReferenceID,
                                                                                       timeStepRaster['cellArray'])

                # Otherwise, set the raster text properties
                else:
                    wmsRasterDatasetFile.rasterText = timeStepRaster['rasterText']

            # Add current file object to the session
            session.add(self)

        else:
            log.warning("Could not read {0}. Mask Map must be supplied "
                     "to read WMS Datasets.".format(filename))