Determine the minimum sequence length given a dataset_split.

    Args:
      dataset_split: A problem.DatasetSplit.

    Returns:
      The minimum length that a sequence can be for this dataset_split.
970091
def min_sequence_length(self, dataset_split):
    
    return {
        problem.DatasetSplit.TRAIN: 8,
        problem.DatasetSplit.EVAL: 65,
        problem.DatasetSplit.TEST: 65
    }[dataset_split]
865111
def length(cls, dataset):
        
        return np.product([len(d.points) for d in dataset.data.coords(dim_coords=True)], dtype=np.intp)
13294
def min(self):
        """"""
        if "min" not in self.attrs.keys():

            def f(dataset, s):
                return np.nanmin(dataset[s])

            self.attrs["min"] = np.nanmin(list(self.chunkwise(f).values()))
        return self.attrs["min"]
1261411
public static function split(DataSet $dataSet, float $split): array
    {
        if ($split <= 0 || $split > 1) {
            throw new WrongUsageException('The split must be between 0 and 1');
        }

        $data = $dataSet->getRawData();
        $dataCount = floor($dataSet->getSize() * $split);
        if ($dataCount == 0) {
            $dataCount++;
        }
        $randomKeys = array_rand($data, $dataCount);
        $randomKeys = is_array($randomKeys)? $randomKeys : [$randomKeys];
        $data1 = $data2 = [];

        foreach ($data as $index => $row) {
            if (in_array($index, $randomKeys)) {
                $data1[] = $row;
            } else {
                $data2[] = $row;
            }
        }


        $dataSet1 = self::create($data1);
        $dataSet1->prepare($dataSet->getMapper());

        $dataSet2 = self::create($data2);
        $dataSet2->prepare($dataSet->getMapper());

        return [$dataSet1, $dataSet2];
    }
436872
def set_split_extents_by_split_size(self):
        

        if self.split_size is None:
            if (
                _np.all([s is not None for s in self.split_num_slices_per_axis])
                and
                _np.all([s > 0 for s in self.split_num_slices_per_axis])
            ):
                self.split_size = _np.product(self.split_num_slices_per_axis)
            else:
                raise ValueError(
                    (
                        "Got invalid self.split_num_slices_per_axis=%s, all elements "
                        +
                        "need to be integers greater than zero when self.split_size is None."
                    )
                    %
                    self.split_num_slices_per_axis
                )
        self.logger.debug(
            "Pre  cannonicalise: self.split_num_slices_per_axis=%s",
            self.split_num_slices_per_axis)
        self.split_num_slices_per_axis = \
            calculate_num_slices_per_axis(
                self.split_num_slices_per_axis,
                self.split_size,
                self.array_shape
            )
        self.logger.debug(
            "Post cannonicalise: self.split_num_slices_per_axis=%s",
            self.split_num_slices_per_axis)
        # Define the start and stop indices (extents) for each axis slice
        self.split_shape = self.split_num_slices_per_axis.copy()
        self.split_begs = [[], ] * len(self.array_shape)
        self.split_ends = [[], ] * len(self.array_shape)
        for i in range(len(self.array_shape)):
            self.split_begs[i], self.split_ends[i] = \
                self.calculate_axis_split_extents(
                    self.split_shape[i],
                    self.array_shape[i]
            )
840469
private static int calcAdjustedMinimumSequenceLength(
			List<Subunit> subunits, int absMinLen, double fraction, int minLen) {

		int maxLength = Integer.MIN_VALUE;
		int minLength = Integer.MAX_VALUE;

		// Extract the length List, the min and the max
		List<Integer> lengths = new ArrayList<Integer>();
		for (int i = 0; i < subunits.size(); i++) {
			if (subunits.get(i).size() >= absMinLen) {
				maxLength = Math.max(subunits.get(i).size(), maxLength);
				minLength = Math.min(subunits.get(i).size(), minLength);
				lengths.add(subunits.get(i).size());

			}
		}

		int adjustedMinimumSequenceLength = minLen;

		if (lengths.size() < 2)
			return adjustedMinimumSequenceLength;

		// Calculate the median of the lengths
		double median = 0;
		Collections.sort(lengths);
		if (lengths.size() % 2 == 1) {
			int middle = (lengths.size() - 1) / 2;
			median = lengths.get(middle);
		} else {
			int middle2 = lengths.size() / 2;
			int middle1 = middle2 - 1;
			median = 0.5 * (lengths.get(middle1) + lengths.get(middle2));
		}

		// If the median * fraction is lower than the minLength
		if (minLength >= median * fraction) {
			adjustedMinimumSequenceLength = Math.min(minLength, minLen);
		}

		return adjustedMinimumSequenceLength;
	}
478884
def calculate_size(name, start_sequence, min_count, max_count, filter):
    
    data_size = 0
    data_size += calculate_size_str(name)
    data_size += LONG_SIZE_IN_BYTES
    data_size += INT_SIZE_IN_BYTES
    data_size += INT_SIZE_IN_BYTES
    data_size += BOOLEAN_SIZE_IN_BYTES
    if filter is not None:
        data_size += calculate_size_data(filter)
    return data_size
981637
def length(cls, dataset):
        
        if not dataset.data:
            return 0
        length = 0
        ds = cls._inner_dataset_template(dataset)
        for d in dataset.data:
            ds.data = d
            length += ds.interface.length(ds)
        return length+len(dataset.data)-1
172514
def auto_batch_size(sequence_length,
                    mesh_shape,
                    layout_rules,
                    tokens_per_split=2048):
  
  num_splits = mtf.tensor_dim_to_mesh_dim_size(
      layout_rules, mesh_shape, mtf.Dimension("batch", 0))
  ret = max(1, tokens_per_split // sequence_length) * num_splits
  tf.logging.info(
      "AUTO_BATCH_SIZE tokens_per_split=%s num_splits=%s"
      " sequence_length=%s batch_size=%s"
      % (tokens_per_split, num_splits, sequence_length, ret))
  return ret
146259
def check_line_length(self, splitted_line, expected_length):
        
        if len(splitted_line) != expected_length:
            raise WrongLineFormat(
                            message='WRONG FORMATED PED LINE!',
                            ped_line = '\t'.join(splitted_line))
        return
1271922
def effective_len(self):
    
    if self._effective_len is None:
      self._effective_len = len([nuc for nuc in self.sequenceData
                                 if nuc != "N" and nuc != "n"])
    return self._effective_len
876026
def split_image(self, split_len):
        
        result = copy.copy(self)
        result.data = self.data[:split_len]
        self.data = self.data[split_len:]
        self.addr += split_len
        self.file_offs = None
        result.file_offs = None
        return result
543036
def splits_and_paths(self, data_dir):
    """"""
    filepath_fns = {
        problem.DatasetSplit.TRAIN: self.training_filepaths,
        problem.DatasetSplit.EVAL: self.dev_filepaths,
        problem.DatasetSplit.TEST: self.test_filepaths,
    }

    def append_epoch(paths):
      return [
          "{}.{}".format(path, self.current_epoch)
          for path in paths
      ]

    # We set shuffled=True as we don't want to shuffle on disk later.
    return [
        (split["split"], append_epoch(filepath_fns[split["split"]](
            data_dir, split["shards"], shuffled=True
        )))
        for split in self.dataset_splits
    ]
520143
def dataset_splits(self):
    """"""
    return [{
        "split": problem.DatasetSplit.TRAIN,
        "shards": self.num_train_shards,
    }, {
        "split": problem.DatasetSplit.EVAL,
        "shards": self.num_eval_shards,
    }, {
        "split": problem.DatasetSplit.TEST,
        "shards": self.num_test_shards,
    }]
1179710
private List<F> getHighPrecisionFeatures(GeneralDataset<L,F> dataset, double minPrecision, int maxNumFeatures){
    int[][] feature2label = new int[dataset.numFeatures()][dataset.numClasses()];
    for(int f = 0; f < dataset.numFeatures(); f++)
      Arrays.fill(feature2label[f],0);

    int[][] data = dataset.data;
    int[] labels = dataset.labels;
    for(int d = 0; d < data.length; d++){
      int label = labels[d];
      //System.out.println("datum id:"+d+" label id: "+label);
      if(data[d] != null){
        //System.out.println(" number of features:"+data[d].length);
        for(int n = 0; n < data[d].length; n++){
          feature2label[data[d][n]][label]++;
        }
      }
    }
    Counter<F> feature2freq = new ClassicCounter<F>();
    for(int f = 0; f < dataset.numFeatures(); f++){
     int maxF = ArrayMath.max(feature2label[f]);
     int total = ArrayMath.sum(feature2label[f]);
     double precision = ((double)maxF)/total;
     F feature = dataset.featureIndex.get(f);
     if(precision >= minPrecision){
       feature2freq.incrementCount(feature, total);
     }
    }
    if(feature2freq.size() > maxNumFeatures){
      Counters.retainTop(feature2freq, maxNumFeatures);
    }
    //for(F feature : feature2freq.keySet())
      //System.out.println(feature+" "+feature2freq.getCount(feature));
    //System.exit(0);
    return Counters.toSortedList(feature2freq);
  }
1160829
def slice_on_length(self, data_len, *addSlices):
        '''
        
        '''
        if len(self.ordered_ranges) + len(addSlices) == 0:
            return slice(None,None,None)
        ranges = self.ordered_ranges
        if len(addSlices) > 0:
            ranges = ranges + DimensionRange(*addSlices).ordered_ranges
        return self._combine_lists_of_ranges_on_length(data_len, *ranges)
528579
public static int numExamplesTrain(Set dataSet) {
        switch (dataSet) {
            case COMPLETE:
                return NUM_COMPLETE_TRAIN;
            case MERGE:
                return NUM_MERGE_TRAIN;
            case BALANCED:
                return NUM_BALANCED_TRAIN;
            case LETTERS:
                return NUM_LETTERS_TRAIN;
            case DIGITS:
                return NUM_DIGITS_TRAIN;
            case MNIST:
                return NUM_MNIST_TRAIN;
            default:
                throw new UnsupportedOperationException("Unknown Set: " + dataSet);
        }
    }