:param dataSet: dataset name
  :param noise: amount of noise added to the dataset
  :return:
434704
def readDataSet(dataSet, noise=0):
  
  filePath = 'data/'+dataSet+'.csv'

  if dataSet == 'nyc_taxi' or dataSet == 'nyc_taxi_perturb' or dataSet == 'nyc_taxi_perturb_baseline':
    seq = pd.read_csv(filePath, header=0, skiprows=[1, 2], names=['time', 'data', 'timeofday', 'dayofweek'])
    seq['time'] = pd.to_datetime(seq['time'])

    if noise > 0:
      for i in xrange(len(seq)):
        value = seq['data'][i]
        noiseValue = np.random.normal(scale=(value * noise))
        value += noiseValue
        value = max(0, value)
        value = min(40000, value)
        seq['data'][i] = value
  # elif dataSet == 'sine':
  #   df = pd.read_csv(filePath, header=0, skiprows=[1, 2], names=['time', 'data'])
  #   sequence = df['data']
  #   seq = pd.DataFrame(np.array(sequence), columns=['data'])
  else:
    raise(' unrecognized dataset type ')

  return seq
1356182
def redim(cls, dataset, dimensions):
        
        new_dataset = dataset.data.copy()
        for name, new_dim in dimensions.items():
            if name == new_dataset.name():
                new_dataset.rename(new_dim.name)
            for coord in new_dataset.dim_coords:
                if name == coord.name():
                    coord.rename(new_dim.name)
        return new_dataset
757401
def is_dataset_on_gcs(dataset_name):
  """"""
  dir_name = posixpath.join(GCS_DATASETS_DIR, dataset_name)
  return len(gcs_files(prefix_filter=dir_name)) > 2
185782
def data_available(dataset_name=None):
    """"""
    dr = data_resources[dataset_name]
    if 'dirs' in dr:
        for dirs, files in zip(dr['dirs'], dr['files']):
            for dir, file in zip(dirs, files):
                if not os.path.exists(os.path.join(data_path, dataset_name, dir, file)):
                    return False
    else:
        for file_list in dr['files']:
            for file in file_list:
                if not os.path.exists(os.path.join(data_path, dataset_name, file)):
                    return False
    return True
658197
def _canonize_input(self, dataset):
        
        unpack = lambda x: x
        if isinstance(dataset, _tc.SArray):
            dataset = _tc.SFrame({self.feature: dataset})
        elif isinstance(dataset, _tc.Image):
            dataset = _tc.SFrame({self.feature: [dataset]})
            unpack = lambda x: x[0]
        return dataset, unpack
269616
def save_examples(noise = 0.0, maxVal = 1.0, dataDir="data"):
  
  testDataDir = os.path.join(dataDir, "speech_commands", "test")
  outDir = os.path.join(dataDir, "noise_examples")
  if not os.path.exists(outDir):
    os.mkdir(outDir)

  # Create noise dataset with noise transform
  noiseTransform = transforms.Compose([
    LoadAudio(),
    FixAudioLength(),
    AddNoise(noise, maxVal=maxVal),
  ])

  noiseDataset = SpeechCommandsDataset(
    testDataDir, noiseTransform, silence_percentage=0,
  )

  for i in range(0, 2552, 100):
    d, target = noiseDataset[i]
    fname = os.path.join(outDir, noiseDataset.classes[target] + "_" +
                         str(i) + "_"
                         + str(int(noise*100)) + "_"
                         + str(int(maxVal*100))
                         + ".wav"
                         )
    print(d["path"], fname)
    librosa.output.write_wav(fname, d['samples'], d["sample_rate"])
1319090
def noise_covariance(fit, dof=2, **kw):
    
    ev = fit.eigenvalues

    measurement_noise = ev[-1]/(fit.n-dof)
    return 4*ev*measurement_noise
1055713
def _populate_lp(self, dataset, **kwargs):
        
        logger.debug("{}._populate_lp(dataset={})".format(self.component, dataset))

        profile_rest = kwargs.get('profile_rest', self.lp_profile_rest.get(dataset))

        rv_cols = self._populate_rv(dataset, **kwargs)

        cols = rv_cols
        # rvs = (rv_cols['rvs']*u.solRad/u.d).to(u.m/u.s).value
        # cols['dls'] = rv_cols['rvs']*profile_rest/c.c.si.value

        return cols
312591
def check_dataset(dataset):
    """"""
    if isinstance(dataset, numpy.ndarray) and not len(dataset.shape) == 4:
        check_dataset_shape(dataset)
        check_dataset_range(dataset)
    else:  # must be a list of arrays or a 4D NumPy array
        for i, d in enumerate(dataset):
            if not isinstance(d, numpy.ndarray):
                raise ValueError(
                    'Requires a NumPy array (rgb x rows x cols) '
                    'with integer values in the range [0, 255].'
                )
            try:
                check_dataset_shape(d)
                check_dataset_range(d)
            except ValueError as err:
                raise ValueError(
                    '{}\nAt position {} in the list of arrays.'
                    .format(err, i)
                )
583292
def dataset_status_cli(self, dataset, dataset_opt=None):
        
        dataset = dataset or dataset_opt
        return self.dataset_status(dataset)
893619
def try_fix_dataset(dataset):
    """"""
    if isinstance(dataset, numpy.ndarray):
        if len(dataset.shape) == 3:  # NumPy 3D
            if dataset.shape[-1] == 3:
                return dataset.transpose((2, 0, 1))
        elif len(dataset.shape) == 4:  # NumPy 4D
            if dataset.shape[-1] == 3:
                return dataset.transpose((0, 3, 1, 2))
        # Otherwise couldn't fix it.
        return dataset
    # List of Numpy 3D arrays.
    for i, d in enumerate(dataset):
        if not isinstance(d, numpy.ndarray):
            return dataset
        if not (len(d.shape) == 3 and d.shape[-1] == 3):
            return dataset
        dataset[i] = d.transpose()
    return dataset
1173557
public void fit(DataSet dataSet) {
        if (numInputArrays != 1 || numOutputArrays != 1)
            throw new UnsupportedOperationException("Cannot train ComputationGraph network with "
                    + " multiple inputs or outputs using a DataSet");

        boolean hasMaskArrays = dataSet.hasMaskArrays();
        if (hasMaskArrays) {
            INDArray[] fMask = (dataSet.getFeaturesMaskArray() != null ? new INDArray[]{dataSet.getFeaturesMaskArray()}
                    : null);
            INDArray[] lMask = (dataSet.getLabelsMaskArray() != null ? new INDArray[]{dataSet.getLabelsMaskArray()}
                    : null);
            fit(new INDArray[]{dataSet.getFeatures()}, new INDArray[]{dataSet.getLabels()}, fMask, lMask);
        } else {
            fit(new INDArray[]{dataSet.getFeatures()}, new INDArray[]{dataSet.getLabels()});
        }

        if (hasMaskArrays)
            clearLayerMaskArrays();

        clearLayersStates();
    }
9017
def noisepickle(d, data, u, v, w, chunk=200):
    

    if d['savenoise']:
        noisefile = getnoisefile(d)

        if os.path.exists(noisefile):
            logger.warn('noisefile %s already exists' % noisefile)
        else:
            nints = len(data)
            chunk = min(chunk, nints)  # ensure at least one measurement
            results = []

            rr = range(0, nints, chunk)
            if len(rr) == 1: rr.append(1)   # hack. need to make sure it iterates for nints=1 case
            for i in range(len(rr)-1):
                imid = (rr[i]+rr[i+1])/2
                noiseperbl = estimate_noiseperbl(data[rr[i]:rr[i+1]])
                imstd = sample_image(d, data, u, v, w, imid, verbose=0).std()
                zerofrac = float(len(n.where(data[rr[i]:rr[i+1]] == 0j)[0]))/data[rr[i]:rr[i+1]].size
                results.append( (d['segment'], noiseperbl, zerofrac, imstd) )

            with open(noisefile, 'a') as pkl:
                pickle.dump(results, pkl)
            logger.info('Wrote %d noise measurement%s to %s.' % (len(results), 's'[:len(results)-1], noisefile))
70305
def remove_noise(time, magnitude, error, error_limit=3, std_limit=5):
    
    data, mjd = magnitude, time

    data_len = len(mjd)
    error_mean = np.mean(error)
    error_tolerance = error_limit * (error_mean or 1)
    data_mean = np.mean(data)
    data_std = np.std(data)

    mjd_out, data_out, error_out = [], [], []
    for i in range(data_len):
        is_not_noise = (
            error[i] < error_tolerance and
            (np.absolute(data[i] - data_mean) / data_std) < std_limit)

        if is_not_noise:
            mjd_out.append(mjd[i])
            data_out.append(data[i])
            error_out.append(error[i])

    data_out = np.asarray(data_out)
    mjd_out = np.asarray(mjd_out)
    error_out = np.asarray(error_out)

    return mjd_out, data_out, error_out
1286915
def addDataset(self, dataset):
        
        id_ = dataset.getId()
        self._datasetIdMap[id_] = dataset
        self._datasetNameMap[dataset.getLocalId()] = dataset
        self._datasetIds.append(id_)
147876
def _addNoise(self, pattern, noiseLevel, inputSize):
    
    if pattern is None:
      return None

    # Bits that could be noise. These can't be from the original set.
    candidateBits = list(set(range(inputSize)) - set(pattern))
    random.shuffle(candidateBits)

    newBits = set()
    for bit in pattern:
      if random.random() < noiseLevel:
        newBits.add(candidateBits.pop())
      else:
        newBits.add(bit)

    return newBits
1083699
def createNoiseExperimentArgs():
  """"""
  experimentArguments = []
  n = 6000
  for a in [128]:
    noisePct = 0.75
    while noisePct <= 0.85:
      noise = int(round(noisePct*a,0))
      # Some parameter combinations are just not worth running!
      experimentArguments.append(
        ("./sdr_calculations2",
         "results_noise_10m/temp_"+str(n)+"_"+str(a)+"_"+str(noise)+"_30.csv",
         "200000", str(n), str(a), str(noise))
      )
      noisePct += 0.05
  return experimentArguments