Checks that the feature types of this dataset are consistent with a timeseries-profile-orthogonal dataset.

        :param netCDF4.Dataset dataset: An open netCDF dataset
333194
def check_dimensions(self, dataset):
        '''
        
        '''
        results = []
        required_ctx = TestCtx(BaseCheck.HIGH, 'All geophysical variables are timeseries-profile-orthogonal feature types')

        message = '{} must be a valid profile-orthogonal feature type. It must have dimensions of (station, time, z).'
        message += ' If it\'s a single station, it must have dimensions (time, z). x and y dimensions must be scalar or have'
        message += ' dimensions (station). time must be a coordinate variable with dimension (time) and z must be a'
        message += ' coordinate variabel with dimension (z).'

        for variable in util.get_geophysical_variables(dataset):
            is_valid = util.is_timeseries_profile_single_station(dataset, variable)
            is_valid = is_valid or util.is_timeseries_profile_multi_station(dataset, variable)
            required_ctx.assert_true(
                is_valid,
                message.format(variable)
            )
        results.append(required_ctx.to_result())
        return results
1138363
def is_timeseries_profile_single_station(nc, variable):
    '''
    
    '''

    # x, y, z(z), t(t)
    # X(t, z)
    dims = nc.variables[variable].dimensions
    cmatrix = coordinate_dimension_matrix(nc)

    for req in ('x', 'y', 'z', 't'):
        if req not in cmatrix:
            return False
    if len(cmatrix['x']) != 0:
        return False
    if cmatrix['x'] != cmatrix['y']:
        return False

    z = get_z_variable(nc)
    if cmatrix['z'] != (z,):
        return False
    t = get_time_variable(nc)
    if cmatrix['t'] != (t,):
        return False

    if dims == (t, z):
        return True
    return False
210350
def _check_feature_types(self):
        
        if self.default_feature_type is not None and self.default_feature_type not in self.allowed_feature_types:
            raise ValueError('Default feature type parameter must be one of the allowed feature types')

        for feature_type in self.feature_collection:
            if feature_type is not None and feature_type not in self.allowed_feature_types:
                raise ValueError('Feature type has to be one of {}, but {} found'.format(self.allowed_feature_types,
                                                                                         feature_type))
1241971
def check_recommended_attributes(self, dataset):
        '''
        
        '''
        results = []
        recommended_ctx = TestCtx(BaseCheck.MEDIUM, 'Recommended global attributes')
        # Check time_coverage_duration and resolution
        for attr in ['time_coverage_duration', 'time_coverage_resolution']:
            attr_value = getattr(dataset, attr, '')
            try:
                parse_duration(attr_value)
                recommended_ctx.assert_true(True, '')  # Score it True!
            except Exception:
                recommended_ctx.assert_true(False, '{} should exist and be ISO-8601 format (example: PT1M30S), currently: {}'.format(attr, attr_value))
        results.append(recommended_ctx.to_result())
        return results
711568
def id_match_check(self, data_df, meta_df, dim):
        
        if dim == "row":
            if len(data_df.index) == len(meta_df.index) and set(data_df.index) == set(meta_df.index):
                return True
            else:
                msg = ("The rids are inconsistent between data_df and row_metadata_df.\n" +
                 "data_df.index.values:\n{}\nrow_metadata_df.index.values:\n{}").format(data_df.index.values, meta_df.index.values)
                self.logger.error(msg)
                raise Exception("GCToo GCToo.id_match_check " + msg)
        elif dim == "col":
            if len(data_df.columns) == len(meta_df.index) and set(data_df.columns) == set(meta_df.index):
                return True
            else:
                msg = ("The cids are inconsistent between data_df and col_metadata_df.\n" +
                 "data_df.columns.values:\n{}\ncol_metadata_df.index.values:\n{}").format(data_df.columns.values, meta_df.index.values)
                self.logger.error(msg)
                raise Exception("GCToo GCToo.id_match_check " + msg)
1148304
def set_features_types_from_dataframe(self, data_frame):
        
        if self.__feature_types_set:
            return
        self.__feature_types_set = True

        dtypes = data_frame.dtypes
        for feature in self.__iter__():
            name = feature.get_name()
            type_name = data_type_to_type_name(dtypes[name])
            feature.set_type_name(type_name)
782918
def available_datasets(self):
        """"""
        res = self.resolution
        coordinates = ['pixel_longitude', 'pixel_latitude']
        for var_name, val in self.file_content.items():
            if isinstance(val, netCDF4.Variable):
                ds_info = {
                    'file_type': self.filetype_info['file_type'],
                    'resolution': res,
                }
                if not self.is_geo:
                    ds_info['coordinates'] = coordinates
                yield DatasetID(name=var_name, resolution=res), ds_info
446661
def _maybe_pandas_data(data, feature_names, feature_types):
    

    if not isinstance(data, DataFrame):
        return data, feature_names, feature_types

    data_dtypes = data.dtypes
    if not all(dtype.name in PANDAS_DTYPE_MAPPER for dtype in data_dtypes):
        bad_fields = [data.columns[i] for i, dtype in
                      enumerate(data_dtypes) if dtype.name not in PANDAS_DTYPE_MAPPER]

        msg = """DataFrame.dtypes for data must be int, float or bool.
                Did not expect the data types in fields """
        raise ValueError(msg + ', '.join(bad_fields))

    if feature_names is None:
        if isinstance(data.columns, MultiIndex):
            feature_names = [
                ' '.join([str(x) for x in i])
                for i in data.columns
            ]
        else:
            feature_names = data.columns.format()

    if feature_types is None:
        feature_types = [PANDAS_DTYPE_MAPPER[dtype.name] for dtype in data_dtypes]

    data = data.values.astype('float')

    return data, feature_names, feature_types
535278
def observe(self, dataset, kind, components=None, distance=1.0, l3=0.0, **kwargs):
        

        meshes = self.meshes
        if kind=='lp':
            def sv(p, p0, w):
                # Subsidiary variable:
                return (p0-p)/(w/2)

            def lorentzian(sv):
                return 1-1./(1+sv**2)

            def gaussian(sv):
                return 1-np.exp(-np.log(2)*sv**2)

            profile_func = kwargs.get('profile_func')
            profile_rest = kwargs.get('profile_rest')
            profile_sv = kwargs.get('profile_sv')
            wavelengths = kwargs.get('wavelengths')
            if profile_func == 'gaussian':
                func = gaussian
            elif profile_func == 'lorentzian':
                func = lorentzian
            else:
                raise NotImplementedError("profile_func='{}' not supported".format(profile_func))

            visibilities = meshes.get_column_flat('visibilities', components)

            abs_intensities = meshes.get_column_flat('abs_intensities:{}'.format(dataset), components)
            # mus here will be from the tnormals of the triangle and will not
            # be weighted by the visibility of the triangle
            mus = meshes.get_column_flat('mus', components)
            areas = meshes.get_column_flat('areas_si', components)
            ldint = meshes.get_column_flat('ldint:{}'.format(dataset), components)

            rvs = (meshes.get_column_flat("rvs:{}".format(dataset), components)*u.solRad/u.d).to(u.m/u.s).value
            dls = rvs*profile_rest/c.c.si.value

            line = func(sv(wavelengths, profile_rest, profile_sv))
            lines = np.array([np.interp(wavelengths, wavelengths+dl, line) for dl in dls])
            if not np.any(visibilities):
                avg_line = np.full_like(wavelengths, np.nan)
            else:
                avg_line = np.average(lines, axis=0, weights=abs_intensities*areas*mus*ldint*visibilities)

            return {'flux_densities': avg_line}


        elif kind=='rv':
            visibilities = meshes.get_column_flat('visibilities', components)

            if np.all(visibilities==0):
                # then no triangles are visible, so we should return nan
                return {'rv': np.nan}

            rvs = meshes.get_column_flat("rvs:{}".format(dataset), components)
            abs_intensities = meshes.get_column_flat('abs_intensities:{}'.format(dataset), components)
            # mus here will be from the tnormals of the triangle and will not
            # be weighted by the visibility of the triangle
            mus = meshes.get_column_flat('mus', components)
            areas = meshes.get_column_flat('areas_si', components)
            ldint = meshes.get_column_flat('ldint:{}'.format(dataset), components)
            # NOTE: don't need ptfarea because its a float (same for all
            # elements, regardless of component)

            # NOTE: the intensities are already projected but are per unit area
            # so we need to multiply by the /projected/ area of each triangle (thus the extra mu)
            return {'rv': np.average(rvs, weights=abs_intensities*areas*mus*ldint*visibilities)}

        elif kind=='lc':
            visibilities = meshes.get_column_flat('visibilities')

            if np.all(visibilities==0):
                # then no triangles are visible, so we should return nan -
                # probably shouldn't ever happen for lcs
                return {'flux': np.nan}

            intensities = meshes.get_column_flat("intensities:{}".format(dataset), components)
            mus = meshes.get_column_flat('mus', components)
            areas = meshes.get_column_flat('areas_si', components)
            ldint = meshes.get_column_flat('ldint:{}'.format(dataset), components)

            # assume that all bodies are using the same passband and therefore
            # will have the same ptfarea.  If this assumption is ever a problem -
            # then we will need to build a flat column based on the component
            # of each element so that ptfarea is an array with the same shape
            # as those above
            if isinstance(self.bodies[0], Envelope):
                # for envelopes, we'll make the same assumption and just grab
                # that value stored in the first "half"
                ptfarea = self.bodies[0]._halves[0].get_ptfarea(dataset)
            else:
                ptfarea = self.bodies[0].get_ptfarea(dataset)

            # intens_proj is the intensity in the direction of the observer per unit surface area of the triangle
            # areas is the area of each triangle
            # areas*mus is the area of each triangle projected in the direction of the observer
            # visibilities is 0 for hidden, 0.5 for partial, 1.0 for visible
            # areas*mus*visibilities is the visibile projected area of each triangle (ie half the area for a partially-visible triangle)
            # so, intens_proj*areas*mus*visibilities is the intensity in the direction of the observer per the observed projected area of that triangle
            # and the sum of these values is the observed flux

            # note that the intensities are already projected but are per unit area
            # so we need to multiply by the /projected/ area of each triangle (thus the extra mu)

            return {'flux': np.sum(intensities*areas*mus*visibilities)*ptfarea/(distance**2)+l3}

        else:
            raise NotImplementedError("observe for dataset with kind '{}' not implemented".format(kind))
521789
public static List<List<Writable>> toRecords(DataSet dataSet) {
        if (isClassificationDataSet(dataSet)) {
            return getClassificationWritableMatrix(dataSet);
        } else {
            return getRegressionWritableMatrix(dataSet);
        }
    }
1173557
public void fit(DataSet dataSet) {
        if (numInputArrays != 1 || numOutputArrays != 1)
            throw new UnsupportedOperationException("Cannot train ComputationGraph network with "
                    + " multiple inputs or outputs using a DataSet");

        boolean hasMaskArrays = dataSet.hasMaskArrays();
        if (hasMaskArrays) {
            INDArray[] fMask = (dataSet.getFeaturesMaskArray() != null ? new INDArray[]{dataSet.getFeaturesMaskArray()}
                    : null);
            INDArray[] lMask = (dataSet.getLabelsMaskArray() != null ? new INDArray[]{dataSet.getLabelsMaskArray()}
                    : null);
            fit(new INDArray[]{dataSet.getFeatures()}, new INDArray[]{dataSet.getLabels()}, fMask, lMask);
        } else {
            fit(new INDArray[]{dataSet.getFeatures()}, new INDArray[]{dataSet.getLabels()});
        }

        if (hasMaskArrays)
            clearLayerMaskArrays();

        clearLayersStates();
    }
1030484
def _validate_indexers(
        self, indexers: Mapping,
    ) -> List[Tuple[Any, Union[slice, Variable]]]:
        
        from .dataarray import DataArray

        invalid = [k for k in indexers if k not in self.dims]
        if invalid:
            raise ValueError("dimensions %r do not exist" % invalid)

        # all indexers should be int, slice, np.ndarrays, or Variable
        indexers_list = []  # type: List[Tuple[Any, Union[slice, Variable]]]
        for k, v in indexers.items():
            if isinstance(v, slice):
                indexers_list.append((k, v))
                continue

            if isinstance(v, Variable):
                pass
            elif isinstance(v, DataArray):
                v = v.variable
            elif isinstance(v, tuple):
                v = as_variable(v)
            elif isinstance(v, Dataset):
                raise TypeError('cannot use a Dataset as an indexer')
            elif isinstance(v, Sequence) and len(v) == 0:
                v = IndexVariable((k, ), np.zeros((0,), dtype='int64'))
            else:
                v = np.asarray(v)

                if v.dtype.kind == 'U' or v.dtype.kind == 'S':
                    index = self.indexes[k]
                    if isinstance(index, pd.DatetimeIndex):
                        v = v.astype('datetime64[ns]')
                    elif isinstance(index, xr.CFTimeIndex):
                        v = _parse_array_of_cftime_strings(v, index.date_type)

                if v.ndim == 0:
                    v = Variable((), v)
                elif v.ndim == 1:
                    v = IndexVariable((k,), v)
                else:
                    raise IndexError(
                        "Unlabeled multi-dimensional array cannot be "
                        "used for indexing: {}".format(k))

            if v.ndim == 1:
                v = v.to_index_variable()

            indexers_list.append((k, v))

        return indexers_list
1041962
def check_dimensions(self, dataset):
        '''
        
        '''
        required_ctx = TestCtx(BaseCheck.HIGH, 'All geophysical variables are time-series incomplete feature types')
        message = '{} must be a valid timeseries feature type. It must have dimensions of (timeSeries, time).'
        message += ' And all coordinates must have dimensions of (timeSeries)'
        for variable in util.get_geophysical_variables(dataset):
            is_valid = util.is_multi_timeseries_incomplete(dataset, variable)
            required_ctx.assert_true(
                is_valid,
                message.format(variable)
            )
        return required_ctx.to_result()
813785
def _validate_dataset(dataset):
    
    if not (isinstance(dataset, _SFrame)):
        raise TypeError("Input 'dataset' must be an SFrame.")

    if dataset.num_rows() == 0 or dataset.num_columns() == 0:
        raise ValueError("Input 'dataset' has no data.")
628731
def _validate_slices_form_uniform_grid(slice_datasets):
    '''
    
    '''
    invariant_properties = [
        'Modality',
        'SOPClassUID',
        'SeriesInstanceUID',
        'Rows',
        'Columns',
        'PixelSpacing',
        'PixelRepresentation',
        'BitsAllocated',
        'BitsStored',
        'HighBit',
    ]

    for property_name in invariant_properties:
        _slice_attribute_equal(slice_datasets, property_name)

    _validate_image_orientation(slice_datasets[0].ImageOrientationPatient)
    _slice_ndarray_attribute_almost_equal(slice_datasets, 'ImageOrientationPatient', 1e-5)

    slice_positions = _slice_positions(slice_datasets)
    _check_for_missing_slices(slice_positions)
122113
def determine_endpoint_type(features):
        
    counter={k.name: v for k, v in features.columns.to_series().groupby(features.dtypes).groups.items()}
    if(len(features.groupby('class').apply(list))==2):
        return('binary')
    if ('float64' in counter):
        return ('float')
    return ('integer')
542001
def validate_metadata(train_config):
  

  # Make sure we have a default for every column
  if len(train_config['csv_header']) != len(train_config['csv_defaults']):
    raise ValueError('Unequal number of columns in input features file and '
                     'schema file.')

  # Check there are no missing columns. sorted_colums has two copies of the
  # target column because the target column is also listed in
  # categorical_columns or numerical_columns.
  sorted_columns = sorted(train_config['csv_header'] +
                          [train_config['target_column']])

  sorted_columns2 = sorted(train_config['categorical_columns'] +
                           train_config['numerical_columns'] +
                           [train_config['key_column']] +
                           [train_config['target_column']])
  if sorted_columns2 != sorted_columns:
    raise ValueError('Each csv header must be a numerical/categorical type, a '
                     ' key, or a target.')