Verify consistency of dataframes that pass through this stream
549473
def verify(self, x):
        
        if type(x) != type(self.data):
            raise TypeError("Input expected to be of type %s, got %s." %
                            (type(self.data).__name__, type(x).__name__))
        elif isinstance(x, np.ndarray):
            if x.ndim != 2:
                raise ValueError('Streamed array data must be two-dimensional')
            elif x.shape[1] != self.data.shape[1]:
                raise ValueError("Streamed array data expeced to have %d columns, "
                                 "got %d." % (self.data.shape[1], x.shape[1]))
        elif util.pd and isinstance(x, util.pd.DataFrame) and list(x.columns) != list(self.data.columns):
            raise IndexError("Input expected to have columns %s, got %s" %
                             (list(self.data.columns), list(x.columns)))
        elif isinstance(x, dict):
            if any(c not in x for c in self.data):
                raise IndexError("Input expected to have columns %s, got %s" %
                                 (sorted(self.data.keys()), sorted(x.keys())))
            elif len(set(len(v) for v in x.values())) > 1:
                raise ValueError("Input columns expected to have the "
                                 "same number of rows.")
1367824
def check_dataframe(self, dataframe):
        
        if dataframe is None:
            return None

        # is this a DataFrame
        if not isinstance(dataframe, pd.DataFrame):
            raise Exception(
                'The provided dataframe object is not a pandas.DataFrame'
            )

        for column in self.required_columns:
            if column not in dataframe:
                raise Exception('Required column not in dataframe: {0}'.format(
                    column
                ))
        return dataframe
1171185
def is_consistent(self) -> bool:
        
        from ledger.compact_merkle_tree import CompactMerkleTree
        return self.nodeCount == CompactMerkleTree.get_expected_node_count(
            self.leafCount)
914450
def check_transfers(
    feed: "Feed", *, as_df: bool = False, include_warnings: bool = False
) -> List:
    
    table = "transfers"
    problems = []

    # Preliminary checks
    if feed.transfers is None:
        return problems

    f = feed.transfers.copy()
    problems = check_for_required_columns(problems, table, f)
    if problems:
        return format_problems(problems, as_df=as_df)

    if include_warnings:
        problems = check_for_invalid_columns(problems, table, f)

    # Check from_stop_id and to_stop_id
    for col in ["from_stop_id", "to_stop_id"]:
        problems = check_column_linked_id(
            problems, table, f, col, feed.stops, "stop_id"
        )

    # Check transfer_type
    v = lambda x: pd.isnull(x) or x in range(5)
    problems = check_column(
        problems, table, f, "transfer_type", v, column_required=False
    )

    # Check min_transfer_time
    v = lambda x: x >= 0
    problems = check_column(
        problems, table, f, "min_transfer_time", v, column_required=False
    )

    return format_problems(problems, as_df=as_df)
939758
private void verifyChunkedSums(
      byte[] data, int dataOff, int dataLen,
      byte[] checksums, int checksumsOff, String fileName,
      long basePos) throws ChecksumException {
    
    int remaining = dataLen;
    int dataPos = 0;
    while (remaining > 0) {
      int n = Math.min(remaining, bytesPerChecksum);
      
      summer.reset();
      summer.update(data, dataOff + dataPos, n);
      dataPos += n;
      remaining -= n;
      
      int calculated = (int)summer.getValue();
      int stored = (checksums[checksumsOff] << 24 & 0xff000000) |
        (checksums[checksumsOff + 1] << 16 & 0xff0000) |
        (checksums[checksumsOff + 2] << 8 & 0xff00) |
        checksums[checksumsOff + 3] & 0xff;
      checksumsOff += 4;
      if (calculated != stored) {
        long errPos = basePos + dataPos - n;
        throw new ChecksumException(
            "Checksum error: "+ fileName + " at "+ errPos +
            " exp: " + stored + " got: " + calculated, errPos);
      }
    }
  }
745645
def trainOn(self, dstream):
        """"""
        self._validate(dstream)

        def update(rdd):
            self._model.update(rdd, self._decayFactor, self._timeUnit)

        dstream.foreachRDD(update)
78609
def _check_values(self, values):
        """"""
        assert isinstance(values, Iterable) and not \
            isinstance(values, (str, dict, bytes, bytearray)), \
            'values should be a list or tuple. Got {}'.format(type(values))
        assert len(values) == len(self.datetimes), \
            'Length of values list must match length of datetimes list. {} != {}'.format(
                len(values), len(self.datetimes))
        assert len(values) > 0, 'Data Collection must include at least one value'
1370099
def verify_chunks(self, chunks):
        '''
        
        '''
        err = []
        for chunk in chunks:
            err.extend(self.verify_data(chunk))
        return err
450511
def raw(self, drop_collections = False):
        
        base_df = self._data
        if drop_collections is True:
            out_df = self._drop_collections(base_df)
        else:
            out_df = base_df
        return out_df
872241
protected void _verifyAllStreamsImpl() {
		// This method checks that the streams are of correct size.
		// It resizes the streams to ensure they are not shorter than
		// m_PointCount
		// _ASSERT(_HasDirtyFlag(enum_value1(DirtyFlags,
		// DirtyVerifiedStreams)));
		if (m_reservedPointCount < m_pointCount) // an optimization to skip this
													// expensive loop when
													// adding point by point
		{
			if (m_vertexAttributes == null)
				m_vertexAttributes = new AttributeStreamBase[m_description
						.getAttributeCount()];

			m_reservedPointCount = NumberUtils.intMax();
			for (int attributeIndex = 0; attributeIndex < m_description
					.getAttributeCount(); attributeIndex++) {
				int semantics = m_description._getSemanticsImpl(attributeIndex);
				if (m_vertexAttributes[attributeIndex] != null) {
					int ncomp = VertexDescription.getComponentCount(semantics);
					int size = m_vertexAttributes[attributeIndex].virtualSize()
							/ ncomp;
					if (size < m_pointCount) {
						size = (m_reservedPointCount > m_pointCount + 5) ? (m_pointCount * 5 + 3) / 4
								: m_pointCount;// reserve 25% more than user
												// asks
						m_vertexAttributes[attributeIndex].resize(size * ncomp,
								VertexDescription.getDefaultValue(semantics));
					}

					if (size < m_reservedPointCount)
						m_reservedPointCount = size;
				} else {
					m_vertexAttributes[attributeIndex] = AttributeStreamBase
							.createAttributeStreamWithSemantics(semantics,
									m_pointCount);
					m_reservedPointCount = m_pointCount;
				}
			}
		}
		_verifyStreamsImpl();

		_setDirtyFlag(DirtyFlags.DirtyVerifiedStreams, false);
	}
632063
def broadcast(df):
    """"""

    sc = SparkContext._active_spark_context
    return DataFrame(sc._jvm.functions.broadcast(df._jdf), df.sql_ctx)
110529
def get_valid_cell_indecies(self):
        
        return pd.DataFrame(self).groupby(self.frame_columns).apply(lambda x: list(x['cell_index'])).\
            reset_index().rename(columns={0:'valid'})
1161195
def verify(self):
        
        collections = self.get_data_collections()

        allIDs = []
        for coll in collections:
            IDs = self.noteDB[coll].find({"ID": {"$exists": True}},
                                         {"ID": 1, "_id": 0})
            for ID in IDs:
                allIDs.append(int(ID["ID"]))

        query = {"currentMax": {"$exists": True}}
        maxID = int(self.noteDB['IDs'].find_one(query)["currentMax"])
        query = {"unusedIDs": {"$exists": True}}
        unusedIDs = self.noteDB['IDs'].find_one(query)["unusedIDs"]
        unusedIDs = [int(ii) for ii in unusedIDs]

        unusedIDsMatch = True
        for ID in allIDs:
            if ID in unusedIDs:
                unusedIDsMatch = False

        maxIDMatch = True
        if maxID is not max(allIDs):
            maxIDMatch = False

        if maxIDMatch and unusedIDsMatch:
            print "Database is valid"
        elif not maxIDMatch and not unusedIDsMatch:
            print "Database not valid, max ID and unused IDs are incorrent"
        elif not maxIDMatch:
            print "Database not valid, max ID is incorrent"
        elif not unusedIDsMatch:
            print "Database not valid, unusedIDs is incorrent"
896784
def _has_data(self):
        """"""
        return any([
            len([
                v for a in (s[0] if is_list_like(s) else [s])
                for v in (a if is_list_like(a) else [a]) if v is not None
            ]) for s in self.raw_series
        ])
1148452
def verify_data(self, data):
        '''
        
        '''
        errors = []
        if 'state' not in data:
            errors.append('Missing "state" data')
        if 'fun' not in data:
            errors.append('Missing "fun" data')
        if 'name' not in data:
            errors.append('Missing "name" data')
        if data['name'] and not isinstance(data['name'], six.string_types):
            errors.append(
                'ID \'{0}\' {1}is not formed as a string, but is a {2}'.format(
                    data['name'],
                    'in SLS \'{0}\' '.format(data['__sls__'])
                        if '__sls__' in data else '',
                    type(data['name']).__name__
                )
            )
        if errors:
            return errors
        full = data['state'] + '.' + data['fun']
        if full not in self.states:
            if '__sls__' in data:
                errors.append(
                    'State \'{0}\' was not found in SLS \'{1}\''.format(
                        full,
                        data['__sls__']
                        )
                    )
                reason = self.states.missing_fun_string(full)
                if reason:
                    errors.append('Reason: {0}'.format(reason))
            else:
                errors.append(
                        'Specified state \'{0}\' was not found'.format(
                            full
                            )
                        )
        else:
            # First verify that the parameters are met
            aspec = salt.utils.args.get_function_argspec(self.states[full])
            arglen = 0
            deflen = 0
            if isinstance(aspec.args, list):
                arglen = len(aspec.args)
            if isinstance(aspec.defaults, tuple):
                deflen = len(aspec.defaults)
            for ind in range(arglen - deflen):
                if aspec.args[ind] not in data:
                    errors.append(
                        'Missing parameter {0} for state {1}'.format(
                            aspec.args[ind],
                            full
                        )
                    )
        # If this chunk has a recursive require, then it will cause a
        # recursive loop when executing, check for it
        reqdec = ''
        if 'require' in data:
            reqdec = 'require'
        if 'watch' in data:
            # Check to see if the service has a mod_watch function, if it does
            # not, then just require
            # to just require extend the require statement with the contents
            # of watch so that the mod_watch function is not called and the
            # requisite capability is still used
            if '{0}.mod_watch'.format(data['state']) not in self.states:
                if 'require' in data:
                    data['require'].extend(data.pop('watch'))
                else:
                    data['require'] = data.pop('watch')
                reqdec = 'require'
            else:
                reqdec = 'watch'
        if reqdec:
            for req in data[reqdec]:
                reqfirst = next(iter(req))
                if data['state'] == reqfirst:
                    if (fnmatch.fnmatch(data['name'], req[reqfirst])
                            or fnmatch.fnmatch(data['__id__'], req[reqfirst])):
                        err = ('Recursive require detected in SLS {0} for'
                               ' require {1} in ID {2}').format(
                                   data['__sls__'],
                                   req,
                                   data['__id__'])
                        errors.append(err)
        return errors
1249942
def verify_consistency(self, expected_leaf_count) -> bool:
        
        if expected_leaf_count != self.leafCount:
            raise ConsistencyVerificationFailed()
        if self.get_expected_node_count(self.leafCount) != self.nodeCount:
            raise ConsistencyVerificationFailed()
        return True
353455
def _check_node_attributes_consistency(self):
        
        # Get list of nodes from the node attributes dict
        nodes_from_attributes = set(self._node_attributes.keys())

        # Perform consistency checks on each node.
        for node in nodes_from_attributes:

            # Check 2.1: make sure that the forward star for the node
            # exists.
            if node not in self._forward_star:
                raise ValueError(
                    'Consistency Check 2.1 Failed: node ' +
                    '%s not in forward star dict' % (str(node)))

            # Check 2.2: make sure that the backward star for the node
            # exists.
            if node not in self._backward_star:
                raise ValueError(
                    'Consistency Check 2.2 Failed: node ' +
                    '%s not in backward star dict' % (str(node)))

            # Get backward star and forward star
            node_fstar = self._forward_star[node]
            node_bstar = self._backward_star[node]

            # Check 2.3: make sure every hyperedge id in the forward
            # star contains the node in the tail
            for hyperedge_id in node_fstar:
                if hyperedge_id not in self._hyperedge_attributes or \
                   node not in \
                   self._hyperedge_attributes[hyperedge_id]['tail']:
                    raise ValueError(
                        'Consistency Check 2.3 Failed: node %s ' % str(node) +
                        'has hyperedge id %s in the forward ' % hyperedge_id +
                        'star, but %s is not in the tail of ' % str(node) +
                        '%s' % hyperedge_id)

            # Check 2.4: make sure every hyperedge id in the backward
            # star contains the node in the head
            for hyperedge_id in node_bstar:
                if hyperedge_id not in self._hyperedge_attributes or \
                   node not in \
                   self._hyperedge_attributes[hyperedge_id]['head']:
                    raise ValueError(
                        'Consistency Check 2.4 Failed: node %s ' % str(node) +
                        'has hyperedge id %s in the forward ' % hyperedge_id +
                        'star, but %s is not in the tail of ' % str(node) +
                        '%s' % hyperedge_id)