Convert descriptor and rows to Pandas
328122
def convert_descriptor_and_rows(self, descriptor, rows):
        

        # Prepare
        primary_key = None
        schema = tableschema.Schema(descriptor)
        if len(schema.primary_key) == 1:
            primary_key = schema.primary_key[0]
        elif len(schema.primary_key) > 1:
            message = 'Multi-column primary keys are not supported'
            raise tableschema.exceptions.StorageError(message)

        # Get data/index
        data_rows = []
        index_rows = []
        jtstypes_map = {}
        for row in rows:
            values = []
            index = None
            for field, value in zip(schema.fields, row):
                try:
                    if isinstance(value, float) and np.isnan(value):
                        value = None
                    if value and field.type == 'integer':
                        value = int(value)
                    value = field.cast_value(value)
                except tableschema.exceptions.CastError:
                    value = json.loads(value)
                # http://pandas.pydata.org/pandas-docs/stable/gotchas.html#support-for-integer-na
                if value is None and field.type in ('number', 'integer'):
                    jtstypes_map[field.name] = 'number'
                    value = np.NaN
                if field.name == primary_key:
                    index = value
                else:
                    values.append(value)
            data_rows.append(tuple(values))
            index_rows.append(index)

        # Get dtypes
        dtypes = []
        for field in schema.fields:
            if field.name != primary_key:
                field_name = field.name
                if six.PY2:
                    field_name = field.name.encode('utf-8')
                dtype = self.convert_type(jtstypes_map.get(field.name, field.type))
                dtypes.append((field_name, dtype))

        # Create dataframe
        index = None
        columns = schema.headers
        array = np.array(data_rows, dtype=dtypes)
        if primary_key:
            index_field = schema.get_field(primary_key)
            index_dtype = self.convert_type(index_field.type)
            index_class = pd.Index
            if index_field.type in ['datetime', 'date']:
                index_class = pd.DatetimeIndex
            index = index_class(index_rows, name=primary_key, dtype=index_dtype)
            columns = filter(lambda column: column != primary_key, schema.headers)
        dataframe = pd.DataFrame(array, index=index, columns=columns)

        return dataframe
235163
def to_csv(self, dest:str)->None:
        ""
        self.to_df().to_csv(self.path/dest, index=False)
1316826
def _vertex_list_to_dataframe(ls, id_column_name):
    
    assert HAS_PANDAS, 'Cannot use dataframe because Pandas is not available or version is too low.'
    cols = reduce(set.union, (set(v.attr.keys()) for v in ls))
    df = pd.DataFrame({id_column_name: [v.vid for v in ls]})
    for c in cols:
        df[c] = [v.attr.get(c) for v in ls]
    return df
409256
def to_pandas_dataframe(self):
        
        pd_index = self.index().to_pandas_index()
        return pd.DataFrame.from_items(self.collect()).set_index(pd_index)
281916
def to_dataframe(self):
        
        keys = self.data[0].keys()
        column_list =[]
        for k in keys:
            key_list = []
            for i in xrange(0,len(self.data)):
                key_list.append(self.data[i][k])
            column_list.append(key_list)
        df = DataFrame(np.asarray(column_list).transpose(), columns=keys)
        for i in xrange(0,df.shape[1]):
            if is_number(df.iloc[:,i]):
                df.iloc[:,i] = df.iloc[:,i].astype(float)
        return df
1034692
def to_dataframe(self):
        """"""
        def row_from_variant(variant):
            return OrderedDict([
                ("chr", variant.contig),
                ("start", variant.original_start),
                ("ref", variant.original_ref),
                ("alt", variant.original_alt),
                ("gene_name", ";".join(variant.gene_names)),
                ("gene_id", ";".join(variant.gene_ids))
            ])
        rows = [row_from_variant(v) for v in self]
        if len(rows) == 0:
            # TODO: return a DataFrame with the appropriate columns
            return pd.DataFrame()
        return pd.DataFrame.from_records(rows, columns=rows[0].keys())
230773
def NdarrayToEntry(self, x):
    """"""
    row_counts = []
    for row in x:
      try:
        rc = np.count_nonzero(~np.isnan(row))
        if rc != 0:
          row_counts.append(rc)
      except TypeError:
        try:
          row_counts.append(row.size)
        except AttributeError:
          row_counts.append(1)

    data_type = self.DtypeToType(x.dtype)
    converter = self.DtypeToNumberConverter(x.dtype)
    flattened = x.ravel()
    orig_size = len(flattened)

    # Remove all None and nan values and count how many were removed.
    flattened = flattened[flattened != np.array(None)]
    if converter:
      flattened = converter(flattened)
    if data_type == self.fs_proto.STRING:
      flattened_temp = []
      for x in flattened:
        try:
          if str(x) != 'nan':
            flattened_temp.append(x)
        except UnicodeEncodeError:
          if x.encode('utf-8') != 'nan':
            flattened_temp.append(x)
      flattened = flattened_temp
    else:
      flattened = flattened[~np.isnan(flattened)].tolist()
    missing = orig_size - len(flattened)
    return {
        'vals': flattened,
        'counts': row_counts,
        'missing': missing,
        'type': data_type
    }
945979
def caps_str_to_df(caps_str):
    '''
    
    '''
    structures = [dict([process_dict(v.groupdict())
                   for v in re.finditer(
                       r'(?P<key>[^ ]*)=\((?P<type>.*?)\)((?P<value>[^{].*?)|{(?P<values>.*?)})(,|$)', s)])
                  for s in caps_str.split(';')]

    df = pd.DataFrame(structures)
    df.reset_index(drop=True, inplace=True)

    def compute_multi(df):
        multi_values = [[(c, k) for k, v in df[c].iteritems()
                         if isinstance(v, list)] for c in df]
        value_lists = [m for m in multi_values if m]
        if value_lists:
            return pd.DataFrame(np.concatenate(value_lists),
                             columns=['label', 'index'])
        else:
            return pd.DataFrame()

    df_multi = compute_multi(df)

    while df_multi.shape[0] > 0:
        df = resolve_multi(df, df_multi)
        df_multi = compute_multi(df)

    if 'framerate' in df:
        df['framerate_numerator'] = df['framerate'].map(lambda v: v.num)
        df['framerate_denominator'] = df['framerate'].map(lambda v: v.denom)
        df['framerate'] = df['framerate_numerator'] / df['framerate_denominator']
    if 'pixel-aspect-ratio' in df:
        df['pixel-aspect-ratio_numerator'] = df['pixel-aspect-ratio'].map(lambda v: v.num)
        df['pixel-aspect-ratio_denominator'] = df['pixel-aspect-ratio'].map(lambda v: v.denom)
        df['pixel-aspect-ratio'] = (df['pixel-aspect-ratio_numerator'] /
                                    df['pixel-aspect-ratio_numerator'])

    return df.sort(['framerate', 'width', 'height']).reset_index(drop=True)
446661
def _maybe_pandas_data(data, feature_names, feature_types):
    

    if not isinstance(data, DataFrame):
        return data, feature_names, feature_types

    data_dtypes = data.dtypes
    if not all(dtype.name in PANDAS_DTYPE_MAPPER for dtype in data_dtypes):
        bad_fields = [data.columns[i] for i, dtype in
                      enumerate(data_dtypes) if dtype.name not in PANDAS_DTYPE_MAPPER]

        msg = """DataFrame.dtypes for data must be int, float or bool.
                Did not expect the data types in fields """
        raise ValueError(msg + ', '.join(bad_fields))

    if feature_names is None:
        if isinstance(data.columns, MultiIndex):
            feature_names = [
                ' '.join([str(x) for x in i])
                for i in data.columns
            ]
        else:
            feature_names = data.columns.format()

    if feature_types is None:
        feature_types = [PANDAS_DTYPE_MAPPER[dtype.name] for dtype in data_dtypes]

    data = data.values.astype('float')

    return data, feature_names, feature_types
763755
def decode_timeseries_row(self, tsrow, tscols=None,
                              convert_timestamp=False):
        
        row = []
        for i, cell in enumerate(tsrow.cells):
            col = None
            if tscols is not None:
                col = tscols[i]
            if cell.HasField('varchar_value'):
                if col and not (col.type == TsColumnType.Value('VARCHAR') or
                                col.type == TsColumnType.Value('BLOB')):
                    raise TypeError('expected VARCHAR or BLOB column')
                else:
                    row.append(cell.varchar_value)
            elif cell.HasField('sint64_value'):
                if col and col.type != TsColumnType.Value('SINT64'):
                    raise TypeError('expected SINT64 column')
                else:
                    row.append(cell.sint64_value)
            elif cell.HasField('double_value'):
                if col and col.type != TsColumnType.Value('DOUBLE'):
                    raise TypeError('expected DOUBLE column')
                else:
                    row.append(cell.double_value)
            elif cell.HasField('timestamp_value'):
                if col and col.type != TsColumnType.Value('TIMESTAMP'):
                    raise TypeError('expected TIMESTAMP column')
                else:
                    dt = cell.timestamp_value
                    if convert_timestamp:
                        dt = datetime_from_unix_time_millis(
                            cell.timestamp_value)
                    row.append(dt)
            elif cell.HasField('boolean_value'):
                if col and col.type != TsColumnType.Value('BOOLEAN'):
                    raise TypeError('expected BOOLEAN column')
                else:
                    row.append(cell.boolean_value)
            else:
                row.append(None)
        return row
337779
def compounds_to_frame(compounds, properties=None):
    
    import pandas as pd
    if isinstance(compounds, Compound):
        compounds = [compounds]
    properties = set(properties) | set(['cid']) if properties else None
    return pd.DataFrame.from_records([c.to_dict(properties) for c in compounds], index='cid')
79970
def _data_from_df(df):
        ''' 

        '''
        _df = df.copy()

        # Flatten columns
        if isinstance(df.columns, pd.MultiIndex):
            try:
                _df.columns = ['_'.join(col) for col in _df.columns.values]
            except TypeError:
                raise TypeError('Could not flatten MultiIndex columns. '
                                'use string column names or flatten manually')
        # Transform columns CategoricalIndex in list
        if isinstance(df.columns, pd.CategoricalIndex):
            _df.columns = df.columns.tolist()
        # Flatten index
        index_name = ColumnDataSource._df_index_name(df)
        if index_name == 'index':
            _df.index = pd.Index(_df.index.values)
        else:
            _df.index = pd.Index(_df.index.values, name=index_name)
        _df.reset_index(inplace=True)

        tmp_data = {c: v.values for c, v in _df.iteritems()}

        new_data = {}
        for k, v in tmp_data.items():
            new_data[k] = v

        return new_data
808060
def to_dataframe(self, columns=None):
    """"""
    if not self.is_value_set:
      raise ValueError('Value must be set before converting to a DataFrame.')
    if not pandas:
      raise RuntimeError('Install pandas to convert to pandas.DataFrame')
    return pandas.DataFrame.from_records(self.value, columns=columns)
354642
def encode_df(df, data_types={}):
    

    numbers = []
    categories = []
    datetimes = []
        
    for column, series in df.iteritems():
        if column in data_types:
            dtypes = data_types[column]
        else:
            dtypes = {}
        
        if 'datetime' in dtypes:
            index = pd.DatetimeIndex(pd.to_datetime(series.values))
            df[column] = index.astype(np.int64).astype(float)
            datetimes.append(column)
            
        elif any([dtype in dtypes for dtype in ['category', 'label']]):
            # If this can be treated as a category, encode it
            df = pd.get_dummies(df,columns=[column])
            categories.append(column)
                    
        elif 'numeric' in dtypes:
            df[column] = clean_numbers(df[column].values)
            df[column] = df[column].astype(float)
            numbers.append(column)
            
        else:
            df.drop(column,1,inplace=True)

    # Scale continuous columns
    scaler = MinMaxScaler()
    df[numbers + datetimes] = scaler.fit_transform(df[numbers + datetimes])
    
    return df, categories, numbers, datetimes
763143
def to_df(obj, cfg={}, raise_on_error=True, silent=False, verbose=False):
    
    
    if type(obj) == str:
        name = obj
    else:
        name = obj.name
    name = name[name.rfind('/') + 1:]
    
    if not raise_on_error:
        try:
            return to_df(obj=obj, cfg=cfg, raise_on_error=True)
        except (pd.parser.CParserError, AttributeError, ValueError, TypeError, IOError) as e:
            if not silent:
                print('WARNING in {}: {} {}'.format(name, e.__class__, e))
            return None
        except:
            if not silent:
                print('WARNING in {}: {}'.format(name, sys.exc_info()[0]))
            return None

    params = {}
    if 'default' in cfg:
        params = deepcopy(cfg['default'])
    if name in cfg:
        for e in cfg[name]:
            params[e] = deepcopy(cfg[name][e])
    if 'custom_date_parser' in params:
        params['date_parser'] = DtParser(params['custom_date_parser']).parse
        del params['custom_date_parser']

    if verbose:
        print(name, params)
    
    if name.lower().startswith('s3:'):
        if not find_spec('boto'):
            raise ImportError('reading from aws-s3 requires the boto package to be installed')
    
    if '.csv' in name.lower():
        # name can be .csv.gz or .csv.bz2
        return pd.read_csv(obj, **params)
        
    elif '.tsv' in name.lower() or '.txt' in name.lower():
        # name can be .tsv.gz or .txt.bz2
        return pd.read_table(obj, **params)
    
    elif name.lower().endswith(('.htm', '.html')):
        if not find_spec('lxml'):
            params['flavor'] = 'bs4'
            if not find_spec('bs4') and not find_spec('html5lib'):
                raise ImportError('reading html requires the lxml or bs4 + html5lib packages to be installed')

        if 'nrows' in params:
            del params['nrows']
        
        if type(obj) == zipfile.ZipExtFile:
            obj = obj.read()
        data = pd.read_html(obj, **params)
        data = {str(i): data[i] for i in range(len(data))}
        return data
    
    elif name.lower().endswith('.xml'):
        if 'nrows' in params:
            del params['nrows']
        
        from icy.utils import xml_to_json
        
        with open(obj) as f:
            json = xml_to_json(str(f.read()))
        
        return pd.read_json(json, **params)
        
    elif name.lower().endswith('.json'):
        if 'nrows' in params:
            del params['nrows']
        
        return pd.read_json(obj, **params)
    
    elif name.lower().endswith(('.xls', '.xlsx')):
        if not find_spec('xlrd'):
            raise ImportError('reading excel files requires the xlrd package to be installed')
        
        if 'nrows' in params:
            del params['nrows']
        
        data = {}
        xls = pd.ExcelFile(obj)
        for key in xls.sheet_names:
            data[key] = xls.parse(key, **params)
        return data
    
    elif name.lower().endswith(('.h5', '.hdf5')):
        if not find_spec('tables'):
            raise ImportError('reading hdf5 files requires the pytables package to be installed')
        
        if 'nrows' in params:
            del params['nrows']
            # params['chunksize'] = params.pop('nrows') # returns iterator
        
        with pd.HDFStore(obj) as store:
            data = {}
            for key in store.keys():
                data[key[1:]] = store[key]
        return data
    
    elif name.lower().endswith(('.sqlite', '.sql', '.db')):
        import sqlite3
        if type(obj) != str:
            raise IOError('sqlite-database must be decompressed before import')
        
        if 'nrows' in params:
            del params['nrows']
            # params['chunksize'] = params.pop('nrows') # returns iterator
        
        with sqlite3.connect(obj) as con:
            data = {}
            cursor = con.cursor()
            cursor.execute('SELECT name FROM sqlite_master WHERE type="table"')
            tables = [t[0] for t in cursor.fetchall()]
            for t in tables:
                sql = 'SELECT * FROM ' + t
                data[t] = pd.read_sql_query(sql, con, **params)
        return data
    
    else:
        try:
            data = {name: odo(obj, pd.DataFrame)}
            if type(data[name]) == pd.DataFrame:
                return data
        except NotImplementedError:
            pass
        raise NotImplementedError('Error creating DataFrame from object', obj)
188030
def convert_descriptor(self, descriptor):
        

        # Fields
        fields = []
        fallbacks = []
        schema = tableschema.Schema(descriptor)
        for index, field in enumerate(schema.fields):
            converted_type = self.convert_type(field.type)
            if not converted_type:
                converted_type = 'STRING'
                fallbacks.append(index)
            mode = 'NULLABLE'
            if field.required:
                mode = 'REQUIRED'
            fields.append({
                'name': _slugify_field_name(field.name),
                'type': converted_type,
                'mode': mode,
            })

        # Descriptor
        converted_descriptor = {
            'fields': fields,
        }

        return (converted_descriptor, fallbacks)
457498
void computeDescriptor(int row, int col) {
		// set location to top-left pixel
		locations.grow().set(col* pixelsPerCell,row* pixelsPerCell);

		TupleDesc_F64 d = descriptions.grow();

		int indexDesc = 0;
		for (int i = 0; i < cellsPerBlockY; i++) {
			for (int j = 0; j < cellsPerBlockX; j++) {
				Cell c = cells[(row+i)*cellCols + (col+j)];

				for (int k = 0; k < c.histogram.length; k++) {
					d.value[indexDesc++] = c.histogram[k];
				}
			}
		}

		// Apply SIFT style L2-Hys normalization
		DescribeSiftCommon.normalizeDescriptor(d,0.2);
	}