Create a Dataframe with the contents of the GCT file
338668
def GCT(gct_obj):
    
    # Handle all the various initialization types and get an IO object
    gct_io = _obtain_io(gct_obj)

    # Load the GCT file into a DataFrame
    df = pd.read_csv(gct_io, sep='\t', header=2, index_col=[0, 1], skip_blank_lines=True)

    # Apply backwards compatible methods
    _apply_backwards_compatibility(df)

    # Return the Dataframe
    return df
28042
def rst_table(data, header=None, fmt=None):
    
    if header is None and hasattr(data, '_fields'):
        header = data._fields
    try:
        # see if data is a composite numpy array
        data.dtype.fields
    except AttributeError:
        # not a composite array
        header = header or ()
    else:
        if not header:
            header = [col.split(':')[0] for col in build_header(data.dtype)]
    if header:
        col_sizes = [len(col) for col in header]
    else:
        col_sizes = [len(str(col)) for col in data[0]]
    body = []
    fmt = functools.partial(scientificformat, fmt=fmt) if fmt else form
    for row in data:
        tup = tuple(fmt(c) for c in row)
        for (i, col) in enumerate(tup):
            col_sizes[i] = max(col_sizes[i], len(col))
        if len(tup) != len(col_sizes):
            raise ValueError('The header has %d fields but the row %d fields!'
                             % (len(col_sizes), len(tup)))
        body.append(tup)

    sepline = ' '.join(('=' * size for size in col_sizes))
    templ = ' '.join(('%-{}s'.format(size) for size in col_sizes))
    if header:
        lines = [sepline, templ % tuple(header), sepline]
    else:
        lines = [sepline]
    for row in body:
        lines.append(templ % row)
    lines.append(sepline)
    return '\n'.join(lines)
516686
def create_info_df(self):
        """"""
        logger.debug("running create_info_df")
        # initializing the reader
        reader = self.reader()
        self.info_df = make_df_from_batch(self.name, batch_col=self.batch_col,
                                          reader=reader)
        logger.debug(str(self.info_df.head(5)))
1300738
def dataframe_setup(self):
        
        # Initialise a dictionary to store the sanitized headers and strings
        genesippr_dict = dict()
        # Try to open all the reports - use pandas to extract the results from any report that exists
        try:
            sippr_matrix = pd.read_csv(os.path.join(self.reportpath, 'genesippr.csv'),
                                       delimiter=',', index_col=0).T.to_dict()
        except FileNotFoundError:
            sippr_matrix = dict()
        try:
            conf_matrix = pd.read_csv(os.path.join(self.reportpath, 'confindr_report.csv'),
                                      delimiter=',', index_col=0).T.to_dict()
        except FileNotFoundError:
            conf_matrix = dict()
        try:
            gdcs_matrix = pd.read_csv(os.path.join(self.reportpath, 'GDCS.csv'),
                                      delimiter=',', index_col=0).T.to_dict()
        except FileNotFoundError:
            gdcs_matrix = dict()
        # Populate the header:sanitized string dictionary with results from all strains
        for sample in self.metadata:
            genesippr_dict[sample.name] = dict()
            try:
                genesippr_dict[sample.name]['eae'] = self.data_sanitise(sippr_matrix[sample.name]['eae'])
            except KeyError:
                genesippr_dict[sample.name]['eae'] = 0
            try:
                genesippr_dict[sample.name]['hlyAEc'] = self.data_sanitise(sippr_matrix[sample.name]['hlyAEc'])
            except KeyError:
                genesippr_dict[sample.name]['hlyAEc'] = 0
            try:
                genesippr_dict[sample.name]['VT1'] = self.data_sanitise(sippr_matrix[sample.name]['VT1'])
            except KeyError:
                genesippr_dict[sample.name]['VT1'] = 0
            try:
                genesippr_dict[sample.name]['VT2'] = self.data_sanitise(sippr_matrix[sample.name]['VT2'])
            except KeyError:
                genesippr_dict[sample.name]['VT2'] = 0
            try:
                genesippr_dict[sample.name]['hlyALm'] = self.data_sanitise(sippr_matrix[sample.name]['hlyALm'])
            except KeyError:
                genesippr_dict[sample.name]['hlyALm'] = 0
            try:
                genesippr_dict[sample.name]['IGS'] = self.data_sanitise(sippr_matrix[sample.name]['IGS'])
            except KeyError:
                genesippr_dict[sample.name]['IGS'] = 0
            try:
                genesippr_dict[sample.name]['inlJ'] = self.data_sanitise(sippr_matrix[sample.name]['inlJ'])
            except KeyError:
                genesippr_dict[sample.name]['inlJ'] = 0
            try:
                genesippr_dict[sample.name]['invA'] = self.data_sanitise(sippr_matrix[sample.name]['invA'])
            except KeyError:
                genesippr_dict[sample.name]['invA'] = 0
            try:
                genesippr_dict[sample.name]['stn'] = self.data_sanitise(sippr_matrix[sample.name]['stn'])
            except KeyError:
                genesippr_dict[sample.name]['stn'] = 0
            try:
                genesippr_dict[sample.name]['GDCS'] = self.data_sanitise(gdcs_matrix[sample.name]['Pass/Fail'],
                                                                         header='Pass/Fail')
            except KeyError:
                genesippr_dict[sample.name]['GDCS'] = 0
            try:
                genesippr_dict[sample.name]['Contamination'] = self.data_sanitise(
                    conf_matrix[sample.name]['ContamStatus'], header='ContamStatus')
            except KeyError:
                genesippr_dict[sample.name]['Contamination'] = 0
            try:
                genesippr_dict[sample.name]['Coverage'] = self.data_sanitise(
                    gdcs_matrix[sample.name]['MeanCoverage'], header='MeanCoverage')
            except KeyError:
                genesippr_dict[sample.name]['Coverage'] = 0
        # Create a report from the header: sanitized string dictionary to be used in the creation of the report image
        with open(self.image_report, 'w') as csv:
            data = '{}\n'.format(','.join(self.header_list))
            for strain in sorted(genesippr_dict):
                data += '{str},'.format(str=strain)
                for header in self.header_list[1:]:
                    data += '{value},'.format(value=genesippr_dict[strain][header])

                data = data.rstrip(',')
                data += '\n'
            csv.write(data)
1158900
def _create_dataframe(cls, value):
        
        try:

            ordered_jo = json.loads(six.text_type(value), object_pairs_hook=collections.OrderedDict)

            #Pandas does not maintain the order of dicts, so we must break the dict
            #up and put it into the dataframe manually to maintain the order.

            cols = list(ordered_jo.keys())

            if len(cols) == 0:
                raise ValueError("Dataframe has no columns")

            #Assume all sub-dicts have the same set of keys
            if isinstance(ordered_jo[cols[0]], list):
                index = range(len(ordered_jo[cols[0]]))
            else:
                index = list(ordered_jo[cols[0]].keys())
            data = []
            for c in cols:
                if isinstance(ordered_jo[c], list):
                    data.append(ordered_jo[c])
                else:
                    data.append(list(ordered_jo[c].values()))

            # This goes in 'sideways' (cols=index, index=cols), so it needs to be transposed after to keep
            # the correct structure
            # We also try to coerce the data to a regular numpy array first. If the shape is correct
            # this is a much faster way of creating the DataFrame instance.
            try:
                np_data = np.array(data)
            except ValueError:
                np_data = None

            if np_data is not None and np_data.shape == (len(cols), len(index)):
                df = pd.DataFrame(np_data, columns=index, index=cols).transpose()
            else:
                # TODO should these heterogenous structure be supported?
                # See https://github.com/hydraplatform/hydra-base/issues/72
                df = pd.DataFrame(data, columns=index, index=cols).transpose()


        except ValueError as e:
            """ Raised on scalar types used as pd.DataFrame values
                in absence of index arg
            """
            raise HydraError(str(e))

        except AssertionError as e:
            log.warning("An error occurred creating the new data frame: %s. Defaulting to a simple read_json"%(e))
            df = pd.read_json(value).fillna(0)

        return df
218028
def as_struct_array(*columns):
    
    columns = [np.asarray(c) for c in columns]
    rows = len(columns[0])

    names = ['f'+str(i) for i in range(len(columns))]
    dtype = [(names[i], c.dtype, c.shape[1:]) for i, c in enumerate(columns)]
    data = np.empty(rows, dtype)
    for i, c in enumerate(columns):
        data[names[i]] = c
    return data
1015351
def add_dataframe(df, name, pkg=None, description=''):
    
    from warnings import warn
    from metapack.cli.core import alt_col_name, type_map
    import numpy as np

    if name is None or df is None:
        warn("Did not find dataframe for reference '{}' ".format(ref))
        return

    pkg = pkg or open_source_package()

    resource_ref = 'file:' + get_notebook_rel_path(pkg) + '#' + name

    t = pkg.find_first('Root.Datafile', value=resource_ref)
    col_props = {}

    if t:
        print("Datafile exists for url '{}', deleting".format(resource_ref))

        if t.schema_term:
            col_props = { c['name']:c for c in t.columns()}
            pkg.remove_term(t.schema_term)

        pkg.remove_term(t)

    t = pkg['Resources'].new_term('Root.Datafile', resource_ref, name=name, description=description)

    st = pkg['Schema'].new_term('Table', t.schema_name, description=description)

    for i, name in enumerate(df.columns):

        props = col_props.get(name,{})

        try:
            native_type = type(np.asscalar(df[name].dtype.type(0))).__name__
        except ValueError:
            native_type = df[name].dtype.name
        except AttributeError:
            native_type = type(df[name][0]).__name__

        for pn in 'datatype name pos header'.split():
            if pn in props:
                del props[pn]

        if 'altname' in props:
            altname = props['altname']
            del props['altname']
        else:
            raw_alt_name = alt_col_name(name, i)
            altname = raw_alt_name if raw_alt_name != name else ''

        col = df[name]

        if hasattr(col, 'description'):  # custom property
            props['description'] = col.description

        t = st.new_child('Column', name,
                            datatype=type_map.get(native_type, native_type),
                            altname=altname,
                            **props)


    pkg.write_csv()
471335
def dataframe(self):
        
        frames = []
        for game in self.__iter__():
            df = game.dataframe
            if df is not None:
                frames.append(df)
        if frames == []:
            return None
        return pd.concat(frames)
417222
def main(df):
  '''
  
  '''
  import numpy as np
  from ast import literal_eval as make_tuple

  test = {}
  test['row'] = df['mat'].index.tolist()
  test['col'] = df['mat'].columns.tolist()

  # if type( test_row ) is not str and type( test_row ) is not tuple:

  found_tuple = {}
  found_number = {}
  for inst_rc in ['row','col']:

    inst_name = test[inst_rc][0]

    found_tuple[inst_rc] = False
    found_number[inst_rc] = False

    if type(inst_name) != tuple:

      if type(inst_name) is int or type(inst_name) is float or type(inst_name) is np.int64:
        found_number[inst_rc] = True

      else:
        check_open = inst_name[0]
        check_comma = inst_name.find(',')
        check_close = inst_name[-1]

        if check_open == '(' and check_close == ')' and check_comma > 0 \
          and check_comma < len(inst_name):
          found_tuple[inst_rc] = True

  # convert to tuple if necessary
  #################################################
  if found_tuple['row']:
    row_names = df['mat'].index.tolist()
    row_names = [make_tuple(x) for x in row_names]
    df['mat'].index = row_names

  if found_tuple['col']:
    col_names = df['mat'].columns.tolist()
    col_names = [make_tuple(x) for x in col_names]
    df['mat'].columns = col_names

  # convert numbers to string if necessary
  #################################################
  if found_number['row']:
    row_names = df['mat'].index.tolist()
    row_names = [str(x) for x in row_names]
    df['mat'].index = row_names

  if found_number['col']:
    col_names = df['mat'].columns.tolist()
    col_names = [str(x) for x in col_names]
    df['mat'].columns = col_names

  return df
493257
def df_to_dat(net, df, define_cat_colors=False):
  '''
  
  '''
  from . import categories

  # check if df has unique values
  df['mat'] = make_unique_labels.main(net, df['mat'])

  net.dat['mat'] = df['mat'].values
  net.dat['nodes']['row'] = df['mat'].index.tolist()
  net.dat['nodes']['col'] = df['mat'].columns.tolist()

  for inst_rc in ['row', 'col']:

    if type(net.dat['nodes'][inst_rc][0]) is tuple:
      # get the number of categories from the length of the tuple
      # subtract 1 because the name is the first element of the tuple
      num_cat = len(net.dat['nodes'][inst_rc][0]) - 1

      net.dat['node_info'][inst_rc]['full_names'] = net.dat['nodes']\
          [inst_rc]

      for inst_rcat in range(num_cat):
        net.dat['node_info'][inst_rc]['cat-' + str(inst_rcat)] = \
          [i[inst_rcat + 1] for i in net.dat['nodes'][inst_rc]]

      net.dat['nodes'][inst_rc] = [i[0] for i in net.dat['nodes'][inst_rc]]

  if 'mat_up' in df:
    net.dat['mat_up'] = df['mat_up'].values
    net.dat['mat_dn'] = df['mat_dn'].values

  if 'mat_orig' in df:
    net.dat['mat_orig'] = df['mat_orig'].values

  categories.dict_cat(net, define_cat_colors=define_cat_colors)
1025090
def _append_dataframe(self, df, source_info="", units=None):
        

        units = units or {}

        t = df.index
        index_name = df.index.name
        time_name = index_name or "time"

        version = self.version

        timestamps = t

        # if self.version < '3.10':
        #     if timestamps.dtype.byteorder == '>':
        #         timestamps = timestamps.byteswap().newbyteorder()
        #     for signal in signals:
        #         if signal.samples.dtype.byteorder == '>':
        #             signal.samples = signal.samples.byteswap().newbyteorder()

        if self.version >= "3.00":
            channel_size = v23c.CN_DISPLAYNAME_BLOCK_SIZE
        elif self.version >= "2.10":
            channel_size = v23c.CN_LONGNAME_BLOCK_SIZE
        else:
            channel_size = v23c.CN_SHORT_BLOCK_SIZE

        file = self._tempfile
        tell = file.tell

        kargs = {
            "module_nr": 0,
            "module_address": 0,
            "type": v23c.SOURCE_ECU,
            "description": b"Channel inserted by Python Script",
        }
        ce_block = ChannelExtension(**kargs)

        dg_cntr = len(self.groups)

        gp = {}
        gp.channels = gp_channels = []
        gp.channel_dependencies = gp_dep = []
        gp.signal_types = gp_sig_types = []
        gp.string_dtypes = []

        self.groups.append(gp)

        cycles_nr = len(timestamps)
        fields = []
        types = []
        parents = {}
        ch_cntr = 0
        offset = 0
        field_names = UniqueDB()

        if df.shape[0]:
            # conversion for time channel
            kargs = {
                "conversion_type": v23c.CONVERSION_TYPE_NONE,
                "unit": b"s",
                "min_phy_value": timestamps[0] if cycles_nr else 0,
                "max_phy_value": timestamps[-1] if cycles_nr else 0,
            }
            conversion = ChannelConversion(**kargs)
            conversion.unit = "s"
            source = ce_block

            # time channel
            t_type, t_size = fmt_to_datatype_v3(timestamps.dtype, timestamps.shape)
            kargs = {
                "short_name": time_name.encode("latin-1"),
                "channel_type": v23c.CHANNEL_TYPE_MASTER,
                "data_type": t_type,
                "start_offset": 0,
                "min_raw_value": timestamps[0] if cycles_nr else 0,
                "max_raw_value": timestamps[-1] if cycles_nr else 0,
                "bit_count": t_size,
                "block_len": channel_size,
                "version": version,
            }
            channel = Channel(**kargs)
            channel.name = name = time_name
            channel.conversion = conversion
            channel.source = source

            gp_channels.append(channel)

            self.channels_db.add(name, (dg_cntr, ch_cntr))
            self.masters_db[dg_cntr] = 0
            # data group record parents
            parents[ch_cntr] = name, 0

            # time channel doesn't have channel dependencies
            gp_dep.append(None)

            fields.append(timestamps)
            types.append((name, timestamps.dtype))
            field_names.add(name)

            offset += t_size
            ch_cntr += 1

            gp_sig_types.append(0)

        for signal in df:

            sig = df[signal]
            name = signal

            sig_type = v23c.SIGNAL_TYPE_SCALAR

            gp_sig_types.append(sig_type)

            new_source = ce_block

            # compute additional byte offset for large records size
            if offset > v23c.MAX_UINT16:
                additional_byte_offset = ceil((offset - v23c.MAX_UINT16) / 8)
                start_bit_offset = offset - additional_byte_offset * 8
            else:
                start_bit_offset = offset
                additional_byte_offset = 0

            s_type, s_size = fmt_to_datatype_v3(sig.dtype, sig.shape)

            kargs = {
                "channel_type": v23c.CHANNEL_TYPE_VALUE,
                "data_type": s_type,
                "min_raw_value": 0,
                "max_raw_value": 0,
                "start_offset": start_bit_offset,
                "bit_count": s_size,
                "additional_byte_offset": additional_byte_offset,
                "block_len": channel_size,
                "version": version,
            }

            if s_size < 8:
                s_size = 8

            channel = Channel(**kargs)
            channel.name = name
            channel.source = new_source

            unit = units.get(name, b"")
            if unit:
                if hasattr(unit, "encode"):
                    unit = unit.encode("latin-1")
                # conversion for time channel
                kargs = {
                    "conversion_type": v23c.CONVERSION_TYPE_NONE,
                    "unit": unit,
                    "min_phy_value": 0,
                    "max_phy_value": 0,
                }
                conversion = ChannelConversion(**kargs)
                conversion.unit = unit

            gp_channels.append(channel)

            offset += s_size

            self.channels_db.add(name, (dg_cntr, ch_cntr))

            # update the parents as well
            field_name = field_names.get_unique_name(name)
            parents[ch_cntr] = field_name, 0

            if sig.dtype.kind == "S":
                gp.string_dtypes.append(sig.dtype)

            fields.append(sig)
            types.append((field_name, sig.dtype))

            ch_cntr += 1

            # simple channels don't have channel dependencies
            gp_dep.append(None)

        # channel group
        kargs = {
            "cycles_nr": cycles_nr,
            "samples_byte_nr": offset // 8,
            "ch_nr": ch_cntr,
        }
        if self.version >= "3.30":
            kargs["block_len"] = v23c.CG_POST_330_BLOCK_SIZE
        else:
            kargs["block_len"] = v23c.CG_PRE_330_BLOCK_SIZE
        gp.channel_group = ChannelGroup(**kargs)
        gp.channel_group.comment = source_info

        # data group
        if self.version >= "3.20":
            block_len = v23c.DG_POST_320_BLOCK_SIZE
        else:
            block_len = v23c.DG_PRE_320_BLOCK_SIZE
        gp.data_group = DataGroup(block_len=block_len)

        # data block
        types = dtype(types)

        gp.types = types
        gp.parents = parents
        gp.sorted = True

        if df.shape[0]:
            samples = fromarrays(fields, dtype=types)
        else:
            samples = array([])

        block = samples.tostring()

        gp.data_location = v23c.LOCATION_TEMPORARY_FILE
        if cycles_nr:
            data_address = tell()
            gp.data_group.data_block_addr = data_address
            size = len(block)
            self._tempfile.write(block)

            gp.data_blocks.append(
                DataBlockInfo(
                    address=data_address,
                    block_type=0,
                    raw_size=size,
                    size=size,
                    param=0,
                )
            )
        else:
            gp.data_location = v23c.LOCATION_TEMPORARY_FILE
        # data group trigger
        gp.trigger = None
1259987
def create_sc_table(scfile, colnames=None):
    """"""

    if utils.is_fits_file(scfile) and colnames is None:
        return create_table_from_fits(scfile, 'SC_DATA')

    if utils.is_fits_file(scfile):
        files = [scfile]
    else:
        files = [line.strip() for line in open(scfile, 'r')]

    tables = [create_table_from_fits(f, 'SC_DATA', colnames)
              for f in files]

    return vstack(tables)
1301411
def create_dataset_file(root_dir, data):
  """"""
  file_path = os.path.join(root_dir, '{dataset_type}', '{dataset_name}.py')
  context = (
      _HEADER + _DATASET_DEFAULT_IMPORTS + _CITATION
      + _DESCRIPTION + _DATASET_DEFAULTS
  )

  with gfile.GFile(file_path.format(**data), 'w') as f:
    f.write(context.format(**data))
1161798
def _make_df(rows):
    """"""

    # Make DataFrame
    df = pd.DataFrame(rows).set_index('ts')

    # TimeDelta Support: https://issues.apache.org/jira/browse/ARROW-835
    for column in df.columns:
        if(df[column].dtype == 'timedelta64[ns]'):
            print('Converting timedelta column {:s}...'.format(column))
            df[column] = df[column].astype(str)
    return df
1306998
def to_frame(self, **kwargs):
        
        """
        df = export.write_dataframe(self._values, **kwargs)
        df.name = self.title
        return df
1285333
def __dump(df,relation='data',description=''):
    
    attrs = []
    for col in df.columns:
        attr = col.split('@')
        if attr[1].count('{')>0 and attr[1].count('}')>0:
            vals = attr[1].replace('{','').replace('}','').split(',')
            attrs.append((attr[0],vals))
        else:
            attrs.append((attr[0],attr[1]))

    data = list(df.values)
    result = {
        'attributes':attrs,
        'data':data,
        'description':description,
        'relation':relation
    }
    return result
1141463
def cache_to_df(dir_path):
    
    table = {}
    for attrib in glob.glob(os.path.join(dir_path, '*')):
        attrib_name, attrib_ext = os.path.splitext(os.path.basename(attrib))
        if attrib_ext == '.lf8':
            attrib_data = np.fromfile(attrib, np.float64)
            table[attrib_name] = attrib_data

        elif attrib_ext == '.lf4':
            attrib_data = np.fromfile(attrib, np.float32)
            table[attrib_name] = attrib_data

        elif attrib_ext == '.li2':
            attrib_data = np.fromfile(attrib, np.int16)
            table[attrib_name] = attrib_data

        elif attrib_ext == '.li4':
            attrib_data = np.fromfile(attrib, np.int32)
            table[attrib_name] = attrib_data

        elif attrib_ext == '.li8':
            attrib_data = np.fromfile(attrib, np.int64)
            table[attrib_name] = attrib_data

        elif attrib_ext == '.ib1':
            attrib_data = np.fromfile(attrib, np.bool_)
            table[attrib_name] = attrib_data

        elif attrib_ext.startswith('.iS'):
            length_string = int(attrib_ext[3:])
            attrib_data = np.fromfile(attrib, ('a' + str(length_string)))
            table[attrib_name] = attrib_data

        else:
            print('Array {} is not a recognized data type'.format(attrib))

    df = pd.DataFrame(table)
    return df