Given an open file for `hip_main.dat.gz`, return a parsed dataframe.

    If your copy of ``hip_main.dat`` has already been unzipped, pass the
    optional argument ``compression=None``.
321015
def load_dataframe(fobj, compression='gzip'):
    
    try:
        from pandas import read_fwf
    except ImportError:
        raise ImportError(PANDAS_MESSAGE)

    names, colspecs = zip(
        ('hip', (2, 14)),
        ('magnitude', (41, 46)),
        ('ra_degrees', (51, 63)),
        ('dec_degrees', (64, 76)),
        ('parallax_mas', (79, 86)),  # TODO: have Star load this
        ('ra_mas_per_year', (87, 95)),
        ('dec_mas_per_year', (96, 104)),
    )

    df = read_fwf(fobj, colspecs, names=names, compression=compression)
    df = df.assign(
        ra_hours = df['ra_degrees'] / 15.0,
        epoch_year = 1991.25,
    )
    return df.set_index('hip')
1133945
def load(self):
        
        # read file, keep all values as strings
        df = pd.read_csv(self.input_file,
                         sep=',',
                         quotechar='"',
                         encoding='utf-8',
                         dtype=object)

        # wer are only interested in the NUTS code and description, rename them also
        df = df[['NUTS-Code', 'Description']]
        df.columns = ['key', 'name']

        # we only want NUTS2 regions (4-digit codes)
        df = df[df['key'].str.len() == 4]

        # drop 'Extra Regio' codes ending in 'ZZ'
        df = df[df['key'].str[2:] != 'ZZ']

        return df
292135
def gzip_uncompress(data, truncated=False):
    '''
    '''
    decompressor = SimpleGzipDecompressor()
    inflated_data = decompressor.decompress(data)

    if not truncated:
        inflated_data += decompressor.flush()

    return inflated_data
495559
def _open_dataset(self, urlpath):
        
        import dask.dataframe

        if self.pattern is None:
            self._dataframe = dask.dataframe.read_csv(
                urlpath, storage_options=self._storage_options,
                **self._csv_kwargs)
            return

        if not (DASK_VERSION >= '0.19.0'):
            raise ValueError("Your version of dask is '{}'. "
                "The ability to include filenames in read_csv output "
                "(``include_path_column``) was added in 0.19.0, so "
                "pattern urlpaths are not supported.".format(DASK_VERSION))

        drop_path_column = 'include_path_column' not in self._csv_kwargs
        path_column = self._path_column()

        self._dataframe = dask.dataframe.read_csv(
            urlpath, storage_options=self._storage_options, **self._csv_kwargs)

        # add the new columns to the dataframe
        self._set_pattern_columns(path_column)

        if drop_path_column:
            self._dataframe = self._dataframe.drop([path_column], axis=1)
1086221
def nii_ugzip(imfile, outpath=''):
    ''''''
    import gzip
    with gzip.open(imfile, 'rb') as f:
        s = f.read()
    # Now store the uncompressed data
    if outpath=='':
        fout = imfile[:-3]
    else:
        fout = os.path.join(outpath, os.path.basename(imfile)[:-3])
    # store uncompressed file data from 's' variable
    with open(fout, 'wb') as f:
        f.write(s)
    return fout
747955
def parse_gzip(file_path):
    """"""
    newDecoder = MMTFDecoder()
    newDecoder.decode_data(_unpack(gzip.open(file_path, "rb")))
    return newDecoder
673821
def compress(info):
    
    if PY3:
        file_obj = BytesIO()
    else:
        file_obj = StringIO()

    with zipfile.ZipFile(
            file_obj,
            mode='w',
            compression=zipfile.ZIP_DEFLATED) as zipper:
        for name, data in info.items():
            if hasattr(data, 'read'):
                # if we were passed a file object, read it
                data = data.read()
            zipper.writestr(name, data)
    file_obj.seek(0)
    compressed = file_obj.read()
    return compressed
564344
def _open(self):
        
        if self.fileName.endswith(".gz"):
            return gzip.open(self.fileName)
        elif self.fileName.endswith(".bz2"):
            return bz2.BZ2File(self.fileName)
        else:
            return open(self.fileName)
1315782
def _gzip_open_handle(handle):
    """""""
    import gzip
    if sys.version_info[0] > 2:
        import io
        handle = io.TextIOWrapper(gzip.GzipFile(fileobj=handle), encoding="UTF-8")
    else:
        handle = gzip.GzipFile(fileobj=handle)
    return handle
1117685
def load_local_dataset(self, ds_str):
        '''
        
        '''
        if cdl.is_cdl(ds_str):
            ds_str = self.generate_dataset(ds_str)

        if netcdf.is_netcdf(ds_str):
            return MemoizedDataset(ds_str)

        # Assume this is just a Generic File if it exists
        if os.path.isfile(ds_str):
            return GenericFile(ds_str)

        raise ValueError("File is an unknown format")
549561
def save_dataframes(self, outdir, prefix='df_'):
        
        # Get list of attributes that start with "df_"
        dfs = list(filter(lambda x: x.startswith(prefix), dir(self)))

        counter = 0
        for df in dfs:
            outpath = ssbio.utils.outfile_maker(inname=df, outext='.csv', outdir=outdir)
            my_df = getattr(self, df)
            if not isinstance(my_df, pd.DataFrame):
                raise TypeError('{}: object is not a Pandas DataFrame'.format(df))

            if my_df.empty:
                log.debug('{}: empty dataframe, not saving'.format(df))
            else:
                my_df.to_csv(outpath)
                log.debug('{}: saved dataframe'.format(outpath))
                counter += 1

        log.debug('Saved {} dataframes at {}'.format(counter, outdir))
132696
def unpack(self, name):
        
        path = self.list[name]
        df = pd.read_pickle(path, compression='gzip')
        return df
1110011
def load(cls, filename, gzipped, byteorder='big'):
        
        open_file = gzip.open if gzipped else open
        with open_file(filename, 'rb') as buff:
            return cls.from_buffer(buff, byteorder)
725083
def unzip_file_if_necessary(self, source_file):
        
        if source_file.endswith(".gz"):
            self.print_message("Decompressing '%s'" % source_file)
            subprocess.check_call(["gunzip", "--force", source_file])
            source_file = source_file[:-len(".gz")]
        return source_file
629670
def gzip(self, data, truncate=True):
        '''
        
        '''
        # Entropy is a simple ratio of: <zlib compressed size> / <original
        # size>
        e = float(float(len(zlib.compress(str2bytes(data), 9))) / float(len(data)))

        if truncate and e > 1.0:
            e = 1.0

        return e
715281
def _decompress_data(self, data):
        ''''''
        if self._decompressor:
            try:
                return self._decompressor.decompress(data)
            except zlib.error as error:
                raise ProtocolError(
                    'zlib error: {0}.'.format(error)
                ) from error
        else:
            return data
1057928
def pp_file_to_dataframe(pp_filename):

    

    df = pd.read_csv(pp_filename, delim_whitespace=True,
                     header=None, names=PP_NAMES,usecols=[0,1,2,3,4])
    df.loc[:,"name"] = df.name.apply(str).apply(str.lower)
    return df