Read out an hdf5 file.

        Takes the output of :class:`gwsnrcalc.genconutils.genprocess.GenProcess`
        and reads it out to an HDF5 file.
968254
def hdf5_read_out(self):
        
        with h5py.File(self.WORKING_DIRECTORY + '/' + self.output_file_name, 'w') as f:

            header = f.create_group('header')
            header.attrs['Title'] = 'Generated SNR Out'
            header.attrs['Author'] = 'Generator by: Michael Katz'
            header.attrs['Date/Time'] = str(datetime.datetime.now())

            for which in ['x', 'y']:
                header.attrs[which + 'val_name'] = getattr(self, which + 'val_name')
                header.attrs['num_' + which + '_pts'] = getattr(self, 'num_' + which)

            ecc = 'eccentricity' in self.__dict__
            if ecc:
                name_list = ['observation_time', 'start_frequency', 'start_separation'
                             'eccentricity']
            else:
                name_list = ['spin_1', 'spin_2', 'spin', 'end_time']

            name_list += ['total_mass', 'mass_ratio', 'start_time', 'luminosity_distance',
                          'comoving_distance', 'redshift']

            for name in name_list:
                if name != self.xval_name and name != self.yval_name:
                    try:
                        getattr(self, name)
                        header.attrs[name] = getattr(self, name)
                    except AttributeError:
                        pass

            if self.added_note != '':
                header.attrs['Added note'] = self.added_note

            data = f.create_group('data')

            # read out x,y values in compressed data set
            dset = data.create_dataset(self.x_col_name, data=self.xvals,
                                       dtype='float64', chunks=True,
                                       compression='gzip', compression_opts=9)

            dset = data.create_dataset(self.y_col_name, data=self.yvals,
                                       dtype='float64', chunks=True,
                                       compression='gzip', compression_opts=9)

            # read out all datasets
            for key in self.output_dict.keys():
                dset = data.create_dataset(key, data=self.output_dict[key],
                                           dtype='float64', chunks=True,
                                           compression='gzip', compression_opts=9)
332807
def _generate_examples(self, filepath):
    
    # Simultaneously iterating through the different data sets in the hdf5
    # file will be slow with a single file. Instead, we first load everything
    # into memory before yielding the samples.
    image_array, values_array = _load_data(filepath)

    # We need to calculate the class labels from the float values in the file.
    labels_array = np.zeros_like(values_array, dtype=np.int64)
    for i in range(values_array.shape[1]):
      labels_array[:, i] = _discretize(values_array[:, i])  # pylint: disable=unsupported-assignment-operation

    for image, labels, values in moves.zip(image_array, labels_array,
                                           values_array):
      yield {
          "image": image,
          "label_floor_hue": labels[0],
          "label_wall_hue": labels[1],
          "label_object_hue": labels[2],
          "label_scale": labels[3],
          "label_shape": labels[4],
          "label_orientation": labels[5],
          "value_floor_hue": values[0],
          "value_wall_hue": values[1],
          "value_object_hue": values[2],
          "value_scale": values[3],
          "value_shape": values[4],
          "value_orientation": values[5],
      }
503049
def _read_group_h5(filename, groupname):
    
    with h5py.File(filename, 'r') as h5f:
        data = h5f[groupname][()]
    return data
857308
def h5fmem(**kwargs):
    """"""

    # need a file name even tho nothing is ever written
    fn = tempfile.mktemp()

    # file creation args
    kwargs['mode'] = 'w'
    kwargs['driver'] = 'core'
    kwargs['backing_store'] = False

    # open HDF5 file
    h5f = h5py.File(fn, **kwargs)

    return h5f
1064628
def _reader_factory(name, filetype, descr):
    """"""
    def basic_reader(filename, last = None, first = 0):
        """Common "template" to all read functions."""
        hdl = Sndfile(filename, 'r')
        try:
            if not hdl.format.file_format == filetype:
                raise ValueError, "%s is not a %s file (is %s)" \
                      % (filename, filetype, hdl.format.file_format)

            fs = hdl.samplerate
            enc = hdl.encoding
            # Set the pointer to start position
            nf  = hdl.seek(first, 1)
            if not nf == first:
                raise IOError("Error while seeking at starting position")

            if last is None:
                nframes = hdl.nframes - first
                data    = hdl.read_frames(nframes)
            else:
                data    = hdl.read_frames(last)
        finally:
            hdl.close()

        return data, fs, enc
    doc = \
    """Simple reader for %(format)s audio files.

    Parameters
    ----------
    filename : str
        Name of the file to read
    last : int
        Last frame to read. If None, this is equal to the number of frames in
        the file.
    first : int
        First frame to read. If 0, means starting from the beginning of the
        file.

    Returns
    -------
    data : array
        the read data (one column per channel)
    fs : int
        the sampling rate
    enc : str
        the encoding of the file, such as 'pcm16', 'float32', etc...

    Notes
    -----
    For total control over options, such as output's dtype, etc...,
    you should use Sndfile class instances instead""" % {'format': str(descr)}
    basic_reader.__doc__    = doc
    basic_reader.__name__   = name
    return basic_reader
1147028
def getfile(self):
        
        current_file = str(self.selectedFiles()[0])
        if os.path.isfile(current_file):
            print 'current_file', current_file
            if current_file.endswith('.raw') or current_file.endswith('.pst'):
                fmode = 'r'
            else:
                fmode = 'a'
        else:
            if not current_file.endswith('.hdf5') and not current_file.endswith('.h5'):
                current_file += '.hdf5'
            fmode = 'w-'
        return current_file, fmode
791538
def register_hdf5_array_io(array_type, format='hdf5', identify=True):
    
    def from_hdf5(*args, **kwargs):
        """Read an array from HDF5
        """
        kwargs.setdefault('array_type', array_type)
        return read_hdf5_array(*args, **kwargs)

    io_registry.register_reader(format, array_type, from_hdf5)
    if issubclass(array_type, Series):
        io_registry.register_writer(format, array_type, write_hdf5_series)
    else:
        io_registry.register_writer(format, array_type, write_hdf5_array)

    if identify:
        io_registry.register_identifier(format, array_type,
                                        io_hdf5.identify_hdf5)
495830
def with_read_hdf5(func):
    
    @wraps(func)
    def decorated_func(fobj, *args, **kwargs):
        # pylint: disable=missing-docstring
        if not isinstance(fobj, h5py.HLObject):
            if isinstance(fobj, FILE_LIKE):
                fobj = fobj.name
            with h5py.File(fobj, 'r') as h5f:
                return func(h5f, *args, **kwargs)
        return func(fobj, *args, **kwargs)

    return decorated_func
612026
def write_hdf5_cells(E, filename):
    ''' '''

    E = E.tocsr()

    hf = h5py.File(filename, 'w')
    counts_group = hf.create_group('counts')
    gix_group = hf.create_group('gene_ix')

    hf.attrs['ncells'] = E.shape[0]
    hf.attrs['ngenes'] = E.shape[1]

    for iC in range(E.shape[0]):
        counts = E[iC,:].A.squeeze()
        gene_ix = np.nonzero(counts)[0]
        counts = counts[gene_ix]
        counts_group.create_dataset(str(iC), data = counts)
        gix_group.create_dataset(str(iC), data = gene_ix)

    hf.close()
876901
def load_hdf(cls, filename, path='', ic=None):
        
        store = pd.HDFStore(filename)
        try:
            samples = store[path+'/df']
            attrs = store.get_storer(path+'/df').attrs
        except:
            store.close()
            raise
        df = store[path+'/df']
        new = cls.from_df(df)

        if ic is None:
            ic = get_ichrone('mist')

        new.define_models(ic, N=attrs.N, index=attrs.index)
        new.spectroscopy = attrs.spectroscopy
        new.parallax = attrs.parallax
        store.close()
        return new
217095
def write_resultfiles(self):
        
        t0=time.time()
        writer = ow.output_writer(output_dictionary=self.result)
        if len(self.options.outpath)>=3 and self.options.outpath[-3:]==".h5":
            writer.write_hdf5(filename=self.options.outpath,timestamp=self.options.timestamp)
        else:
            writer.write_txt(outdir=self.options.outpath,timestamp=self.options.timestamp,delimiter=self.options.delimiter,float_format=self.options.float_format)
        t1=time.time()
        print(("Elapsed time for writing the output files is %.2f seconds" % (t1-t0)))
1311839
def read(f):
    
    try:
      return PyVDF.reads(f.read())
    except AttributeError:
      pass

    try:
      with open(f, 'r') as filec:
        data = PyVDF.reads(filec.read())
        filec.close()
        return data
    except IOError as e:
      print("Could not open '" + f + "' for reading.")
      print("Ignore this if you are creating a new file.")

    return PyVDF.__UseDict()
483873
def make_pizzly_gtf(gtf_file, out_file, data):
    
    if file_exists(out_file):
        return out_file
    db = gtf.get_gtf_db(gtf_file)
    with file_transaction(data, out_file) as tx_out_file:
        with open(tx_out_file, "w") as out_handle:
            for gene in db.features_of_type("gene"):
                children = [x for x in db.children(id=gene)]
                for child in children:
                    if child.attributes.get("gene_biotype", None):
                        gene_biotype = child.attributes.get("gene_biotype")
                gene.attributes['gene_biotype'] = gene_biotype
                gene.source = gene_biotype[0]
                print(gene, file=out_handle)
                for child in children:
                    child.source = gene_biotype[0]
                    # gffread produces a version-less FASTA file
                    child.attributes.pop("transcript_version", None)
                    print(child, file=out_handle)
    return out_file
578253
def _generate_examples(self, filepath):
    
    # Simultaneously iterating through the different data sets in the hdf5
    # file is >100x slower and the data set is small (26.7MB). Hence, we first
    # load everything into memory before yielding the samples.
    image_array, class_array, values_array = _load_data(filepath)
    for image, classes, values in moves.zip(image_array, class_array,
                                            values_array):
      yield dict(
          image=np.expand_dims(image, -1),
          label_shape=classes[1],
          label_scale=classes[2],
          label_orientation=classes[3],
          label_x_position=classes[4],
          label_y_position=classes[5],
          value_shape=values[1],
          value_scale=values[2],
          value_orientation=values[3],
          value_x_position=values[4],
          value_y_position=values[5])
721012
def simulate_reads(self):
        
        logging.info('Read simulation')
        for sample in self.metadata:
            # Create the simulated_reads GenObject
            sample.simulated_reads = GenObject()
            # Iterate through all the desired depths of coverage
            for depth in self.read_depths:
                # Create the depth GenObject
                setattr(sample.simulated_reads, depth, GenObject())
                # Set the depth and output directory attributes for the depth GenObject
                sample.simulated_reads[depth].depth = depth
                sample.simulated_reads[depth].depth_dir = os.path.join(sample.outputdir, 'simulated', depth)
                # Create the output directory
                make_path(sample.simulated_reads[depth].depth_dir)
                # Iterate through all the desired forward and reverse read pair lengths
                for read_pair in self.read_lengths:
                    # Create the read_pair GenObject within the depth GenObject
                    setattr(sample.simulated_reads[depth], read_pair, GenObject())
                    # Set and create the output directory
                    sample.simulated_reads[depth][read_pair].outputdir = \
                        os.path.join(sample.simulated_reads[depth].depth_dir, read_pair)
                    make_path(sample.simulated_reads[depth][read_pair].outputdir)
                    # Create both forward_reads and reverse_reads sub-GenObjects
                    sample.simulated_reads[depth][read_pair].forward_reads = GenObject()
                    sample.simulated_reads[depth][read_pair].reverse_reads = GenObject()
                    # Extract the forward and reverse reads lengths from the read_pair variable
                    sample.simulated_reads[depth][read_pair].forward_reads.length, \
                        sample.simulated_reads[depth][read_pair].reverse_reads.length = read_pair.split('_')
                    # Set the name of the forward reads - include the depth and read length information
                    sample.simulated_reads[depth][read_pair].forward_reads.fastq = \
                        os.path.join(sample.simulated_reads[depth][read_pair].outputdir,
                                     '{name}_{depth}_{read_pair}_R1.fastq.gz'
                                     .format(name=sample.name,
                                             depth=depth,
                                             read_pair=read_pair))
                    # Reverse reads
                    sample.simulated_reads[depth][read_pair].reverse_reads.fastq = \
                        os.path.join(sample.simulated_reads[depth][read_pair].outputdir,
                                     '{name}_{depth}_{read_pair}_R2.fastq.gz'
                                     .format(name=sample.name,
                                             depth=depth,
                                             read_pair=read_pair))
                    # Create the trimmed output directory attribute
                    sample.simulated_reads[depth][read_pair].simulated_trimmed_outputdir \
                        = os.path.join(sample.simulated_reads[depth][read_pair].outputdir,
                                       'simulated_trimmed')
                    # Set the name of the forward trimmed reads - include the depth and read length information
                    # This is set now, as the untrimmed files will be removed, and a check is necessary
                    sample.simulated_reads[depth][read_pair].forward_reads.trimmed_simulated_fastq = \
                        os.path.join(sample.simulated_reads[depth][read_pair].simulated_trimmed_outputdir,
                                     '{name}_simulated_{depth}_{read_pair}_R1.fastq.gz'
                                     .format(name=sample.name,
                                             depth=depth,
                                             read_pair=read_pair))
                    # Reverse reads
                    sample.simulated_reads[depth][read_pair].reverse_reads.trimmed_simulated_fastq = \
                        os.path.join(sample.simulated_reads[depth][read_pair].simulated_trimmed_outputdir,
                                     '{name}_simulated_{depth}_{read_pair}_R2.fastq.gz'
                                     .format(name=sample.name,
                                             depth=depth,
                                             read_pair=read_pair))
                    # Calculate the number of reads required for the forward and reverse reads to yield the
                    # desired coverage depth e.g. 5Mbp genome at 20X coverage: 100Mbp in reads. 50bp forward reads
                    # 150bp reverse reads: forward proportion is 50 / (150 + 50) = 0.25 (and reverse is 0.75).
                    # Forward total reads is 25Mbp (75Mbp reverse). Number of reads required = 25Mbp / 50 bp
                    # 500000 reads total (same for reverse, as the reads are longer)
                    sample.simulated_reads[depth][read_pair].num_reads = \
                        int(sample.assembly_length *
                            int(depth) *
                            (int(sample.simulated_reads[depth][read_pair].forward_reads.length) /
                             (int(sample.simulated_reads[depth][read_pair].forward_reads.length) +
                              int(sample.simulated_reads[depth][read_pair].reverse_reads.length)
                              )
                             ) /
                            int(sample.simulated_reads[depth][read_pair].forward_reads.length)
                            )
                    logging.info(
                        'Simulating {num_reads} paired reads for sample {name} with the following parameters:\n'
                        'depth {dp}, forward reads {fl}bp, and reverse reads {rl}bp'
                        .format(num_reads=sample.simulated_reads[depth][read_pair].num_reads,
                                dp=depth,
                                name=sample.name,
                                fl=sample.simulated_reads[depth][read_pair].forward_reads.length,
                                rl=sample.simulated_reads[depth][read_pair].reverse_reads.length))
                    # If the reverse reads are set to 0, supply different parameters to randomreads
                    if sample.simulated_reads[depth][read_pair].reverse_reads.length != '0':
                        # Ensure that both the simulated reads, and the trimmed simulated reads files don't
                        # exist before simulating the reads
                        if not os.path.isfile(sample.simulated_reads[depth][read_pair].forward_reads.fastq) and \
                                not os.path.isfile(
                                    sample.simulated_reads[depth][read_pair].forward_reads.trimmed_simulated_fastq):
                            # Use the randomreads method in the OLCTools bbtools wrapper to simulate the reads
                            out, \
                                err, \
                                sample.simulated_reads[depth][read_pair].forward_reads.simulate_call = bbtools\
                                .randomreads(reference=sample.bestassemblyfile,
                                             length=sample.simulated_reads[depth][read_pair].reverse_reads.length,
                                             reads=sample.simulated_reads[depth][read_pair].num_reads,
                                             out_fastq=sample.simulated_reads[depth][read_pair].forward_reads.fastq,
                                             paired=True,
                                             returncmd=True,
                                             **{'ziplevel': '9',
                                                'illuminanames': 't',
                                                'Xmx': self.mem}
                                             )
                        else:
                            try:
                                forward_size = os.path.getsize(sample.simulated_reads[depth][read_pair]
                                                               .forward_reads.fastq)
                            except FileNotFoundError:
                                forward_size = 0
                            try:
                                reverse_size = os.path.getsize(sample.simulated_reads[depth][read_pair]
                                                               .reverse_reads.fastq)
                            except FileNotFoundError:
                                reverse_size = 0
                            if forward_size <= 100 or reverse_size <= 100:
                                try:
                                    os.remove(sample.simulated_reads[depth][read_pair].forward_reads.fastq)
                                except FileNotFoundError:
                                    pass
                                try:
                                    os.remove(sample.simulated_reads[depth][read_pair].reverse_reads.fastq)
                                except FileNotFoundError:
                                    pass
                                # Use the randomreads method in the OLCTools bbtools wrapper to simulate the reads
                                out, \
                                err, \
                                sample.simulated_reads[depth][read_pair].forward_reads.simulate_call = bbtools \
                                    .randomreads(reference=sample.bestassemblyfile,
                                                 length=sample.simulated_reads[depth][read_pair].reverse_reads.length,
                                                 reads=sample.simulated_reads[depth][read_pair].num_reads,
                                                 out_fastq=sample.simulated_reads[depth][read_pair].forward_reads.fastq,
                                                 paired=True,
                                                 returncmd=True,
                                                 **{'ziplevel': '9',
                                                    'illuminanames': 't'}
                                                 )
                    else:
                        if not os.path.isfile(sample.simulated_reads[depth][read_pair].forward_reads.fastq):
                            # Use the randomreads method in the OLCTools bbtools wrapper to simulate the reads
                            out, \
                                err, \
                                sample.simulated_reads[depth][read_pair].forward_reads.simulate_call = bbtools\
                                .randomreads(reference=sample.bestassemblyfile,
                                             length=sample.simulated_reads[depth][read_pair].forward_reads.length,
                                             reads=sample.simulated_reads[depth][read_pair].num_reads,
                                             out_fastq=sample.simulated_reads[depth][read_pair].forward_reads.fastq,
                                             paired=False,
                                             returncmd=True,
                                             **{'ziplevel': '9',
                                                'illuminanames': 't'}
                                             )
                # Update the JSON file
                self.write_json(sample)
999913
def km3h5concat(input_files, output_file, n_events=None, **kwargs):
    """"""
    from km3pipe import Pipeline    # noqa
    from km3pipe.io import HDF5Pump, HDF5Sink    # noqa

    pipe = Pipeline()
    pipe.attach(HDF5Pump, filenames=input_files, **kwargs)
    pipe.attach(StatusBar, every=250)
    pipe.attach(HDF5Sink, filename=output_file, **kwargs)
    pipe.drain(n_events)
131392
def read_hdf5_timeseries(h5f, path=None, start=None, end=None, **kwargs):
    
    # read data
    kwargs.setdefault('array_type', TimeSeries)
    series = read_hdf5_array(h5f, path=path, **kwargs)
    # crop if needed
    if start is not None or end is not None:
        return series.crop(start, end)
    return series