Retrieve the fastq directory within Solexa flowcell output.
188944
def get_fastq_dir(fc_dir):
    
    full_goat_bc = glob.glob(os.path.join(fc_dir, "Data", "*Firecrest*", "Bustard*"))
    bustard_bc = glob.glob(os.path.join(fc_dir, "Data", "Intensities", "*Bustard*"))
    machine_bc = os.path.join(fc_dir, "Data", "Intensities", "BaseCalls")
    if os.path.exists(machine_bc):
        return os.path.join(machine_bc, "fastq")
    elif len(full_goat_bc) > 0:
        return os.path.join(full_goat_bc[0], "fastq")
    elif len(bustard_bc) > 0:
        return os.path.join(bustard_bc[0], "fastq")
    # otherwise assume we are in the fastq directory
    # XXX What other cases can we end up with here?
    else:
        return fc_dir
758644
def _get_flowcell_id(in_file, require_single=True):
    
    fc_ids = set([x[0] for x in _read_input_csv(in_file)])
    if require_single and len(fc_ids) > 1:
        raise ValueError("There are several FCIDs in the same samplesheet file: %s" % in_file)
    else:
        return fc_ids
8178
def stream_fastq(file_handler):
    ''' 
    '''
    next_element = ''
    for i, line in enumerate(file_handler):
        next_element += line
        if i % 4 == 3:
            yield next_element
            next_element = ''
331491
def get_fastq_files(directory, lane, fc_name):
    
    files = glob.glob(os.path.join(directory, "%s_*%s*txt*" % (lane, fc_name)))
    files.sort()
    if len(files) > 2 or len(files) == 0:
        raise ValueError("Did not find correct files for %s %s %s %s" %
                (directory, lane, fc_name, files))
    ready_files = []
    for fname in files:
        if fname.endswith(".gz"):
            cl = ["gunzip", fname]
            subprocess.check_call(cl)
            ready_files.append(os.path.splitext(fname)[0])
        else:
            ready_files.append(fname)
    return ready_files[0], (ready_files[1] if len(ready_files) > 1 else None)
1101348
def reference_fasta(self):
        """"""
        if self._db_location:
            ref_files = glob.glob(os.path.join(self._db_location, "*", self._REF_FASTA))
            if ref_files:
                return ref_files[0]
548696
def fastq_to_csv(in_file, fastq_format, work_dir):
    
    out_file = "%s.csv" % (os.path.splitext(os.path.basename(in_file))[0])
    out_file = os.path.join(work_dir, out_file)
    if not (os.path.exists(out_file) and os.path.getsize(out_file) > 0):
        with open(in_file) as in_handle:
            with open(out_file, "w") as out_handle:
                writer = csv.writer(out_handle)
                for rec in SeqIO.parse(in_handle, fastq_format):
                    writer.writerow([rec.id] + rec.letter_annotations["phred_quality"])
    return out_file
463453
def read_fastq(filename):
    
    if not filename:
        return itertools.cycle((None,))
    if filename == "-":
        filename_fh = sys.stdin
    elif filename.endswith('gz'):
        if is_python3:
            filename_fh = gzip.open(filename, mode='rt')
        else:
            filename_fh = BufferedReader(gzip.open(filename, mode='rt'))
    else:
        filename_fh = open(filename)
    return stream_fastq(filename_fh)
169051
private VoltTable[] fastPath(List<QueuedSQL> batch, final boolean finalTask) {
        final int batchSize = batch.size();
        Object[] params = new Object[batchSize];
        long[] fragmentIds = new long[batchSize];
        String[] sqlTexts = new String[batchSize];
        boolean[] isWriteFrag = new boolean[batchSize];
        int[] sqlCRCs = new int[batchSize];
        int succeededFragmentsCount = 0;

        int i = 0;
        for (final QueuedSQL qs : batch) {
            assert (qs.stmt.collector == null);
            fragmentIds[i] = qs.stmt.aggregator.id;
            // use the pre-serialized params if it exists
            params[i] = qs.params;
            sqlTexts[i] = qs.stmt.getText();
            isWriteFrag[i] = !qs.stmt.isReadOnly;
            sqlCRCs[i] = SQLStmtAdHocHelper.getHash(qs.stmt);
            i++;
        }

        VoltTable[] results = null;
        // Before executing the fragments, tell the EE if this batch should be
        // timed.
        getExecutionEngine().setPerFragmentTimingEnabled(m_perCallStats.samplingStmts());
        try {
            FastDeserializer fragResult = m_site.executePlanFragments(batchSize, fragmentIds, null, params,
                    m_determinismHash, sqlTexts, isWriteFrag, sqlCRCs, m_txnState.txnId, m_txnState.m_spHandle,
                    m_txnState.uniqueId, m_isReadOnly, VoltTrace.log(VoltTrace.Category.EE) != null);
            final int totalSize;
            try {
                // read the size of the DR buffer used
                fragResult.readInt();
                // read the complete size of the buffer used
                totalSize = fragResult.readInt();
            } catch (final IOException ex) {
                log.error("Failed to deserialze result table" + ex);
                throw new EEException(ExecutionEngine.ERRORCODE_WRONG_SERIALIZED_BYTES);
            }
            final ByteBuffer rawDataBuff;
            if ((m_batchIndex == 0 && !m_site.usingFallbackBuffer()) || finalTask) {
                // If this is the first or final batch, skip the copy of the
                // underlying byte array
                rawDataBuff = fragResult.buffer();
            } else {
                rawDataBuff = fragResult.readBuffer(totalSize);
            }
            results = TableHelper.convertBackedBufferToTables(rawDataBuff, batchSize);
        } catch (Throwable ex) {
            if (!m_isReadOnly) {
                // roll back the current batch and re-throw the EE exception
                m_site.truncateUndoLog(true, false,
                        m_spBigBatchBeginToken >= 0 ? m_spBigBatchBeginToken : m_site.getLatestUndoToken(),
                        m_txnState.m_spHandle, null);
            }

            throw ex;
        } finally {
            long[] executionTimes = null;
            if (m_perCallStats.samplingStmts()) {
                executionTimes = new long[batchSize];
            }
            succeededFragmentsCount = getExecutionEngine().extractPerFragmentStats(batchSize, executionTimes);

            for (i = 0; i < batchSize; i++) {
                QueuedSQL qs = batch.get(i);
                // No coordinator task for a single partition procedure.
                boolean isCoordinatorTask = false;
                // If all the fragments in this batch are executed successfully,
                // succeededFragmentsCount == batchSize.
                // Otherwise, the fragment whose index equals
                // succeededFragmentsCount is the one that failed.
                boolean failed = i == succeededFragmentsCount;

                m_perCallStats.recordStatementStats(qs.stmt.getStmtName(), isCoordinatorTask, failed,
                        executionTimes == null ? 0 : executionTimes[i], results == null ? null : results[i], qs.params);

                // If this fragment failed, no subsequent fragments will be
                // executed.
                if (failed) {
                    break;
                }
            }
        }
        return results;
    }
991557
def snapshot(opts):
    
    seqrepo_dir = os.path.join(opts.root_directory, opts.instance_name)

    dst_dir = opts.destination_name
    if not dst_dir.startswith("/"):
        # interpret dst_dir as relative to parent dir of seqrepo_dir
        dst_dir = os.path.join(opts.root_directory, dst_dir)

    src_dir = os.path.realpath(seqrepo_dir)
    dst_dir = os.path.realpath(dst_dir)

    if commonpath([src_dir, dst_dir]).startswith(src_dir):
        raise RuntimeError("Cannot nest seqrepo directories " "({} is within {})".format(dst_dir, src_dir))

    if os.path.exists(dst_dir):
        raise IOError(dst_dir + ": File exists")

    tmp_dir = tempfile.mkdtemp(prefix=dst_dir + ".")

    _logger.debug("src_dir = " + src_dir)
    _logger.debug("dst_dir = " + dst_dir)
    _logger.debug("tmp_dir = " + tmp_dir)

    # TODO: cleanup of tmpdir on failure
    makedirs(tmp_dir, exist_ok=True)
    wd = os.getcwd()
    os.chdir(src_dir)

    # make destination directories (walk is top-down)
    for rp in (os.path.join(dirpath, dirname) for dirpath, dirnames, _ in os.walk(".") for dirname in dirnames):
        dp = os.path.join(tmp_dir, rp)
        os.mkdir(dp)

    # hard link sequence files
    for rp in (os.path.join(dirpath, filename) for dirpath, _, filenames in os.walk(".") for filename in filenames
               if ".bgz" in filename):
        dp = os.path.join(tmp_dir, rp)
        os.link(rp, dp)

    # copy sqlite databases
    for rp in ["aliases.sqlite3", "sequences/db.sqlite3"]:
        dp = os.path.join(tmp_dir, rp)
        shutil.copyfile(rp, dp)

    # recursively drop write perms on snapshot
    mode_aw = stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH

    def _drop_write(p):
        mode = os.lstat(p).st_mode
        new_mode = mode & ~mode_aw
        os.chmod(p, new_mode)

    for dp in (os.path.join(dirpath, dirent)
               for dirpath, dirnames, filenames in os.walk(tmp_dir) for dirent in dirnames + filenames):
        _drop_write(dp)
    _drop_write(tmp_dir)
    os.rename(tmp_dir, dst_dir)

    _logger.info("snapshot created in " + dst_dir)
    os.chdir(wd)
370111
def filer(self):
        
        # Get a list of all the .fastq files in the sequence path
        fastq = sorted(glob(os.path.join(self.sequencepath, '*fastq*')))
        # Initialise a list to store .fasta formatted files
        fasta = list()
        # Get all the files in the sequence path
        for sequencefile in glob(os.path.join(self.sequencepath, '*')):
            # Split the extension from the file
            extension = os.path.splitext(sequencefile)[1]
            # If the extension is one of the acceptable FASTA extensions, add it to the list of FASTA files
            if extension in self.extensions:
                fasta.append(sequencefile)
        # Use filer from accessoryFunctions to find the base file name of each .fastq file/pair
        fastqnames = filer(fastq)
        # Get all the .fastq files into metadata objects
        for fastqname in sorted(fastqnames):
            # Create the metadata object
            sample = MetadataObject()
            sample.name = os.path.basename(fastqname)
            # Create an attribute for the current analysis
            setattr(sample, self.analysistype, GenObject())
            # Set the destination folder
            sample[self.analysistype].outputdir = os.path.join(self.path, fastqname)
            # Make the destination folder
            make_path(sample[self.analysistype].outputdir)
            # Get the fastq files specific to the fastqname
            specificfastq = glob(os.path.join(self.sequencepath, '{fq}*.fastq*'.format(fq=fastqname)))
            # Set the file type for the downstream analysis
            sample[self.analysistype].filetype = 'fastq'
            # Link the files to the output folder
            for fastq in specificfastq:
                relative_symlink(src_file=fastq,
                                 output_dir=sample[self.analysistype].outputdir)
            # Initialise the general and run categories
            sample.general = GenObject()
            # Populate the .fastqfiles category of :self.metadata
            sample.general.fastqfiles = [fastq for fastq in
                                         glob(os.path.join(sample[self.analysistype].outputdir, '{fq}*.fastq*'
                                                           .format(fq=fastqname))) if 'trimmed' not in fastq]
            # Populate certain attributes in order to be compatible with other local software
            sample.general.bestassemblyfile = sample.general.fastqfiles
            sample.general.trimmedcorrectedfastqfiles = sample.general.fastqfiles
            sample.general.outputdirectory = sample[self.analysistype].outputdir
            sample[self.analysistype].baitedfastq = os.path.join(
                sample[self.analysistype].outputdir,
                '{at}_targetMatches.fastq.gz'.format(at=self.analysistype))
            # Append the metadata to the list of samples
            self.metadata.append(sample)
        # Add any .fasta formatted files to the metadata object
        for fastafile in sorted(fasta):
            sample = MetadataObject()
            sample.name = os.path.splitext(os.path.split(fastafile)[-1])[0]
            setattr(sample, self.analysistype, GenObject())
            # Set the destination folder
            sample[self.analysistype].outputdir = os.path.join(self.path, 'detailed_reports', sample.name)
            # Make the destination folder
            make_path(sample[self.analysistype].outputdir)
            # Set the file type for the downstream analysis
            sample[self.analysistype].filetype = 'fasta'
            # Link the .fasta files to :self.path/:filename
            relative_symlink(src_file=fastafile,
                             output_dir=sample[self.analysistype].outputdir)
            # Initialise the general and run categories
            sample.general = GenObject()
            # Populate the .fastqfiles category of :self.metadata
            sample.general.fastqfiles = [fastq for fastq in glob(os.path.join(sample[self.analysistype].outputdir,
                                                                              '{sn}*{ext}*'
                                                                              .format(
                                                                                  sn=sample.name,
                                                                                  ext=os.path.splitext(fastafile)[1])))]
            sample.general.bestassemblyfile = sample.general.fastqfiles
            sample.general.trimmedcorrectedfastqfiles = sample.general.fastqfiles
            sample.general.outputdirectory = sample[self.analysistype].outputdir
            sample[self.analysistype].baitedfastq = os.path.join(
                sample[self.analysistype].outputdir,
                '{at}_targetMatches.fastq.gz'.format(at=self.analysistype))
            sample[self.analysistype].assemblyfile = fastafile
            # Append the metadata to the list of samples
            self.metadata.append(sample)
1058553
def process_fastq_minimal(fastq, **kwargs):
    """"""
    infastq = handle_compressed_input(fastq)
    try:
        df = pd.DataFrame(
            data=[rec for rec in fq_minimal(infastq) if rec],
            columns=["timestamp", "lengths"]
        )
    except IndexError:
        logging.error("Fatal: Incorrect file structure for fastq_minimal")
        sys.exit("Error: file does not match expected structure for fastq_minimal")
    return ut.reduce_memory_usage(df)
1292129
def seq(self):
        """"""

        if self.sequence_file:
            file_to_load = copy(self.sequence_path)
            log.debug('{}: reading sequence from sequence file {}'.format(self.id, file_to_load))
            tmp_sr = SeqIO.read(file_to_load, 'fasta')
            return tmp_sr.seq

        else:
            if not self._seq:
                log.debug('{}: no sequence stored in memory'.format(self.id))
            else:
                log.debug('{}: reading sequence from memory'.format(self.id))

            return self._seq
666804
def fastq(args):
    
    p = OptionParser(fastq.__doc__)
    opts, args = p.parse_args(args)

    if len(args) != 2:
        sys.exit(not p.print_help())

    bamfile, pf = args
    singletons = pf + ".se.fastq"
    a = pf + ".read1.fastq"
    b = pf + ".read2.fastq"

    cmd  = "samtools collate -uOn 128 {} tmp-prefix".format(bamfile)
    cmd += " | samtools fastq -s {} -1 {} -2 {} -"\
                .format(singletons, a, b)
    sh(cmd)

    if os.stat(singletons).st_size == 0:  # singleton file is empty
        os.remove(singletons)
    return a, b
1137127
def get_fasta_record(fasta_file, transcript_name):
    
    with open_pysam_file(fname=fasta_file, ftype='fasta') as f:
        sequence = f.fetch(transcript_name)
    return {transcript_name: sequence}
1374520
def directory(self, query, **kwargs):
        """"""
        if isinstance(query, dict):
            query = str(query).replace("'", '"')

        return self.__call_api_get('directory', query=query, kwargs=kwargs)
714828
def parse_dirname(fc_dir):
    
    (_, fc_dir) = os.path.split(fc_dir)
    parts = fc_dir.split("_")
    name = None
    date = None
    for p in parts:
        if p.endswith(("XX", "xx", "XY", "X2")):
            name = p
        elif len(p) == 6:
            try:
                int(p)
                date = p
            except ValueError:
                pass
    if name is None or date is None:
        raise ValueError("Did not find flowcell name: %s" % fc_dir)
    return name, date
987325
def _read_seq_from_fasta(self, fasta, offset, nr_lines):
        
        fasta.seek(offset)
        lines = [fasta.readline().strip() for _ in range(nr_lines)]
        return "".join(lines)