Retrieve fastq files for the given lane, ready to process.
331491
def get_fastq_files(directory, lane, fc_name):
    
    files = glob.glob(os.path.join(directory, "%s_*%s*txt*" % (lane, fc_name)))
    files.sort()
    if len(files) > 2 or len(files) == 0:
        raise ValueError("Did not find correct files for %s %s %s %s" %
                (directory, lane, fc_name, files))
    ready_files = []
    for fname in files:
        if fname.endswith(".gz"):
            cl = ["gunzip", fname]
            subprocess.check_call(cl)
            ready_files.append(os.path.splitext(fname)[0])
        else:
            ready_files.append(fname)
    return ready_files[0], (ready_files[1] if len(ready_files) > 1 else None)
831557
def readfq(fp):
    """"""
    last = None  # this is a buffer keeping the last unprocessed line
    while True:  # mimic closure; is it a bad idea?
        if not last:  # the first record or a record following a fastq
            for l in fp:  # search for the start of the next record
                if l[0] in '>@':  # fasta/q header line
                    last = l[:-1]  # save this line
                    break
        if not last:
            break
        name, seqs, last = last[1:].partition(" ")[0], [], None
        for l in fp:  # read the sequence
            if l[0] in '@+>':
                last = l[:-1]
                break
            seqs.append(l[:-1])
        if not last or last[0] != '+':  # this is a fasta record
            yield name, ''.join(seqs), None  # yield a fasta record
            if not last:
                break
        else:  # this is a fastq record
            seq, leng, seqs = ''.join(seqs), 0, []
            for l in fp:  # read the quality
                seqs.append(l[:-1])
                leng += len(l) - 1
                if leng >= len(seq):  # have read enough quality
                    last = None
                    yield name, seq, ''.join(seqs)  # yield a fastq record
                    break
            if last:  # reach EOF before reading enough quality
                yield name, seq, None  # yield a fasta record instead
                break
1255493
def _organize_lanes(info_iter, barcode_ids):
    
    all_lanes = []
    for (fcid, lane, sampleref), info in itertools.groupby(info_iter, lambda x: (x[0], x[1], x[1])):
        info = list(info)
        cur_lane = dict(flowcell_id=fcid, lane=lane, genome_build=info[0][3], analysis="Standard")
        if not _has_barcode(info):
            cur_lane["description"] = info[0][1]
        else: # barcoded sample
            cur_lane["description"] = "Barcoded lane %s" % lane
            multiplex = []
            for (_, _, sample_id, _, bc_seq) in info:
                bc_type, bc_id = barcode_ids[bc_seq]
                multiplex.append(dict(barcode_type=bc_type,
                                      barcode_id=bc_id,
                                      sequence=bc_seq,
                                      name=sample_id))
            cur_lane["multiplex"] = multiplex
        all_lanes.append(cur_lane)
    return all_lanes
742119
def download(self):
        
        self.downloaded_paths = list()
        for path in self.paths_for_download:
            downloaded_path = list()
            utils.mkdir_p(os.path.abspath(self.directory))

            sra_run = path.split("/")[-1]
            logger.info("Analysing %s" % sra_run)
            url = type(self).FTP_ADDRESS_TPL.format(
                range_subdir=sra_run[:6],
                file_dir=sra_run)
            logger.debug("URL: %s", url)
            filepath = os.path.abspath(
                os.path.join(self.directory, "%s.sra" % sra_run))
            utils.download_from_url(
                url,
                filepath,
                aspera=self.aspera,
                silent=self.silent,
                force=self.force)

            if self.filetype in ("fasta", "fastq"):
                if utils.which('fastq-dump') is None:
                    logger.error("fastq-dump command not found")
                ftype = ""
                if self.filetype == "fasta":
                    ftype = " --fasta "
                cmd = "fastq-dump"
                if utils.which('parallel-fastq-dump') is None:
                    cmd += " %s --outdir %s %s"
                else:
                    logger.debug("Using parallel fastq-dump")
                    cmd = " parallel-fastq-dump --threads %s"
                    cmd = cmd % self.threads
                    cmd += " %s --outdir %s -s %s"
                cmd = cmd % (ftype, self.directory, filepath)

                for fqoption, fqvalue in iteritems(self.fastq_dump_options):
                    if fqvalue:
                        cmd += (" --%s %s" % (fqoption, fqvalue))
                    elif fqvalue is None:
                        cmd += (" --%s" % fqoption)
                logger.debug(cmd)
                process = sp.Popen(cmd, stdout=sp.PIPE,
                                   stderr=sp.PIPE,
                                   shell=True)
                logger.info("Converting to %s/%s*.%s.gz\n" % (
                    self.directory, sra_run, self.filetype))
                pout, perr = process.communicate()
                downloaded_path = glob.glob(os.path.join(
                    self.directory,
                    "%s*.%s.gz" % (sra_run, self.filetype)))

            elif self.filetype == 'sra':
                downloaded_path = glob.glob(os.path.join(
                    self.directory,
                    "%s*.%s" % (sra_run, self.filetype)))

            else:
                downloaded_path = glob.glob(os.path.join(
                    self.directory,
                    "%s*" % sra_run))
                logger.error("Filetype %s not supported." % self.filetype)

            if not self.keep_sra and self.filetype != 'sra':
                # Delete sra file
                os.unlink(filepath)

            self.downloaded_paths += downloaded_path
        return self.downloaded_paths
930752
def lane_info(self):
        
        resource_list = self.traffic_incident()
        lane_info = namedtuple('lane_info', 'lane_info')
        if len(resource_list) == 1 and resource_list[0] is None:
            return None
        else:
            try:
                return [lane_info(resource['lane'])
                        for resource in resource_list]
            except (KeyError, TypeError):
                try:
                    return [lane_info(resource['LaneInfo'])
                            for resource in resource_list]
                except KeyError:
                    return None
666804
def fastq(args):
    
    p = OptionParser(fastq.__doc__)
    opts, args = p.parse_args(args)

    if len(args) != 2:
        sys.exit(not p.print_help())

    bamfile, pf = args
    singletons = pf + ".se.fastq"
    a = pf + ".read1.fastq"
    b = pf + ".read2.fastq"

    cmd  = "samtools collate -uOn 128 {} tmp-prefix".format(bamfile)
    cmd += " | samtools fastq -s {} -1 {} -2 {} -"\
                .format(singletons, a, b)
    sh(cmd)

    if os.stat(singletons).st_size == 0:  # singleton file is empty
        os.remove(singletons)
    return a, b
442462
def get_cf_files(path, queue):
    """"""
    for root, _, files in os.walk(os.path.abspath(path)):
        if not files:
            continue
        for filename in files:
            fullname = os.path.join(root, filename)
            if os.path.isfile(fullname) and fullname.endswith('.cf') or \
                    fullname.endswith('.post'):
                queue.put(fullname)
1127265
def StartFileFetch(self, pathspec, request_data=None):
    """"""
    # Create an index so we can find this pathspec later.
    self.state.indexed_pathspecs.append(pathspec)
    self.state.request_data_list.append(request_data)
    self._TryToStartNextPathspec()
184065
def get_nowait_from_queue(queue):
    
    data = []
    for _ in range(queue.qsize()):
        try:
            data.append(queue.get_nowait())
        except q.Empty:
            break
    return data
317367
def queue_files(dirpath, queue):
    """"""
    for root, _, files in os.walk(os.path.abspath(dirpath)):
        if not files:
            continue
        for filename in files:
            queue.put(os.path.join(root, filename))
1236379
def fast_relpath(path, start):
  """"""
  relpath = fast_relpath_optional(path, start)
  if relpath is None:
    raise ValueError('{} is not a directory containing {}'.format(start, path))
  return relpath
1303438
def fasta(self):
        
        logging.info('Converting FASTQ files to FASTA format')
        # Create the threads for the analysis
        for _ in range(self.cpus):
            threads = Thread(target=self.fastathreads, args=())
            threads.setDaemon(True)
            threads.start()
        with progressbar(self.runmetadata.samples) as bar:
            for sample in bar:
                if sample.general.bestassemblyfile != 'NA':
                    # Set the name as the FASTA file - the same as the FASTQ, but with .fa file extension
                    sample[self.analysistype].fasta = \
                        os.path.splitext(sample[self.analysistype].subsampledfastq)[0] + '.fa'
                    # Set the system call
                    sample[self.analysistype].reformatcall = 'reformat.sh in={fastq} out={fasta}'\
                        .format(fastq=sample[self.analysistype].subsampledfastq,
                                fasta=sample[self.analysistype].fasta)
                    # Add the sample to the queue
                    self.fastaqueue.put(sample)
        self.fastaqueue.join()
854312
def rec_load_all(self, zone):
        
        has_more = True
        current_count = 0
        while has_more:
            records = self._request({
                'a': 'rec_load_all',
                'o': current_count,
                'z': zone
            })
            try:
                has_more = records['response']['recs']['has_more']
                current_count += records['response']['recs']['count']
                for record in records['response']['recs']['objs']:
                    yield record
            except KeyError:
                has_more = False
814889
def get_fastq_files(data):
    
    assert "files" in data, "Did not find `files` in input; nothing to process"
    ready_files = []
    should_gzip = True

    # Bowtie does not accept gzipped fastq
    if 'bowtie' in data['reference'].keys():
        should_gzip = False
    for fname in data["files"]:
        if fname.endswith(".bam"):
            if _pipeline_needs_fastq(data["config"], data):
                ready_files = convert_bam_to_fastq(fname, data["dirs"]["work"],
                                                   data, data["dirs"], data["config"])
            else:
                ready_files = [fname]
        elif objectstore.is_remote(fname):
            ready_files.append(fname)
        # Trimming does quality conversion, so if not doing that, do an explicit conversion
        elif not(dd.get_trim_reads(data)) and dd.get_quality_format(data) != "standard":
            out_dir = utils.safe_makedir(os.path.join(dd.get_work_dir(data), "fastq_convert"))
            ready_files.append(fastq.groom(fname, data, out_dir=out_dir))
        else:
            ready_files.append(fname)
    ready_files = [x for x in ready_files if x is not None]
    if should_gzip:
        out_dir = utils.safe_makedir(os.path.join(dd.get_work_dir(data), "fastq"))
        ready_files = [_gzip_fastq(x, out_dir) for x in ready_files]
    for in_file in ready_files:
        if not objectstore.is_remote(in_file):
            assert os.path.exists(in_file), "%s does not exist." % in_file
    return ready_files
749532
def _queue(self, kwargs):
    """"""
    if not 'hard resource_list' in kwargs: return 'all.q'
    d = dict([k.split('=') for k in kwargs['hard resource_list'].split(',')])
    for k in d:
      if k[0] == 'q' and d[k] == 'TRUE': return k
    return 'all.q'
548696
def fastq_to_csv(in_file, fastq_format, work_dir):
    
    out_file = "%s.csv" % (os.path.splitext(os.path.basename(in_file))[0])
    out_file = os.path.join(work_dir, out_file)
    if not (os.path.exists(out_file) and os.path.getsize(out_file) > 0):
        with open(in_file) as in_handle:
            with open(out_file, "w") as out_handle:
                writer = csv.writer(out_handle)
                for rec in SeqIO.parse(in_handle, fastq_format):
                    writer.writerow([rec.id] + rec.letter_annotations["phred_quality"])
    return out_file
746151
def createfastq(self):
        """"""
        # If the fastq destination folder is not provided, make the default value of :path/:miseqfoldername
        self.fastqdestination = self.fastqdestination if self.fastqdestination else \
            os.path.join(self.path, self.miseqfoldername)
        # Make the path
        make_path(self.fastqdestination)
        # Create a new sample sheet using self.project name instead of the provided Sample_Project. This ensures
        # that all the FASTQ files are stored in the same output folder
        projectsamplesheet = os.path.join(self.fastqdestination, 'SampleSheet_modified.csv')
        with open(projectsamplesheet, "w") as modifiedsamplesheet:
            # Use the 'original' sample sheet as the template for the new sheet
            with open(self.customsamplesheet) as samplesheet:
                # Iterate through the template sheet, and write lines until the header for the data portion of the sheet
                for line in samplesheet:
                    modifiedsamplesheet.write(line)
                    if 'Sample_ID' in line:
                        # Create a list of the header values
                        header = line.split(',')
                        for subline in samplesheet:
                            # Split the line on commas
                            data = subline.split(',')
                            # Initialise a list to store the values for each sample
                            updateddata = list()
                            # Iterate through the entries in the header, and extract the corresponding value
                            for i, value in enumerate(header):
                                # Find the Sample_Project value, and update it to be self.projectname
                                if data[i] in self.projectlist:
                                    data[i] = self.projectname
                                # If demultiplexing is disabled, don't add the samples to the SampleSheet
                                if self.demultiplex:
                                    # Add the (modified) data to the list
                                    updateddata.append(data[i])
                            # Write the updated string to the new sheet
                            modifiedsamplesheet.write(','.join(updateddata))
        # Set :forward/reverse length to :header.forward/reverse length if the argument is not provided, or it's 'full',
        # otherwise  use the supplied argument
        self.forwardlength = self.header.forwardlength if self.forwardlength.lower()\
            == 'full' else self.forwardlength
        # Set :reverselength to :header.reverselength
        self.reverselength = self.header.reverselength if self.reverselength.lower() \
            == 'full' else self.reverselength
        # As the number of cycles required is the number of forward reads + the index(8) + the second index(8)
        # Also set the basemask variable as required
        if self.reverselength != '0':
            self.readsneeded = int(self.forwardlength) + int(self.reverselength) + self.indexlength
            basemask = "Y{}n*,{},Y{}n*".format(self.forwardlength, self.index, self.reverselength)
        else:
            self.readsneeded = int(self.forwardlength) + self.indexlength
            basemask = "Y{}n*,{},n*".format(self.forwardlength, self.index)
        # Handle plurality appropriately
        samples = 'samples' if self.samplecount != 1 else 'sample'
        number = 'are' if self.samplecount != 1 else 'is'
        logging.info('There {num} {num_samples} {plural} in this run.\n'
                     'MiSeqPath: {miseqpath},\n'
                     'MiSeqFolder: {miseqfolder},\n'
                     'FASTQ destination: {destination},\n'
                     'SampleSheet: {sample_sheet}'
                     .format(num=number,
                             num_samples=self.samplecount,
                             plural=samples,
                             miseqpath=self.miseqpath,
                             miseqfolder=self.miseqfolder,
                             destination=self.fastqdestination,
                             sample_sheet=projectsamplesheet))
        # Count the number of completed cycles in the run of interest
        cycles = glob(os.path.join(self.miseqpath, self.miseqfolder, 'Data', 'Intensities', 'BaseCalls', 'L001', 'C*'))
        while len(cycles) < self.readsneeded:
            logging.info('Currently at {num_cycles} cycles. Waiting until the MiSeq reaches cycle {target_cycle}'
                         .format(num_cycles=len(cycles),
                                 target_cycle=self.readsneeded))
            sleep(300)
            cycles = glob(os.path.join(self.miseqpath, self.miseqfolder,
                                       'Data', 'Intensities', 'BaseCalls', 'L001', 'C*'))
        # configureBClToFastq requires :self.miseqfolder/Data/Intensities/BaseCalls/config.xml in order to work
        # When you download runs from BaseSpace, this file is not provided. There is an empty config.xml file that
        # can be populated with run-specific values and moved to the appropriate folder
        if not os.path.isfile(os.path.join(self.miseqfolder, 'Data', 'Intensities', 'BaseCalls', 'config.xml')):
            self.configfilepopulator()
        if self.debug:
            # Define the bcl2fastq system call for the unit test
            bclcall = "bcl2fastq --input-dir {basecalls} " \
                      "--output-dir {outdir} --sample-sheet {samplesheet} " \
                      "--barcode-mismatches 0 -r 1 -p 1 -w 1 -R {runfolder} --use-bases-mask {mask} " \
                      "--tiles s_1_1101 --minimum-trimmed-read-length 1" \
                .format(basecalls=os.path.join(self.miseqfolder, 'Data', 'Intensities', 'BaseCalls'),
                        outdir=self.fastqdestination,
                        samplesheet=projectsamplesheet,
                        runfolder=self.miseqfolder,
                        mask=basemask)
        # elif not self.demultiplex:
        #     bclcall = "bcl2fastq --input-dir {basecalls} " \
        #               "--output-dir {outdir} --sample-sheet {samplesheet} --no-lane-splitting " \
        #               "-r 1 -p 1 -w 1 -R {runfolder} --use-bases-mask {mask}"\
        #         .format(basecalls=os.path.join(self.miseqfolder, 'Data', 'Intensities', 'BaseCalls'),
        #                 outdir=self.fastqdestination,
        #                 samplesheet=projectsamplesheet,
        #                 runfolder=self.miseqfolder,
        #                 mask=basemask)
        else:
            bclcall = "bcl2fastq --input-dir {basecalls} " \
                      "--output-dir {outdir} --sample-sheet {samplesheet} " \
                      "--barcode-mismatches 1 -r 1 -p 1 -w 1 -R {runfolder} --use-bases-mask {mask}"\
                .format(basecalls=os.path.join(self.miseqfolder, 'Data', 'Intensities', 'BaseCalls'),
                        outdir=self.fastqdestination,
                        samplesheet=projectsamplesheet,
                        runfolder=self.miseqfolder,
                        mask=basemask)
        process = False
        if self.demultiplex:
            if not os.path.isdir(self.projectpath):
                process = True
        else:
            if not os.path.isfile(os.path.join(self.fastqdestination, 'Undetermined_S0_R1_001.fastq.gz')):
                process = True
        if process:
            # Call bcl2fastq
            logging.info('Running bcl2fastq')
            # Run the command
            out, err = run_subprocess(bclcall)
            write_to_logfile(bclcall,
                             bclcall,
                             self.logfile)
            write_to_logfile(out,
                             err,
                             self.logfile)
        # Populate the metadata
        for sample in self.metadata.samples:
            sample.commands = GenObject()
            sample.commands.bcl = bclcall
            sample.run.forwardlength = self.forwardlength
            sample.run.reverselength = self.reverselength
        # Copy the fastq files to a central folder so they can be processed
        self.fastqmover()