Grooms a FASTQ file from Illumina 1.3/1.5 quality scores into
    sanger format, if it is not already in that format.
197056
def groom(in_file, data, in_qual="illumina", out_dir=None, out_file=None):
    
    if not out_file.endswith("gz"):
        out_file = "%s.gz" % out_file
    seqtk = config_utils.get_program("seqtk", data["config"])
    if in_qual == "fastq-sanger":
        logger.info("%s is already in Sanger format." % in_file)
        return out_file
    with file_transaction(out_file) as tmp_out_file:
        cmd = "{seqtk} seq -Q64 {in_file} | gzip > {tmp_out_file}".format(**locals())
        do.run(cmd, "Converting %s to Sanger format." % in_file)
    return out_file
451199
def process_fastq_plain(fastq, **kwargs):
    """"""
    logging.info("Nanoget: Starting to collect statistics from plain fastq file.")
    inputfastq = handle_compressed_input(fastq)
    return ut.reduce_memory_usage(pd.DataFrame(
        data=[res for res in extract_from_fastq(inputfastq) if res],
        columns=["quals", "lengths"]
    ).dropna())
370111
def filer(self):
        
        # Get a list of all the .fastq files in the sequence path
        fastq = sorted(glob(os.path.join(self.sequencepath, '*fastq*')))
        # Initialise a list to store .fasta formatted files
        fasta = list()
        # Get all the files in the sequence path
        for sequencefile in glob(os.path.join(self.sequencepath, '*')):
            # Split the extension from the file
            extension = os.path.splitext(sequencefile)[1]
            # If the extension is one of the acceptable FASTA extensions, add it to the list of FASTA files
            if extension in self.extensions:
                fasta.append(sequencefile)
        # Use filer from accessoryFunctions to find the base file name of each .fastq file/pair
        fastqnames = filer(fastq)
        # Get all the .fastq files into metadata objects
        for fastqname in sorted(fastqnames):
            # Create the metadata object
            sample = MetadataObject()
            sample.name = os.path.basename(fastqname)
            # Create an attribute for the current analysis
            setattr(sample, self.analysistype, GenObject())
            # Set the destination folder
            sample[self.analysistype].outputdir = os.path.join(self.path, fastqname)
            # Make the destination folder
            make_path(sample[self.analysistype].outputdir)
            # Get the fastq files specific to the fastqname
            specificfastq = glob(os.path.join(self.sequencepath, '{fq}*.fastq*'.format(fq=fastqname)))
            # Set the file type for the downstream analysis
            sample[self.analysistype].filetype = 'fastq'
            # Link the files to the output folder
            for fastq in specificfastq:
                relative_symlink(src_file=fastq,
                                 output_dir=sample[self.analysistype].outputdir)
            # Initialise the general and run categories
            sample.general = GenObject()
            # Populate the .fastqfiles category of :self.metadata
            sample.general.fastqfiles = [fastq for fastq in
                                         glob(os.path.join(sample[self.analysistype].outputdir, '{fq}*.fastq*'
                                                           .format(fq=fastqname))) if 'trimmed' not in fastq]
            # Populate certain attributes in order to be compatible with other local software
            sample.general.bestassemblyfile = sample.general.fastqfiles
            sample.general.trimmedcorrectedfastqfiles = sample.general.fastqfiles
            sample.general.outputdirectory = sample[self.analysistype].outputdir
            sample[self.analysistype].baitedfastq = os.path.join(
                sample[self.analysistype].outputdir,
                '{at}_targetMatches.fastq.gz'.format(at=self.analysistype))
            # Append the metadata to the list of samples
            self.metadata.append(sample)
        # Add any .fasta formatted files to the metadata object
        for fastafile in sorted(fasta):
            sample = MetadataObject()
            sample.name = os.path.splitext(os.path.split(fastafile)[-1])[0]
            setattr(sample, self.analysistype, GenObject())
            # Set the destination folder
            sample[self.analysistype].outputdir = os.path.join(self.path, 'detailed_reports', sample.name)
            # Make the destination folder
            make_path(sample[self.analysistype].outputdir)
            # Set the file type for the downstream analysis
            sample[self.analysistype].filetype = 'fasta'
            # Link the .fasta files to :self.path/:filename
            relative_symlink(src_file=fastafile,
                             output_dir=sample[self.analysistype].outputdir)
            # Initialise the general and run categories
            sample.general = GenObject()
            # Populate the .fastqfiles category of :self.metadata
            sample.general.fastqfiles = [fastq for fastq in glob(os.path.join(sample[self.analysistype].outputdir,
                                                                              '{sn}*{ext}*'
                                                                              .format(
                                                                                  sn=sample.name,
                                                                                  ext=os.path.splitext(fastafile)[1])))]
            sample.general.bestassemblyfile = sample.general.fastqfiles
            sample.general.trimmedcorrectedfastqfiles = sample.general.fastqfiles
            sample.general.outputdirectory = sample[self.analysistype].outputdir
            sample[self.analysistype].baitedfastq = os.path.join(
                sample[self.analysistype].outputdir,
                '{at}_targetMatches.fastq.gz'.format(at=self.analysistype))
            sample[self.analysistype].assemblyfile = fastafile
            # Append the metadata to the list of samples
            self.metadata.append(sample)
307452
def validate(args):
    
    import pyfasta

    p = OptionParser(validate.__doc__)
    p.add_option("--prefix", help="Add prefix to seqid")
    opts, args = p.parse_args(args)

    vcffile, fastafile = args
    pf = opts.prefix
    genome = pyfasta.Fasta(fastafile, record_class=pyfasta.MemoryRecord)
    fp = must_open(vcffile)
    match_ref = match_alt = total = 0
    for row in fp:
        if row[0] == '#':
            continue
        seqid, pos, id, ref, alt = row.split()[:5]
        total += 1
        if pf:
            seqid = pf + seqid
        pos = int(pos)
        if seqid not in genome:
            continue
        true_ref = genome[seqid][pos - 1]
        if total % 100000 == 0:
            print(total, "sites parsed", file=sys.stderr)
        if ref == true_ref:
            match_ref += 1
        elif alt == true_ref:
            match_alt += 1

    logging.debug("Match REF: {}".format(percentage(match_ref, total)))
    logging.debug("Match ALT: {}".format(percentage(match_alt, total)))
365232
def fastqmover(self):
        """"""
        # Create the sequence path if necessary
        make_path(self.sequencepath)
        # Iterate through all the sample names
        for sample in self.metadata.samples:
            # Make directory variables
            outputdir = os.path.join(self.sequencepath, sample.name)
            # Demultiplexed files will be present in the project name subfolder within the fastq destination folder
            if self.demultiplex:
                glob_dir = self.projectpath
            # Undemultiplexed reads (Undetermined_S0_R1_001.fastq.gz) are present in the fastq destination folder
            else:
                glob_dir = self.fastqdestination
            # Sometimes the files are put in self.fastqdestination rather than self.projectpath? Matt has this problem
            # and I don't understands what's going on
            if not os.path.isdir(glob_dir):
                glob_dir = os.path.dirname(glob_dir)
            # Glob all the .gz files in the subfolders - projectpath/Sample_:sample.name/*.gz
            for fastq in sorted(glob(os.path.join(glob_dir, '*.gz'))):
                fastqname = os.path.basename(fastq)
                # Set the name of the destination file
                outputfile = os.path.join(self.sequencepath, fastqname)
                # Link the file if it doesn't already exist
                if not os.path.isfile(outputfile):
                    relative_symlink(src_file=fastq,
                                     output_dir=self.sequencepath)
            # Repopulate .strainfastqfiles with the freshly-linked/copied files
            if self.demultiplex:
                fastqfiles = glob(os.path.join(self.sequencepath, '{}*.fastq*'.format(sample.name)))
                fastqfiles = sorted([fastq for fastq in fastqfiles if 'trimmed' not in os.path.basename(fastq)])
            # Undemultiplexed files will not have the sample name in the file name
            else:
                fastqfiles = sorted(glob(os.path.join(glob_dir, '*.gz')))
            # Populate the metadata object with the name/path of the fastq files
            sample.general.fastqfiles = fastqfiles
            # Save the outputdir to the metadata object
            sample.run.outputdirectory = outputdir
            sample.general.outputdirectory = outputdir
            sample.general.bestassemblyfile = True
            sample.general.trimmedcorrectedfastqfiles = sorted(sample.general.fastqfiles)
            sample.general.logout = os.path.join(sample.general.outputdirectory, 'logout')
            sample.general.logerr = os.path.join(sample.general.outputdirectory, 'logerr')
            sample.commands = GenObject()
751101
def qual(args):
    
    from jcvi.formats.sizes import Sizes

    p = OptionParser(qual.__doc__)
    p.add_option("--qv", default=31, type="int",
                 help="Dummy qv score for extended bases")
    p.set_outfile()
    opts, args = p.parse_args(args)

    if len(args) != 1:
        sys.exit(not p.print_help())

    fastafile, = args
    sizes = Sizes(fastafile)
    qvchar = str(opts.qv)
    fw = must_open(opts.outfile, "w")
    total = 0
    for s, slen in sizes.iter_sizes():
        print(">" + s, file=fw)
        print(" ".join([qvchar] * slen), file=fw)
        total += 1
    fw.close()
    logging.debug("Written {0} records in `{1}`.".format(total, opts.outfile))
909234
def probes_used_extract_scores(full_scores, same_probes):
  """"""
  if full_scores.shape[1] != same_probes.shape[0]: raise "Size mismatch"
  import numpy as np
  model_scores = np.ndarray((full_scores.shape[0],np.sum(same_probes)), 'float64')
  c=0
  for i in range(0,full_scores.shape[1]):
    if same_probes[i]:
      for j in range(0,full_scores.shape[0]):
        model_scores[j,c] = full_scores[j,i]
      c+=1
  return model_scores
194759
def align(fastq_file, pair_file, ref_file, names, align_dir, data,
          extra_args=None):
    
    config = data["config"]
    analysis_config = ANALYSIS.get(data["analysis"].lower())
    assert analysis_config, "Analysis %s is not supported by bowtie2" % (data["analysis"])
    out_file = os.path.join(align_dir, "{0}-sort.bam".format(dd.get_sample_name(data)))
    if data.get("align_split"):
        final_file = out_file
        out_file, data = alignprep.setup_combine(final_file, data)
        fastq_file, pair_file = alignprep.split_namedpipe_cls(fastq_file, pair_file, data)
    else:
        final_file = None
    if not utils.file_exists(out_file) and (final_file is None or not utils.file_exists(final_file)):
        with postalign.tobam_cl(data, out_file, pair_file is not None) as (tobam_cl, tx_out_file):
            cl = [config_utils.get_program("bowtie2", config)]
            cl += extra_args if extra_args is not None else []
            cl += ["-q",
                   "-x", ref_file]
            cl += analysis_config.get("params", [])
            if pair_file:
                cl += ["-1", fastq_file, "-2", pair_file]
            else:
                cl += ["-U", fastq_file]
            if names and "rg" in names:
                cl += ["--rg-id", names["rg"]]
                for key, tag in [("sample", "SM"), ("pl", "PL"), ("pu", "PU"), ("lb", "LB")]:
                    if names.get(key):
                        cl += ["--rg", "%s:%s" % (tag, names[key])]
            cl += _bowtie2_args_from_config(config, cl)
            cl = [str(i) for i in cl]
            cmd = "unset JAVA_HOME && " + " ".join(cl) + " | " + tobam_cl
            do.run(cmd, "Aligning %s and %s with Bowtie2." % (fastq_file, pair_file))
    return out_file
317180
def fasta(args):
    
    p = OptionParser(fasta.__doc__)
    p.add_option("--seqtk", default=False, action="store_true",
                 help="Use seqtk to convert")
    p.set_outdir()
    p.set_outfile(outfile=None)
    opts, args = p.parse_args(args)

    if len(args) < 1:
        sys.exit(not p.print_help())

    fastqfiles = args
    outdir = opts.outdir
    if outdir and outdir != ".":
        mkdir(outdir)

    fastqfile = fastqfiles[0]
    pf = op.basename(fastqfile)
    gzinput = pf.endswith(".gz")
    if gzinput:
        pf = pf.rsplit(".", 1)[0]

    pf, sf = pf.rsplit(".", 1)
    if sf not in ("fq", "fastq"):
        logging.debug("Assumed FASTA: suffix not `fq` or `fastq`")
        return fastqfile, None

    fastafile, qualfile = pf + ".fasta", pf + ".qual"
    outfile = opts.outfile or fastafile
    outfile = op.join(outdir, outfile)
    if opts.seqtk:
        if need_update(fastqfiles, outfile):
            for i, fastqfile in enumerate(fastqfiles):
                cmd = "seqtk seq -A {0} -L 30 -l 70".format(fastqfile)
                # First one creates file, following ones append to it
                sh(cmd, outfile=outfile, append=i)
        else:
            logging.debug("Outfile `{0}` already exists.".format(outfile))
        return outfile, None

    for fastqfile in fastqfiles:
        SeqIO.convert(fastqfile, "fastq", fastafile, "fasta")
        SeqIO.convert(fastqfile, "fastq", qualfile, "qual")

    return fastafile, qualfile
314623
def prep_fastq_inputs(in_files, data):
    
    if len(in_files) == 1 and _is_bam_input(in_files):
        out = _bgzip_from_bam(in_files[0], data["dirs"], data)
    elif len(in_files) == 1 and _is_cram_input(in_files):
        out = _bgzip_from_cram(in_files[0], data["dirs"], data)
    elif len(in_files) in [1, 2] and _ready_gzip_fastq(in_files, data):
        out = _symlink_in_files(in_files, data)
    else:
        if len(in_files) > 2:
            fpairs = fastq.combine_pairs(in_files)
            pair_types = set([len(xs) for xs in fpairs])
            assert len(pair_types) == 1
            fpairs.sort(key=lambda x: os.path.basename(x[0]))
            organized = [[xs[0] for xs in fpairs]]
            if len(fpairs[0]) > 1:
                organized.append([xs[1] for xs in fpairs])
            in_files = organized
        parallel = {"type": "local", "num_jobs": len(in_files),
                    "cores_per_job": max(1, data["config"]["algorithm"]["num_cores"] // len(in_files))}
        inputs = [{"in_file": x, "read_num": i, "dirs": data["dirs"], "config": data["config"],
                   "is_cwl": "cwl_keys" in data,
                   "rgnames": data["rgnames"]}
                  for i, x in enumerate(in_files) if x]
        out = run_multicore(_bgzip_from_fastq_parallel, [[d] for d in inputs], data["config"], parallel)
    return out
736957
def _merge_list_fastqs(files, out_file, config):
    """"""
    if not all(map(fastq.is_fastq, files)):
        raise ValueError("Not all of the files to merge are fastq files: %s " % (files))
    assert all(map(utils.file_exists, files)), ("Not all of the files to merge "
                                                "exist: %s" % (files))
    if not file_exists(out_file):
        files = [_gzip_fastq(fn) for fn in files]
        if len(files) == 1:
            if "remove_source" in config and config["remove_source"]:
                shutil.move(files[0], out_file)
            else:
                os.symlink(files[0], out_file)
            return out_file
        with file_transaction(out_file) as file_txt_out:
            files_str = " ".join(list(files))
            cmd = "cat {files_str} > {file_txt_out}".format(**locals())
            do.run(cmd, "merge fastq files %s" % files)
    return out_file
1341885
def export(self, xformat='csv'):
        """"""
        if self.annot is None:  # remove if buttons are disabled
            self.parent.statusBar().showMessage('No score file loaded')
            return

        if xformat == 'csv':
            filename = splitext(self.annot.xml_file)[0] + '.csv'
            filename, _ = QFileDialog.getSaveFileName(self, 'Export stages',
                                                      filename,
                                                      'Sleep stages (*.csv)')
        if 'remlogic' in xformat:
            filename = splitext(self.annot.xml_file)[0] + '.txt'
            filename, _ = QFileDialog.getSaveFileName(self, 'Export stages',
                                                      filename,
                                                      'Sleep stages (*.txt)')
        if filename == '':
            return

        self.annot.export(filename, xformat=xformat)
814889
def get_fastq_files(data):
    
    assert "files" in data, "Did not find `files` in input; nothing to process"
    ready_files = []
    should_gzip = True

    # Bowtie does not accept gzipped fastq
    if 'bowtie' in data['reference'].keys():
        should_gzip = False
    for fname in data["files"]:
        if fname.endswith(".bam"):
            if _pipeline_needs_fastq(data["config"], data):
                ready_files = convert_bam_to_fastq(fname, data["dirs"]["work"],
                                                   data, data["dirs"], data["config"])
            else:
                ready_files = [fname]
        elif objectstore.is_remote(fname):
            ready_files.append(fname)
        # Trimming does quality conversion, so if not doing that, do an explicit conversion
        elif not(dd.get_trim_reads(data)) and dd.get_quality_format(data) != "standard":
            out_dir = utils.safe_makedir(os.path.join(dd.get_work_dir(data), "fastq_convert"))
            ready_files.append(fastq.groom(fname, data, out_dir=out_dir))
        else:
            ready_files.append(fname)
    ready_files = [x for x in ready_files if x is not None]
    if should_gzip:
        out_dir = utils.safe_makedir(os.path.join(dd.get_work_dir(data), "fastq"))
        ready_files = [_gzip_fastq(x, out_dir) for x in ready_files]
    for in_file in ready_files:
        if not objectstore.is_remote(in_file):
            assert os.path.exists(in_file), "%s does not exist." % in_file
    return ready_files
710082
def align(fastq_file, pair_file, ref_file, names, align_dir, data,
          extra_args=None):
    
    num_hits = 1
    if data["analysis"].lower().startswith("smallrna-seq"):
        num_hits = 1000
    config = data['config']
    out_file = os.path.join(align_dir, "{0}-sort.bam".format(dd.get_sample_name(data)))
    if data.get("align_split"):
        final_file = out_file
        out_file, data = alignprep.setup_combine(final_file, data)
        fastq_file, pair_file = alignprep.split_namedpipe_cls(fastq_file, pair_file, data)
    else:
        final_file = None
        if fastq_file.endswith(".gz"):
            fastq_file = "<(gunzip -c %s)" % fastq_file
            if pair_file:
                pair_file = "<(gunzip -c %s)" % pair_file

    if not utils.file_exists(out_file) and (final_file is None or not utils.file_exists(final_file)):
        with postalign.tobam_cl(data, out_file, pair_file is not None) as (tobam_cl, tx_out_file):
            cl = [config_utils.get_program("bowtie", config)]
            cl += _bowtie_args_from_config(data)
            cl += extra_args if extra_args is not None else []
            cl += ["-q",
                   "-v", 2,
                   "-k", num_hits,
                   "-X", 2000, # default is too selective for most data
                   "--best",
                   "--strata",
                   "--sam",
                   ref_file]
            if pair_file:
                cl += ["-1", fastq_file, "-2", pair_file]
            else:
                cl += [fastq_file]
            cl = [str(i) for i in cl]
            fix_rg_cmd = r"samtools addreplacerg -r '%s' -" % novoalign.get_rg_info(data["rgnames"])
            if fix_rg_cmd:
                cmd = " ".join(cl) + " | " + fix_rg_cmd + " | " + tobam_cl
            else:
                cmd = " ".join(cl) + " | " + tobam_cl
            do.run(cmd, "Running Bowtie on %s and %s." % (fastq_file, pair_file), data)
    return out_file
1009709
def gen_nf_quick_check(output, ascii_props=False, append=False, prefix=""):
    """"""

    categories = []
    nf = {}
    all_chars = ALL_ASCII if ascii_props else ALL_CHARS
    file_name = os.path.join(HOME, 'unicodedata', UNIVERSION, 'DerivedNormalizationProps.txt')
    with codecs.open(file_name, 'r', 'utf-8') as uf:
        for line in uf:
            if not line.startswith('#'):
                data = line.split('#')[0].split(';')
                if len(data) < 2:
                    continue
                if not data[1].strip().lower().endswith('_qc'):
                    continue
                span = create_span([int(i, 16) for i in data[0].strip().split('..')], is_bytes=ascii_props)
                if span is None:
                    continue
                name = format_name(data[1][:-3] + 'quickcheck')
                subvalue = format_name(data[2])

                if name not in nf:
                    nf[name] = {}
                    categories.append(name)
                if subvalue not in nf[name]:
                    nf[name][subvalue] = []
                nf[name][subvalue].extend(span)

    for k1, v1 in nf.items():
        temp = set()
        for k2 in list(v1.keys()):
            temp |= set(v1[k2])
        v1['y'] = list(all_chars - temp)

    for k1, v1 in nf.items():
        for name in list(v1.keys()):
            s = set(nf[k1][name])
            nf[k1][name] = sorted(s)

    # Convert characters values to ranges
    char2range(nf, is_bytes=ascii_props)

    with codecs.open(output, 'a' if append else 'w', 'utf-8') as f:
        if not append:
            f.write(HEADER)
        for key, value in sorted(nf.items()):
            # Write out the Unicode properties
            f.write('%s_%s = {\n' % (prefix, key.replace('quickcheck', '_quick_check')))
            count = len(value) - 1
            i = 0
            for k1, v1 in sorted(value.items()):
                f.write('    "%s": "%s"' % (k1, v1))
                if i == count:
                    f.write('\n}\n')
                else:
                    f.write(',\n')
                i += 1

    return categories
596155
def _get_fastq_files(ldetail, read, fastq_dir):
    
    return glob.glob(os.path.join(fastq_dir, "Project_%s" % ldetail["project_name"],
                                  "Sample_%s" % ldetail["name"],
                                  "%s_*_%s_*.fastq.gz" % (ldetail["name"], read)))
634899
def _merge_fastqc(samples):
    
    fastqc_list = collections.defaultdict(list)
    seen = set()
    for data in samples:
        name = dd.get_sample_name(data)
        if name in seen:
            continue
        seen.add(name)
        fns = glob.glob(os.path.join(dd.get_work_dir(data), "qc", dd.get_sample_name(data), "fastqc") + "/*")
        for fn in fns:
            if fn.endswith("tsv"):
                metric = os.path.basename(fn)
                fastqc_list[metric].append([name, fn])

    for metric in fastqc_list:
        dt_by_sample = []
        for fn in fastqc_list[metric]:
            dt = pd.read_csv(fn[1], sep="\t")
            dt['sample'] = fn[0]
            dt_by_sample.append(dt)
        dt = utils.rbind(dt_by_sample)
        dt.to_csv(metric, sep="\t", index=False, mode ='w')
    return samples