Calculates the read length of the fastq files. Short reads will not be able to be assembled properly with the
        default parameters used for spades.
185416
def readlength(self):
        """"""
        logging.info('Estimating read lengths of FASTQ files')
        # Iterate through the samples
        for sample in self.samples:
            sample.run.Date = 'NA'
            sample.run.InvestigatorName = 'NA'
            sample.run.TotalClustersinRun = 'NA'
            sample.run.NumberofClustersPF = 'NA'
            sample.run.PercentOfClusters = 'NA'
            sample.run.SampleProject = 'NA'
            # Only perform this step if the forward and reverse lengths have not been loaded into the metadata
            if not GenObject.isattr(sample.run, 'forwardlength') and not GenObject.isattr(sample.run, 'reverselength'):
                # Initialise the .header attribute for each sample
                sample.header = GenObject()
                sample.commands = GenObject()
                # Set /dev/null
                devnull = open(os.devnull, 'wb')
                # Only process the samples if the file type is a list
                if type(sample.general.fastqfiles) is list:
                    # Set the forward fastq to be the first entry in the list
                    forwardfastq = sorted(sample.general.fastqfiles)[0]
                    # If the files are gzipped, then zcat must be used instead of cat
                    if '.gz' in forwardfastq:
                        command = 'zcat'
                    else:
                        command = 'cat'
                    # Read in the output of the (z)cat of the fastq file piped through head to read only the first 1000
                    # lines. Will make a string of the first 1000 lines in the file
                    forwardreads = subprocess.Popen("{} {} | head -n 1000".format(command, forwardfastq),
                                                    shell=True,
                                                    stdout=subprocess.PIPE,
                                                    stderr=devnull).communicate()[0].rstrip()
                    # Set the length of the reads as follows: the highest value (max) of the length of the sequence. The
                    # sequence was extracted from the rest of the lines in the fastq file. Example of first four lines:
                    """
                    @M02466:126:000000000-AKF4P:1:1101:11875:1838 1:N:0:1
                    TCATAACGCAGTGAAACGCTTTAACAAAAGCGGAGACACGCCACTATTTGTCAATATTTCGTATGATACATTTTTAGAAAATCAAGAAGAGTTGCACGA
                    +
                    AA,B89C,@++B,,,,C,:BFF9,C,,,,,6+++++:,C,8C+BE,EFF9FC,6E,EFGF@F<F@9F9E<FFGGGC8,,,,CC<,,,,,,6CE,C<C,,
                    """
                    # The line with the sequence information occurs every four lines (1, 5, 9, etc). This can be
                    # represented by linenumber % 4 == 1
                    try:
                        # Added due to weird 2to3 conversion issues, was coming
                        forwardreads = forwardreads.decode('utf-8')
                    except UnicodeDecodeError:
                        sample.run.forwardlength = 0
                    # up as a bytes object when we need it as a string.
                    try:
                        forwardlength = max([len(sequence) for iterator, sequence in enumerate(forwardreads.split('\n'))
                                             if iterator % 4 == 1])
                        sample.run.forwardlength = forwardlength
                    except (ValueError, TypeError):
                        sample.run.forwardlength = 0
                    # For paired end analyses, also calculate the length of the reverse reads
                    if len(sample.general.fastqfiles) == 2:
                        reversefastq = sorted(sample.general.fastqfiles)[1]
                        reversereads = subprocess.Popen("{} {} | head -n 1000".format(command, reversefastq),
                                                        shell=True,
                                                        stdout=subprocess.PIPE,
                                                        stderr=devnull).communicate()[0].rstrip()
                        try:
                            reversereads = reversereads.decode('utf-8')
                        except UnicodeDecodeError:
                            sample.run.reverselength = 0
                        try:
                            sample.run.reverselength = max([len(sequence) for iterator, sequence in
                                                            enumerate(reversereads.split('\n')) if iterator % 4 == 1])
                        except (ValueError, TypeError):
                            sample.run.reverselength = 0
                    # Populate metadata of single end reads with 'NA'
                    else:
                        sample.run.reverselength = 0
283533
def _read_index_file(self):
        
        param_file = os.path.join(self.index_dir, self.param_file)
        with open(param_file) as f:
            for line in f.readlines():
                (name, fasta_file, index_file, line_size, total_size) = line.strip().split("\t")
                self.size[name] = int(total_size)
                self.fasta_file[name] = fasta_file
                self.index_file[name] = index_file
                self.line_size[name] = int(line_size)
572478
def fastq_info(path):
    
    numBases = 0
    numReads = 0
    readLengths = Counter()
    GCTot = 0
    with warnings.catch_warnings():
        warnings.simplefilter("ignore")
        handle = gzip.open(path, "rt")
        for record in SeqIO.parse(handle, "fastq"):
            numBases += len(record)
            numReads += 1
            readLengths[len(record)] += 1
            GCTot += sum(record.seq.count(x) for x in ['G', 'C', 'g', 'c', 'S', 's'])
        handle.close()
    GCPer = (GCTot/numBases)
    avgReadLen = (sum(value*count for value,count in readLengths.items())/numReads)
    return {"numBases": numBases,
            "numReads": numReads,
            "numGCBases": GCTot,
            "portionGC": GCPer,
            "avgReadLen": avgReadLen}
1315896
def length
      (@client.redis.multi do
        %w[ locks work scheduled depends ].each do |suffix|
          @client.redis.zcard("ql:q:#{@name}-#{suffix}")
        end
      end).inject(0, :+)
    end
491283
def fastq(self):
        '''
        
        '''
        if self.qual is None:
            self._fastq = None
        else:
            if self._fastq is None:
                self._fastq = '@{}\n{}\n+\n{}'.format(self.id, self.sequence, self.qual)
        return self._fastq
645649
def get_short_reads(vals):
   (args,txome,seed,chunk) = vals
   #fast forward some ammount
   """"""
   txe = TranscriptomeEmitter(txome,TranscriptomeEmitter.Options(seed=seed))
   if args.weights:
      weights = {}
      if args.weights[-3:]=='.gz': inf = gzip.open(args.weights)
      else: inf = open(args.weights)
      for line in inf:
         f = line.rstrip().split("\t")
         weights[f[0]] = float(f[1])
      txs = {}
      for tx in txome.transcripts: txs[tx.name] = tx.length
      for name in weights:
         weights[name] *= txs[name]
      txe.set_weights_by_dict(weights)
   else:
      weights = {}
      txs = {}
      for tx in txome.transcripts: txs[tx.name] = tx.length
      txe.set_weights_by_dict(weights)
   reademit = ReadEmitter(txe)
   shortreads = []
   sp = args.short_read_insert_size
   reademit.cutter.set_custom(sp[0],sp[1],sp[2])
   if args.short_read_error_rate: 
      emfr = ErrorMakerFlatRate(rate=args.short_read_error_rate,rand=reademit.options.rand)
      reademit.add_error_maker(emfr)
   for i in range(0,chunk):
      e = reademit.emit(args.short_read_length)
      shortreads.append(e)
   return shortreads
762248
def genome_lengths(fastas, info):
    
    if info is False:
        info = {}
    for genome in fastas:
        name = genome.rsplit('.', 1)[0].rsplit('/', 1)[-1].rsplit('.contigs')[0]
        if name in info:
            continue
        length = 0
        fragments = 0
        for seq in parse_fasta(genome):
            length += len(seq[1])
            fragments += 1
        info[name] = {'genome size (bp)':length, '# contigs':fragments}
    return info
370111
def filer(self):
        
        # Get a list of all the .fastq files in the sequence path
        fastq = sorted(glob(os.path.join(self.sequencepath, '*fastq*')))
        # Initialise a list to store .fasta formatted files
        fasta = list()
        # Get all the files in the sequence path
        for sequencefile in glob(os.path.join(self.sequencepath, '*')):
            # Split the extension from the file
            extension = os.path.splitext(sequencefile)[1]
            # If the extension is one of the acceptable FASTA extensions, add it to the list of FASTA files
            if extension in self.extensions:
                fasta.append(sequencefile)
        # Use filer from accessoryFunctions to find the base file name of each .fastq file/pair
        fastqnames = filer(fastq)
        # Get all the .fastq files into metadata objects
        for fastqname in sorted(fastqnames):
            # Create the metadata object
            sample = MetadataObject()
            sample.name = os.path.basename(fastqname)
            # Create an attribute for the current analysis
            setattr(sample, self.analysistype, GenObject())
            # Set the destination folder
            sample[self.analysistype].outputdir = os.path.join(self.path, fastqname)
            # Make the destination folder
            make_path(sample[self.analysistype].outputdir)
            # Get the fastq files specific to the fastqname
            specificfastq = glob(os.path.join(self.sequencepath, '{fq}*.fastq*'.format(fq=fastqname)))
            # Set the file type for the downstream analysis
            sample[self.analysistype].filetype = 'fastq'
            # Link the files to the output folder
            for fastq in specificfastq:
                relative_symlink(src_file=fastq,
                                 output_dir=sample[self.analysistype].outputdir)
            # Initialise the general and run categories
            sample.general = GenObject()
            # Populate the .fastqfiles category of :self.metadata
            sample.general.fastqfiles = [fastq for fastq in
                                         glob(os.path.join(sample[self.analysistype].outputdir, '{fq}*.fastq*'
                                                           .format(fq=fastqname))) if 'trimmed' not in fastq]
            # Populate certain attributes in order to be compatible with other local software
            sample.general.bestassemblyfile = sample.general.fastqfiles
            sample.general.trimmedcorrectedfastqfiles = sample.general.fastqfiles
            sample.general.outputdirectory = sample[self.analysistype].outputdir
            sample[self.analysistype].baitedfastq = os.path.join(
                sample[self.analysistype].outputdir,
                '{at}_targetMatches.fastq.gz'.format(at=self.analysistype))
            # Append the metadata to the list of samples
            self.metadata.append(sample)
        # Add any .fasta formatted files to the metadata object
        for fastafile in sorted(fasta):
            sample = MetadataObject()
            sample.name = os.path.splitext(os.path.split(fastafile)[-1])[0]
            setattr(sample, self.analysistype, GenObject())
            # Set the destination folder
            sample[self.analysistype].outputdir = os.path.join(self.path, 'detailed_reports', sample.name)
            # Make the destination folder
            make_path(sample[self.analysistype].outputdir)
            # Set the file type for the downstream analysis
            sample[self.analysistype].filetype = 'fasta'
            # Link the .fasta files to :self.path/:filename
            relative_symlink(src_file=fastafile,
                             output_dir=sample[self.analysistype].outputdir)
            # Initialise the general and run categories
            sample.general = GenObject()
            # Populate the .fastqfiles category of :self.metadata
            sample.general.fastqfiles = [fastq for fastq in glob(os.path.join(sample[self.analysistype].outputdir,
                                                                              '{sn}*{ext}*'
                                                                              .format(
                                                                                  sn=sample.name,
                                                                                  ext=os.path.splitext(fastafile)[1])))]
            sample.general.bestassemblyfile = sample.general.fastqfiles
            sample.general.trimmedcorrectedfastqfiles = sample.general.fastqfiles
            sample.general.outputdirectory = sample[self.analysistype].outputdir
            sample[self.analysistype].baitedfastq = os.path.join(
                sample[self.analysistype].outputdir,
                '{at}_targetMatches.fastq.gz'.format(at=self.analysistype))
            sample[self.analysistype].assemblyfile = fastafile
            # Append the metadata to the list of samples
            self.metadata.append(sample)
314743
def reads(args):
    
    p = OptionParser(reads.__doc__)
    p.add_option("-p", dest="prefix_length", default=4, type="int",
            help="group the reads based on the first N chars [default: %default]")
    opts, args = p.parse_args(args)

    if len(args) != 1:
        sys.exit(p.print_help())

    frgscffile, = args
    prefix_length = opts.prefix_length

    fp = open(frgscffile)
    keyfn = lambda: defaultdict(int)
    counts = defaultdict(keyfn)
    for row in fp:
        f = FrgScfLine(row)
        fi = f.fragmentID[:prefix_length]
        counts[f.scaffoldID][fi] += 1

    for scf, count in sorted(counts.items()):
        print("{0}\t{1}".format(scf,
                ", ".join("{0}:{1}".format(*x) for x in sorted(count.items()))))
619922
def read(self, offset, length, fh):

        

        self.data.seek(offset)
        return self.data.read(length)
204427
def calculate_size(name, sequence):
    
    data_size = 0
    data_size += calculate_size_str(name)
    data_size += LONG_SIZE_IN_BYTES
    return data_size
1219678
def _calculate_osdb_hash(self):
        
        log.debug('_calculate_OSDB_hash() of "{path}" ...'.format(path=self._filepath))
        f = self._filepath.open(mode='rb')

        file_size = self.get_size()

        longlong_format = 'Q'  # unsigned long long little endian
        size_longlong = struct.calcsize(longlong_format)

        block_size = min(file_size, 64 << 10)  # 64kiB
        block_size = block_size & ~(size_longlong - 1)  # lower round on multiple of longlong

        nb_longlong = block_size // size_longlong
        fmt = '<{nbll}{member_format}'.format(
            nbll=nb_longlong,
            member_format=longlong_format)

        hash_int = file_size

        buffer = f.read(block_size)
        list_longlong = struct.unpack(fmt, buffer)
        hash_int += sum(list_longlong)

        f.seek(-block_size, os.SEEK_END)
        buffer = f.read(block_size)
        list_longlong = struct.unpack(fmt, buffer)
        hash_int += sum(list_longlong)

        f.close()
        hash_str = '{:016x}'.format(hash_int)[-16:]
        log.debug('hash("{}")={}'.format(self.get_filepath(), hash_str))
        return hash_str
1341304
def original_query_sequence_length(self):
    
    if not self.is_aligned() or not self.entries.cigar:
      return self.query_sequence_length # take the naive approach
    # we are here with something aligned so take more intelligent cigar apporach
    return sum([x[0] for x in self.cigar_array if re.match('[HMIS=X]',x[1])])
423875
def fastq(args):
    
    from jcvi.formats.fastq import guessoffset

    p = OptionParser(fastq.__doc__)
    p.add_option("--outtie", dest="outtie", default=False, action="store_true",
            help="Are these outie reads? [default: %default]")
    p.set_phred()
    p.set_size()

    opts, args = p.parse_args(args)

    if len(args) < 1:
        sys.exit(p.print_help())

    fastqfiles = [get_abs_path(x) for x in args]
    size = opts.size
    outtie = opts.outtie
    if size > 1000 and (not outtie):
        logging.debug("[warn] long insert size {0} but not outtie".format(size))

    mated = (size != 0)
    libname = op.basename(args[0]).split(".")[0]
    libname = libname.replace("_1_sequence", "")

    frgfile = libname + ".frg"
    mean, sv = get_mean_sv(opts.size)

    cmd = "fastqToCA"
    cmd += " -libraryname {0} ".format(libname)
    fastqs = " ".join("-reads {0}".format(x) for x in fastqfiles)
    if mated:
        assert len(args) in (1, 2), "you need one or two fastq files for mated library"
        fastqs = "-mates {0}".format(",".join(fastqfiles))
        cmd += "-insertsize {0} {1} ".format(mean, sv)
    cmd += fastqs

    offset = int(opts.phred) if opts.phred else guessoffset([fastqfiles[0]])
    illumina = (offset == 64)
    if illumina:
        cmd += " -type illumina"
    if outtie:
        cmd += " -outtie"

    sh(cmd, outfile=frgfile)
370270
def Read(self, length=None):
    """"""

    if self.progress_callback:
      self.progress_callback()

    available_to_read = max(0, (self.size or 0) - self.offset)
    if length is None:
      to_read = available_to_read
    else:
      to_read = min(length, available_to_read)

    with FileHandleManager(self.filename) as fd:
      offset = self.file_offset + self.offset
      pre_padding = offset % self.alignment

      # Due to alignment we read some more data than we need to.
      aligned_offset = offset - pre_padding

      fd.Seek(aligned_offset)

      data = fd.Read(to_read + pre_padding)
      self.offset += len(data) - pre_padding

      return data[pre_padding:]
1013909
def file_length(in_file):
    ''''''

    fid = open(in_file)
    data = fid.readlines()
    fid.close()
    return len(data)
1302171
def _calculate_average_field_lengths(self):
        """"""
        accumulator = defaultdict(int)
        documents_with_field = defaultdict(int)

        for field_ref, length in self.field_lengths.items():
            _field_ref = FieldRef.from_string(field_ref)
            field = _field_ref.field_name

            documents_with_field[field] += 1
            accumulator[field] += length

        for field_name in self._fields:
            accumulator[field_name] /= documents_with_field[field_name]

        self.average_field_length = accumulator