Runs through entire dataset. 
        Returns TSS error, total correct, total count, pcorrect (a dict of layer data)
491492
def sweep(self):
        
        self.preSweep()
        if self.loadOrder == []:
            raise NetworkError('No loadOrder for the inputs. Make sure inputs are properly set.', self.loadOrder)
        if len(self.targets) != 0 and len(self.targets) != len(self.inputs):
            raise NetworkError("Number of inputs does not equal number of targets (inputs=%d, targets=%d)" % (len(self.targets), len(self.inputs)))
        if self.verbosity >= 1: print("Epoch #", self.epoch, "Cycle...")
        if not self.orderedInputs:
            self.randomizeOrder()
        tssError = 0.0; totalCorrect = 0; totalCount = 0; totalPCorrect = {}
        cnt = 0
        if self.saveResults:
            self.results = [(0,0,0) for x in self.loadOrder]
        for i in self.loadOrder:
            if self.verbosity >= 1 or self.interactive:
                print("-----------------------------------Pattern #", self.loadOrder[i] + 1)
            datum = self.getData(i) # creates a dictionary of input/targets from self.inputs, self.targets
            if cnt < len(self.loadOrder) - 1:
                self.currentSweepCount = cnt
            else:
                self.currentSweepCount = None
            self._sweeping = 1
            (error, correct, total, pcorrect) = self.step( **datum )
            self._sweeping = 0
            if self.saveResults:
                self.results[i] = (error, correct, total, pcorrect)
            tssError += error
            totalCorrect += correct
            totalCount += total
            sumMerge(totalPCorrect, pcorrect)
            if self.sweepReportRate and (cnt + 1) % self.sweepReportRate == 0:
                print("   Step # %6d | TSS Error: %.4f | Correct: %.4f" % \
                      (cnt + 1, tssError, totalCorrect * 1.0 / totalCount))
            if self.crossValidationSampleRate and self.epoch % self.crossValidationSampleRate == 0:
                self.saveNetworkForCrossValidation(self.crossValidationSampleFile)
            cnt += 1
        if self.learning and self.batch:
            self.change_weights() # batch mode, otherwise change weights in step
        self.postSweep()
        return (tssError, totalCorrect, totalCount, totalPCorrect)
400451
def perform_pilon(self):
        
        for sample in self.metadata:
            try:
                if sample[self.analysistype].num_contigs > 500 or sample.confindr.contam_status == 'Contaminated':
                    sample.general.polish = False
                else:
                    sample.general.polish = True
            except AttributeError:
                sample.general.polish = True
1013060
def datasets(self):
        
        r = self.session.query(distinct(models.Entry.dataset)).all()
        return [x[0] for x in r]
513323
def cluster(args):
    

    args = _check_args(args)
    read_stats_file = op.join(args.dir_out, "read_stats.tsv")
    if file_exists(read_stats_file):
        os.remove(read_stats_file)

    bam_file, seq_obj = _clean_alignment(args)

    logger.info("Parsing matrix file")
    seqL, y, l = parse_ma_file(seq_obj, args.ffile)
    # y, l = _total_counts(seqL.keys(), seqL)
    logger.info("counts after: %s" % sum(y.values()))
    logger.info("# sequences after: %s" % l)
    dt = pd.DataFrame({'sample': y.keys(), 'counts': y.values()})
    dt['step'] = 'aligned'
    dt.to_csv(read_stats_file, sep="\t", index=False, header=False, mode='a')

    if len(seqL.keys()) < 10:
        logger.error("It seems you have low coverage. Please check your fastq files have enough sequences.")
        raise ValueError("So few sequences.")

    logger.info("Cleaning bam file")
    y, l = _total_counts(seqL.keys(), seqL)
    logger.info("counts after: %s" % sum(y.values()))
    logger.info("# sequences after: %s" % l)
    dt = pd.DataFrame({'sample': y.keys(), 'counts': y.values()})
    dt['step'] = 'cleaned'
    dt.to_csv(read_stats_file, sep="\t", index=False, header=False, mode='a')

    clusL = _create_clusters(seqL, bam_file, args)
    y, l = _total_counts(clusL.seq.keys(), clusL.seq, aligned=True)
    logger.info("counts after: %s" % sum(y.values()))
    logger.info("# sequences after: %s" % l)
    dt = pd.DataFrame({'sample': y.keys(), 'counts': y.values()})
    dt['step'] = 'clusters'
    dt.to_csv(read_stats_file, sep="\t", index=False, header=False, mode='a')

    logger.info("Solving multi-mapping events in the network of clusters")
    clusLred = _cleaning(clusL, args.dir_out)
    y, l = _total_counts(clusLred.clus, seqL)
    logger.info("counts after: %s" % sum(y.values()))
    logger.info("# sequences after: %s" % l)
    dt = pd.DataFrame({'sample': y.keys(), 'counts': y.values()})
    dt['step'] = 'meta-cluster'
    dt.to_csv(read_stats_file, sep="\t", index=False, header=False, mode='a')
    logger.info("Clusters up to %s" % (len(clusLred.clus.keys())))

    if args.show:
        logger.info("Creating sequences alignment to precursor")
        clusLred = show_seq(clusLred, args.index)

    clusLred = peak_calling(clusLred)

    clusLred = _annotate(args, clusLred)

    logger.info("Creating json and count matrix")
    json_file = _create_json(clusLred, args)

    logger.info("Output file in: %s" % args.dir_out)

    if args.db:
        name = args.db + ".db"
        logger.info("Create database: database/" + name)
        data = load_data(json_file)
        out_dir = op.join(args.dir_out, "database")
        make_database(data, name, out_dir)
    logger.info("Finished")
1201147
def _parse_qualimap_globals(table):
    
    out = {}
    want = {"Mapped reads": _parse_num_pct,
            "Duplication rate": lambda k, v: {k: v}}
    for row in table.find_all("tr"):
        col, val = [x.text for x in row.find_all("td")]
        if col in want:
            out.update(want[col](col, val))
    return out
809259
def dump_t_coords(dataset_dir, data_dir, dataset, root=None, compress=True):
    """"""
    if root is None:
        root = {}
    tcoords = dataset.GetPointData().GetTCoords()
    if tcoords:
        dumped_array = dump_data_array(dataset_dir, data_dir, tcoords, {}, compress)
        root['pointData']['activeTCoords'] = len(root['pointData']['arrays'])
        root['pointData']['arrays'].append({'data': dumped_array})
1093510
def _match_error_to_data_set(x, ex):
    
    # Simplest case, ex is None or a number
    if not _fun.is_iterable(ex):
        
        # Just make a matched list of Nones
        if ex is None: ex = [ex]*len(x)
        
        # Make arrays of numbers
        if _fun.is_a_number(ex): 
            value = ex # temporary storage
            ex    = []
            for n in range(len(x)): 
                ex.append([value]*len(x[n]))
    
    # Otherwise, ex is iterable
    
    # Default behavior: If the elements are all numbers and the length matches
    # that of the first x-array, assume this is meant to match all the x
    # data sets
    if _fun.elements_are_numbers(ex) and len(ex) == len(x[0]): ex = [ex]*len(x)

    # The user may specify a list of some iterable and some not. Assume
    # in this case that at least the lists are the same length    
    for n in range(len(x)):
        # do nothing to the None's
        # Inflate single numbers to match
        if _fun.is_a_number(ex[n]): ex[n] = [ex[n]]*len(x[n])
        
    return ex
1356386
def select(cls, dataset, selection_mask=None, **selection):
        
        selection = {k: slice(*sel) if isinstance(sel, tuple) else sel
                     for k, sel in selection.items()}
        coords = tuple(selection[kd.name] if kd.name in selection else slice(None)
                       for kd in dataset.kdims)
        if not any([isinstance(el, slice) for el in coords]):
            return dataset.data[dataset.sheet2matrixidx(*coords)]

        # Apply slices
        xidx, yidx = coords
        l, b, r, t = dataset.bounds.lbrt()
        if isinstance(xidx, slice):
            l = l if xidx.start is None else max(l, xidx.start)
            r = r if xidx.stop is None else min(r, xidx.stop)
        if isinstance(yidx, slice):
            b = b if yidx.start is None else max(b, yidx.start)
            t = t if yidx.stop is None else min(t, yidx.stop)
        bounds = BoundingBox(points=((l, b), (r, t)))
        slc = Slice(bounds, dataset)
        return slc.submatrix(dataset.data)
1159441
def _titancna_summary(call, data):
    
    out = {}
    for svtype, coords in get_coords(data):
        cur_calls = {k: collections.defaultdict(int) for k in coords.keys()}
        with open(call["subclones"]) as in_handle:
            header = in_handle.readline().strip().split()
            for line in in_handle:
                val = dict(zip(header, line.strip().split()))
                start = int(val["Start_Position.bp."])
                end = int(val["End_Position.bp."])
                for region, cur_coords in coords.items():
                    if val["Chromosome"] == cur_coords[0] and are_overlapping((start, end), cur_coords[1:]):
                        cur_calls[region][_check_copy_number_changes(svtype, _to_cn(val["Copy_Number"]),
                                                                     _to_cn(val["MinorCN"]), data)] += 1
        out[svtype] = {r: _merge_cn_calls(c, svtype) for r, c in cur_calls.items()}

    with open(call["hetsummary"]) as in_handle:
        vals = dict(zip(in_handle.readline().strip().split("\t"), in_handle.readline().strip().split("\t")))
    out["purity"] = vals["purity"]
    out["ploidy"] = vals["ploidy"]
    return out
492311
def run_qc(_, data, out_dir):
    
    if cwlutils.is_cwl_run(data):
        qc_data = run_peddy([data], out_dir)
        if tz.get_in(["summary", "qc", "peddy"], qc_data):
            return tz.get_in(["summary", "qc", "peddy"], qc_data)
1241673
def get_bad_rows_and_cols(df, validation_names, type_col_names,
                          value_col_names, verbose=False):
    
    df["num"] = list(range(len(df)))
    problems = df[validation_names.union(["num"])]
    all_problems = problems.dropna(how='all', axis=0, subset=validation_names)
    value_problems = problems.dropna(how='all', axis=0, subset=type_col_names.union(value_col_names))
    all_problems = all_problems.dropna(how='all', axis=1)
    value_problems = value_problems.dropna(how='all', axis=1)
    if not len(problems):
        return None, None, None
    #
    bad_cols = all_problems.columns
    prefixes = ["value_pass_", "type_pass_"]
    missing_prefix = "presence_pass_"
    problem_cols = []
    missing_cols = []
    long_missing_cols = []
    problem_rows = []
    for col in bad_cols:
        pre, stripped_col = extract_col_name(col)
        for prefix in prefixes:
            if col.startswith(prefix):
                problem_cols.append(stripped_col)
                continue
        if col.startswith(missing_prefix):
            missing_cols.append(stripped_col)
            long_missing_cols.append(col)
    if len(value_problems):
        bad_rows = list(zip(list(value_problems["num"]), list(value_problems.index)))
    else:
        bad_rows = []
    if verbose:
        if bad_rows:
            formatted_rows = ["row: {}, name: {}".format(row[0], row[1]) for row in bad_rows]
            if len(bad_rows) > 5:
                print("-W- these rows have problems:\n", "\n".join(formatted_rows[:5]), " ...")
                print("(for full error output see error file)")
            else:
                print("-W- these rows have problems:", "\n".join(formatted_rows))
        if problem_cols:
            print("-W- these columns contain bad values:", ", ".join(set(problem_cols)))
        if missing_cols:
            print("-W- these required columns are missing:", ", ".join(missing_cols))
    return bad_rows, problem_cols, missing_cols
779373
def verify_tops(self, tops):
        '''
        
        '''
        errors = []
        if not isinstance(tops, dict):
            errors.append('Top data was not formed as a dict')
            # No further checks will work, bail out
            return errors
        for saltenv, matches in six.iteritems(tops):
            if saltenv == 'include':
                continue
            if not isinstance(saltenv, six.string_types):
                errors.append(
                    'Environment {0} in top file is not formed as a '
                    'string'.format(saltenv)
                )
            if saltenv == '':
                errors.append('Empty saltenv statement in top file')
            if not isinstance(matches, dict):
                errors.append(
                    'The top file matches for saltenv {0} are not '
                    'formatted as a dict'.format(saltenv)
                )
            for slsmods in six.itervalues(matches):
                if not isinstance(slsmods, list):
                    errors.append('Malformed topfile (state declarations not '
                                  'formed as a list)')
                    continue
                for slsmod in slsmods:
                    if isinstance(slsmod, dict):
                        # This value is a match option
                        for val in six.itervalues(slsmod):
                            if not val:
                                errors.append(
                                    'Improperly formatted top file matcher '
                                    'in saltenv {0}: {1} file'.format(
                                        slsmod,
                                        val
                                    )
                                )
                    elif isinstance(slsmod, six.string_types):
                        # This is a sls module
                        if not slsmod:
                            errors.append(
                                'Environment {0} contains an empty sls '
                                'index'.format(saltenv)
                            )

        return errors
894647
def tfrecord_iterator_for_problem(problem, data_dir,
                                  dataset_split=tf.estimator.ModeKeys.TRAIN):
  """"""
  filenames = tf.gfile.Glob(problem.filepattern(data_dir, mode=dataset_split))
  example_spec = problem.example_reading_spec()[0]
  return tfrecord_iterator(filenames, example_spec=example_spec)
1367232
def show_results(self, ds_type=DatasetType.Valid, rows:int=5, max_len:int=20):
        from IPython.display import display, HTML
        ""
        ds = self.dl(ds_type).dataset
        x,y = self.data.one_batch(ds_type, detach=False, denorm=False)
        preds = self.pred_batch(batch=(x,y))
        y = y.view(*x.size())
        z = preds.view(*x.size(),-1).argmax(dim=2)
        xs = [ds.x.reconstruct(grab_idx(x, i)) for i in range(rows)]
        ys = [ds.x.reconstruct(grab_idx(y, i)) for i in range(rows)]
        zs = [ds.x.reconstruct(grab_idx(z, i)) for i in range(rows)]
        items,names = [],['text', 'target', 'pred']
        for i, (x,y,z) in enumerate(zip(xs,ys,zs)):
            txt_x = ' '.join(x.text.split(' ')[:max_len])
            txt_y = ' '.join(y.text.split(' ')[max_len-1:2*max_len-1])
            txt_z = ' '.join(z.text.split(' ')[max_len-1:2*max_len-1])
            items.append([txt_x, txt_y, txt_z])
        items = np.array(items)
        df = pd.DataFrame({n:items[:,i] for i,n in enumerate(names)}, columns=names)
        with pd.option_context('display.max_colwidth', -1):
            display(HTML(df.to_html(index=False)))
330194
def dataset_spike_counts(dset, threshold, fs):
    """"""
    if len(dset.shape) == 3:
        results = np.zeros(dset.shape[0])
        for itrace in range(dset.shape[0]):
            results[itrace] = count_spikes(dset[itrace], threshold, fs)
        return results
    elif len(dset.shape == 2):
        return count_spikes(dset, threshold, fs)
    else:
        raise Exception("Improper data dimensions")
1323543
def run_all2(protgroup, memornot, subsequences, base_outdir,
            protgroup_dict, protein_feathers_dir, date, errfile, impute_counts=True,
            cutoff_num_proteins=0, core_only_genes=None,
            length_filter_pid=.8, remove_correlated_feats=True,
            force_rerun_counts=False, force_rerun_percentages=False, force_rerun_pca=False):
    """"""
    import ssbio.utils

    # Need to set multiprocessing limit for scipy/numpy stuff if parallelizing anything
    import os
    os.environ['OMP_NUM_THREADS'] = '1'

    # First, filter down the protein group to the membrane/nonmembrane definition
    prots_filtered_feathers = get_protein_feather_paths(protgroup=protgroup, memornot=memornot,
                                                        protgroup_dict=protgroup_dict,
                                                        protein_feathers_dir=protein_feathers_dir,
                                                        core_only_genes=core_only_genes)
    num_proteins = len(prots_filtered_feathers)
    if num_proteins <= cutoff_num_proteins:
        return

    # Make output directories
    protscale = 'proteome_unscaled'
    outdir_d0 = ssbio.utils.make_dir(op.join(base_outdir, protscale))
    outdir_d1 = ssbio.utils.make_dir(op.join(outdir_d0, '-'.join(memornot)))
    outdir_final = ssbio.utils.make_dir(op.join(outdir_d1, '-'.join(protgroup)))

    if impute_counts:
        big_strain_counts_df = get_proteome_counts_impute_missing(prots_filtered_feathers=prots_filtered_feathers,
                                                                  outpath=op.join(outdir_final,
                                                                                  '{}-subsequence_proteome_IMP.fthr'.format(
                                                                                      date)),
                                                                  length_filter_pid=length_filter_pid,
                                                                  force_rerun=force_rerun_counts)

        big_strain_percents_df = get_proteome_percentages(counts_df=big_strain_counts_df,
                                                          outpath=op.join(outdir_final,
                                                                          '{}-subsequence_proteome_perc_IMP.fthr'.format(
                                                                                  date)),
                                                          force_rerun=force_rerun_percentages)
        pca_pickle = op.join(outdir_final, '{}-subsequence_pca.pckl'.format(date))
    # Divide by totals to get percentages in a new dataframe
    else:
        try:
            big_strain_percents_df = get_proteome_correct_percentages(prots_filtered_feathers=prots_filtered_feathers,
                                                                      outpath=op.join(outdir_final,
                                                                                      '{}-subsequence_proteome_perc_AVG.fthr'.format(
                                                                                              date)),
                                                                      length_filter_pid=length_filter_pid,
                                                                      force_rerun=force_rerun_percentages)
            pca_pickle = op.join(outdir_final, '{}-subsequence_pca_AVG.pckl'.format(date))
        except:
            with open(errfile, "a") as myfile:
                myfile.write('PERCENTAGES ERR: ' + '-'.join(memornot) + '\t' + '-'.join(protgroup) + "\n")
            return



    if ssbio.utils.force_rerun(flag=force_rerun_pca, outfile=pca_pickle):

        # Then, get filters for rows of the loaded feathers for interested subsequences
        keep_subsequences = get_interested_subsequences(subsequences=subsequences)

        # Some numbers: number of features
        num_feats = len(big_strain_percents_df)

        # Make an unwieldy title
        big_title = 'LOC={0}; PROTGROUP={1};\n' \
                    'NUMPROTS={2}; NUMFEATS={3}'.format('-'.join(memornot),
                                                                        '-'.join(protgroup),
                                                                        num_proteins,
                                                                        num_feats)

        # Run PCA and make plots
        runner = PCAMultiROS(features_df=big_strain_percents_df, observations_df=pd.DataFrame(), plot_title=big_title)
        try:
            runner.clean_data(keep_features=keep_subsequences, remove_correlated_feats=remove_correlated_feats)
        except:
            with open(errfile, "a") as myfile:
                myfile.write(
                    'CLEAN ERR: ' + '-'.join(memornot) + '\t' + '-'.join(protgroup) + "\n")
            return
        # try:
        runner.run_pca()
        # except:
        #     with open(errfile, "a") as myfile:
        #         myfile.write(
        #             'PCA ERR: ' + '-'.join(memornot) + '\t' + '-'.join(protgroup) + "\n")
        #     return
        with open(pca_pickle, 'wb') as f:
            pickle.dump(runner, f)
    else:
        with open(pca_pickle, 'rb') as f:
            runner = pickle.load(f)
735263
def parse_grid_facets(facets):
    
    valid_seqs = ["('var1', '.')", "('var1', 'var2')",
                  "('.', 'var1')", "((var1, var2), (var3, var4))"]
    error_msg_s = ("Valid sequences for specifying 'facets' look like"
                   " {}".format(valid_seqs))

    valid_forms = ['var1 ~ .', 'var1 ~ var2', '. ~ var1',
                   'var1 + var2 ~ var3 + var4',
                   '. ~ func(var1) + func(var2)',
                   '. ~ func(var1+var3) + func(var2)'
                   ] + valid_seqs
    error_msg_f = ("Valid formula for 'facet_grid' look like"
                   " {}".format(valid_forms))

    if isinstance(facets, (tuple, list)):
        if len(facets) != 2:
            raise PlotnineError(error_msg_s)

        rows, cols = facets

        if isinstance(rows, str):
            rows = [] if rows == '.' else [rows]
        if isinstance(cols, str):
            cols = [] if cols == '.' else [cols]

        return rows, cols

    if not isinstance(facets, str):
        raise PlotnineError(error_msg_f)

    # Example of allowed formulae
    # 'c ~ a + b'
    # '. ~ func(a) + func(b)'
    # 'func(c) ~ func(a+1) + func(b+2)'
    try:
        lhs, rhs = facets.split('~')
    except ValueError:
        raise PlotnineError(error_msg_s)
    else:
        lhs = lhs.strip()
        rhs = rhs.strip()

    lhs = ensure_var_or_dot(lhs)
    rhs = ensure_var_or_dot(rhs)

    lsplitter = ' + ' if ' + ' in lhs else '+'
    rsplitter = ' + ' if ' + ' in rhs else '+'

    if lhs == '.':
        rows = []
    else:
        rows = [var.strip() for var in lhs.split(lsplitter)]

    if rhs == '.':
        cols = []
    else:
        cols = [var.strip() for var in rhs.split(rsplitter)]

    return rows, cols