a helper function for multiprocessing with DistReport.
433396
def _mp_run_check(tasks, results, options):
    

    try:
        for index, change in iter(tasks.get, None):
            # this is the part that takes up all of our time and
            # produces side-effects like writing out files for all of
            # the report formats.
            change.check()

            # rather than serializing the completed change (which
            # could be rather large now that it's been realized), we
            # send back only what we want, which is the squashed
            # overview, and throw away the used bits.
            squashed = squash(change, options=options)
            change.clear()

            results.put((index, squashed))

    except KeyboardInterrupt:
        # prevent a billion lines of backtrace from hitting the user
        # in the face
        return
166690
def _MultiStream(cls, fds):
    """"""
    for fd in fds:
      fd.Seek(0)
      while True:
        chunk = fd.Read(cls.MULTI_STREAM_CHUNK_SIZE)
        if not chunk:
          break
        yield fd, chunk, None
627934
def ParallelCalculate(cls,syslst,properties=['energy'],system_changes=all_changes):
        '''
        
        '''
        print('Launching:',end=' ')
        sys.stdout.flush()
        for n,s in enumerate(syslst):
            try :
                s.calc.block=False
                s.calc.calculate(atoms=s,properties=properties,system_changes=system_changes)
            except CalcNotReadyError:
                s.calc.block=True
            print(n+1, end=' ')
            sys.stdout.flush()
        print()
        print('     Done:', end=' ')
        sys.stdout.flush()
        for n,s in enumerate(syslst):
            s.calc.read_results()
            print( n+1, end=' ')
            sys.stdout.flush()
        print()
        return syslst
41494
def _sort_lambda(sortedby='cpu_percent',
                 sortedby_secondary='memory_percent'):
    """"""
    ret = None
    if sortedby == 'io_counters':
        ret = _sort_io_counters
    elif sortedby == 'cpu_times':
        ret = _sort_cpu_times
    return ret
183573
def sg_parallel(func):
    
    """
    @wraps(func)
    def wrapper(**kwargs):
        r"""Manages arguments of `tf.sg_opt`.

        Args:
          kwargs: keyword arguments. The wrapped function will be provided with gpu_index argument.
        """
        # parse option
        opt = tf.sg_opt(kwargs)

        # loop for all available GPUs
        res = []
        for i in range(sg_gpus()):
            # specify device
            with tf.device('/gpu:%d' % i):
                # give new scope only to operation
                with tf.name_scope('gpu_%d' % i):
                    # save reuse flag
                    with sg_context(reuse=(True if i > 0 else False)):
                        # call function
                        res.append(func(opt * tf.sg_opt(gpu_index=i)))

        return res

    return wrapper
606470
def main():
    '''
    
    '''
    args = get_args()
    try:
        utils.make_output_dir(args.outdir)
        utils.init_logs(args, tool="NanoComp")
        args.format = nanoplotter.check_valid_format(args.format)
        settings = vars(args)
        settings["path"] = path.join(args.outdir, args.prefix)
        sources = [args.fastq, args.bam, args.summary, args.fasta]
        sourcename = ["fastq", "bam", "summary", "fasta"]
        if args.split_runs:
            split_dict = validate_split_runs_file(args.split_runs)
        datadf = nanoget.get_input(
            source=[n for n, s in zip(sourcename, sources) if s][0],
            files=[f for f in sources if f][0],
            threads=args.threads,
            readtype=args.readtype,
            names=args.names,
            barcoded=args.barcoded,
            combine="track")
        datadf, settings = filter_and_transform_data(datadf, vars(args))
        if args.raw:
            datadf.to_csv("NanoComp-data.tsv.gz", sep="\t", index=False, compression="gzip")
        if args.store:
            pickle.dump(
                obj=datadf,
                file=open(settings["path"] + "NanoComp-data.pickle", 'wb'))
        if args.split_runs:
            change_identifiers(datadf, split_dict)
        if args.barcoded:
            datadf["dataset"] = datadf["barcode"]
        identifiers = list(datadf["dataset"].unique())
        write_stats(
            datadfs=[datadf[datadf["dataset"] == i] for i in identifiers],
            outputfile=settings["path"] + "NanoStats.txt",
            names=identifiers)
        if args.plot != 'false':
            plots = make_plots(datadf, settings)
            make_report(plots, path.join(args.outdir, args.prefix))
        logging.info("Succesfully processed all input.")
    except Exception as e:
        logging.error(e, exc_info=True)
        raise
477503
def parallelize_func(iterable, func, chunksz=1, n_jobs=16, *args, **kwargs):
    
    chunker = func
    chunks = more_itertools.chunked(iterable, chunksz)
    chunks_results = Parallel(n_jobs=n_jobs, verbose=50)(
        delayed(chunker)(chunk, *args, **kwargs) for chunk in chunks)
    results = more_itertools.flatten(chunks_results)
    return list(results)
434538
def run_in_subprocess(func, *args, **kwargs):
    """"""
    from multiprocessing import Process
    thread = Process(target=func, args=args, kwargs=kwargs)
    thread.daemon = True
    thread.start()
    return thread
752546
def main():
    """"""
    desc = 'Generate files to benchmark'
    parser = argparse.ArgumentParser(description=desc)
    parser.add_argument(
        '--src',
        dest='src_dir',
        default='src',
        help='The directory containing the templates'
    )
    parser.add_argument(
        '--out',
        dest='out_dir',
        default='generated',
        help='The output directory'
    )
    parser.add_argument(
        '--seed',
        dest='seed',
        default='13',
        help='The random seed (to ensure consistent regeneration)'
    )

    args = parser.parse_args()

    random.seed(int(args.seed))

    mkdir_p(args.out_dir)

    for template in templates_in(args.src_dir):
        modes = template.modes()

        n_range = template.range()
        for n_value in n_range:
            base = template.instantiate(n_value)
            for mode in modes:
                write_file(
                    os.path.join(
                        args.out_dir,
                        out_filename(template, n_value, mode)
                    ),
                    mode.convert_from(base)
                )
        write_file(
            os.path.join(args.out_dir, '{0}.json'.format(template.name)),
            json.dumps({
                'files': {
                    n: {
                        m.identifier: out_filename(template, n, m)
                        for m in modes
                    } for n in n_range
                },
                'name': template.name,
                'x_axis_label': template.property('x_axis_label'),
                'desc': template.property('desc'),
                'modes': {m.identifier: m.description() for m in modes}
            })
        )
54223
def _multi_lpop_pipeline(self, pipe, queue, number):
        ''' '''
        pipe.lrange(queue, 0, number - 1)
        pipe.ltrim(queue, number, -1)
833702
def _bnd(self, xloc, dist, cache):
        """"""
        return numpy.log(evaluation.evaluate_bound(
            dist, numpy.e**xloc, cache=cache))
1188030
def multiprocess_permutation(bed_dict, mut_df, opts, indel_df=None):
    
    chroms = sorted(bed_dict.keys(), key=lambda x: len(bed_dict[x]), reverse=True)
    multiprocess_flag = opts['processes']>0
    if multiprocess_flag:
        num_processes = opts['processes']
    else:
        num_processes = 1
    #file_handle = open(opts['output'], 'w')
    file_handle = opts['handle']
    mywriter = csv.writer(file_handle, delimiter='\t', lineterminator='\n')
    if opts['maf'] and opts['num_iterations']:
        header = ['Gene', 'strand', 'Chromosome', 'Start_Position',
                  'End_Position', 'Reference_Allele', 'Tumor_Allele',
                  'Context', 'DNA_Change', 'Protein_Change', 'Variant_Classification']
    elif opts['maf']:
        header = ['Gene', 'strand', 'Chromosome', 'Start_Position',
                  'End_Position', 'Reference_Allele', 'Tumor_Allele',
                  'DNA_Change', 'Protein_Change', 'Variant_Classification',
                  'Tumor_Sample', 'Tumor_Type']
    else:
        header = ['Gene', 'ID', 'gene length', 'non-silent snv', 'silent snv', 'nonsense', 'lost stop',
                  'splice site', 'lost start', 'missense', 'recurrent missense',
                  'normalized missense position entropy',]
        # add column header for scores, is user provided one
        if opts['score_dir']:
            header += ['Total Missense MGAEntropy', 'Total Missense VEST Score']
        # add indel columns
        header += ['frameshift indel', 'inframe indel', 'normalized mutation entropy']
    mywriter.writerow(header)
    num_iterations = opts['num_iterations']

    # simulate indel counts
    if opts['summary'] and num_iterations:
        fs_cts, inframe_cts, gene_names = indel.simulate_indel_counts(indel_df,
                                                                      bed_dict,
                                                                      num_iterations,
                                                                      opts['seed'])
        name2ix = {gene_names[z]: z for z in range(len(gene_names))}
    # just count observed indels
    elif opts['summary']:
        # get gene names
        gene_names = [mybed.gene_name
                      for chrom in bed_dict
                      for mybed in bed_dict[chrom]]
        name2ix = {gene_names[z]: z for z in range(len(gene_names))}

        # initiate count vectors
        inframe_cts = np.zeros((1, len(gene_names)))
        fs_cts = np.zeros((1, len(gene_names)))

        # populate observed counts
        indel_cts_dict = indel_df['Gene'].value_counts().to_dict()
        fs_cts_dict = indel_df[indel.is_frameshift_annotation(indel_df)]['Gene'].value_counts().to_dict()
        for mygene in indel_cts_dict:
            if mygene in name2ix:
                # gene should be found in BED file annotation
                ix = name2ix[mygene]
                fs_cts[0, ix] = 0 if mygene not in fs_cts_dict else fs_cts_dict[mygene]
                inframe_cts[0, ix] = indel_cts_dict[mygene] - fs_cts[0, ix]

    # simulate snvs
    obs_result = []
    for i in range(0, len(chroms), num_processes):
        if multiprocess_flag:
            pool = Pool(processes=num_processes)
            tmp_num_proc = len(chroms) - i if i + num_processes > len(chroms) else num_processes
            info_repeat = ((bed_dict[chroms[tmp_ix]], mut_df, opts)
                            for tmp_ix in range(i, i+tmp_num_proc))
            process_results = pool.imap(singleprocess_permutation, info_repeat)
            process_results.next = utils.keyboard_exit_wrapper(process_results.next)
            try:
                # iterate through each chromosome result
                for chrom_result in process_results:
                    # add columns for indels
                    if opts['summary']:
                        tmp_chrom_result = []
                        for gname, grp in it.groupby(chrom_result, lambda x: x[0]):
                            for l, row in enumerate(grp):
                                gene_ix = name2ix[gname]
                                fs_count = fs_cts[l, gene_ix]
                                inframe_count = inframe_cts[l, gene_ix]
                                missense_pos_ct = list(row.pop(-1).values())  # missense codon counts
                                silent_pos_ct = [1 for l in range(row[4])]
                                inactivating_ct = sum(row[5:9]) + fs_count
                                tmp_count_list = missense_pos_ct + silent_pos_ct + [inactivating_ct, inframe_count]
                                norm_ent = math.normalized_mutation_entropy(tmp_count_list)
                                tmp_chrom_result.append(row+[fs_count, inframe_count, norm_ent])
                        chrom_result = tmp_chrom_result

                    # write output to file
                    mywriter.writerows(chrom_result)
            except KeyboardInterrupt:
                pool.close()
                pool.join()
                logger.info('Exited by user. ctrl-c')
                sys.exit(0)
            pool.close()
            pool.join()
        else:
            # perform simulation
            info = (bed_dict[chroms[i]], mut_df, opts)
            chrom_results = singleprocess_permutation(info)

            # add indel columns
            if opts['summary']:
                tmp_chrom_result = []
                for gname, grp in it.groupby(chrom_results, lambda x: x[0]):
                    for l, row in enumerate(grp):
                        gene_ix = name2ix[gname]
                        fs_count = fs_cts[l, gene_ix]
                        inframe_count = inframe_cts[l, gene_ix]
                        missense_pos_ct = list(row.pop(-1).values())  # missense codon counts
                        silent_pos_ct = [1 for l in range(row[4])]
                        inactivating_ct = sum(row[5:9]) + fs_count
                        tmp_count_list = missense_pos_ct + silent_pos_ct + [inactivating_ct, inframe_count]
                        norm_ent = math.normalized_mutation_entropy(tmp_count_list)
                        tmp_chrom_result.append(row+[fs_count, inframe_count, norm_ent])
                chrom_results = tmp_chrom_result

            # write to file
            mywriter.writerows(chrom_results)
1141350
def main():
    '''
    
    '''
    args = get_args()
    try:
        utils.make_output_dir(args.outdir)
        utils.init_logs(args)
        args.format = nanoplotter.check_valid_format(args.format)
        settings = vars(args)
        settings["path"] = path.join(args.outdir, args.prefix)
        sources = {
            "fastq": args.fastq,
            "bam": args.bam,
            "cram": args.cram,
            "fastq_rich": args.fastq_rich,
            "fastq_minimal": args.fastq_minimal,
            "summary": args.summary,
            "fasta": args.fasta,
            "ubam": args.ubam,
        }

        if args.pickle:
            datadf = pickle.load(open(args.pickle, 'rb'))
        else:
            datadf = get_input(
                source=[n for n, s in sources.items() if s][0],
                files=[f for f in sources.values() if f][0],
                threads=args.threads,
                readtype=args.readtype,
                combine="simple",
                barcoded=args.barcoded)
        if args.store:
            pickle.dump(
                obj=datadf,
                file=open(settings["path"] + "NanoPlot-data.pickle", 'wb'))
        if args.raw:
            datadf.to_csv(settings["path"] + "NanoPlot-data.tsv.gz",
                          sep="\t",
                          index=False,
                          compression="gzip")

        settings["statsfile"] = [make_stats(datadf, settings, suffix="")]
        datadf, settings = filter_and_transform_data(datadf, settings)
        if settings["filtered"]:  # Bool set when filter was applied in filter_and_transform_data()
            settings["statsfile"].append(make_stats(datadf, settings, suffix="_post_filtering"))

        if args.barcoded:
            barcodes = list(datadf["barcode"].unique())
            plots = []
            for barc in barcodes:
                logging.info("Processing {}".format(barc))
                settings["path"] = path.join(args.outdir, args.prefix + barc + "_")
                dfbarc = datadf[datadf["barcode"] == barc]
                if len(dfbarc) > 5:
                    settings["title"] = barc
                    plots.extend(
                        make_plots(dfbarc, settings)
                    )
                else:
                    sys.stderr.write("Found barcode {} less than 5x, ignoring...\n".format(barc))
                    logging.info("Found barcode {} less than 5 times, ignoring".format(barc))
        else:
            plots = make_plots(datadf, settings)
        make_report(plots, settings)
        logging.info("Finished!")
    except Exception as e:
        logging.error(e, exc_info=True)
        print("\n\n\nIf you read this then NanoPlot has crashed :-(")
        print("Please try updating NanoPlot and see if that helps...\n")
        print("If not, please report this issue at https://github.com/wdecoster/NanoPlot/issues")
        print("If you could include the log file that would be really helpful.")
        print("Thanks!\n\n\n")
        raise
505279
def iparse(self,meter=None,num_processes=1,arbiter='Line',line_lim=None):
		""""""
		from Meter import Meter,genDefault,parse_ent,parse_ent_mp
		import multiprocessing as mp
		meter=self.get_meter(meter)

		# set internal attributes
		self.__parses[meter.id]=[]
		self.__bestparses[meter.id]=[]
		self.__boundParses[meter.id]=[]
		self.__parsed_ents[meter.id]=[]

		lines = self.lines()
		lines=lines[:line_lim]
		numlines = len(lines)

		init=self
		ents=self.ents(arbiter)
		smax=self.config.get('line_maxsylls',100)
		smin=self.config.get('line_minsylls',0)
		#print '>> # of lines to parse:',len(ents)
		ents = [e for e in ents if e.num_syll >= smin and e.num_syll<=smax]
		#print '>> # of lines to parse after applying min/max line settings:',len(ents)

		self.scansion_prepare(meter=meter,conscious=True)

		numents=len(ents)

		#pool=mp.Pool(1)
		toprint=self.config['print_to_screen']
		objects = [(ent,meter,init,False) for ent in ents]

		if num_processes>1:
			print '!! MULTIPROCESSING PARSING IS NOT WORKING YET !!'
			pool = mp.Pool(num_processes)
			jobs = [pool.apply_async(parse_ent_mp,(x,)) for x in objects]
			for j in jobs:
				print j.get()
				yield j.get()
		else:
			now=time.time()
			clock_snum=0
			#for ei,ent in enumerate(pool.imap(parse_ent_mp,objects)):
			for ei,objectx in enumerate(objects):
				clock_snum+=ent.num_syll
				if ei and not ei%100:
					nownow=time.time()
					if self.config['print_to_screen']:
						print '>> parsing line #',ei,'of',numents,'lines','[',round(float(clock_snum/(nownow-now)),2),'syllables/second',']'
					now=nownow
					clock_snum=0

				yield parse_ent_mp(objectx)

		if self.config['print_to_screen']:
			print '>> parsing complete in:',time.time()-now,'seconds'
490586
def run_matrix_in_parallel(self, process_data):
        """"""
        worker_data = [{'matrix': entry, 'pipeline': process_data.pipeline,
                        'model': process_data.model, 'options': process_data.options,
                        'hooks': process_data.hooks} for entry in self.matrix
                       if Matrix.can_process_matrix(entry, process_data.options.matrix_tags)]
        output = []
        success = True
        with closing(multiprocessing.Pool(multiprocessing.cpu_count())) as pool:
            for result in pool.map(matrix_worker, worker_data):
                output += result['output']
                if not result['success']:
                    success = False
        return {'success': success, 'output': output}
1207070
def __det_hsml_old(self, pos, nb):
        
        manager = Manager()
        out_hsml  = manager.Queue()
        size  = multiprocessing.cpu_count()	

        if self.__verbose:
            print('Building a KDTree...')
        tree = self.__make_kdtree(pos)

        index  = np.arange(np.shape(pos)[1])
		#I split the job among the number of available processors
        pos   = np.array_split(pos, size, axis=1)	
        
        procs = []

        #We distribute the tasks among different processes
        if self.__verbose:
                print('Searching the ', nb,
                      'closer neighbors to each particle...')
        for rank in range(size):
            task = multiprocessing.Process(target=self.__nbsearch, 
                                           args=(pos[rank], nb, tree, 
                                                 out_hsml,rank))
            procs.append(task) 
            task.start()
            
            #Wait until all processes finish
        for p in procs:
            p.join()

            index = []
            hsml  = []
        for i in range(size):
            a, b = out_hsml.get()
            index.append(a)
            hsml.append(b)
    #	    if a == 0: print(b[0])

            #I have to order the data before return it
        k = np.argsort(index)
        hsml1 = np.array([])
        for i in k:
            hsml1 = np.append(hsml1,hsml[i])
        if self.__verbose:
            print('Done...')
        return hsml1
163144
def _run_parallel_multiprocess(self):
        
        _log.debug("run.parallel.multiprocess.start")
        processes = []
        ProcRunner.instance = self
        for i in range(self._ncores):
            self._status.running(i)
            proc = multiprocessing.Process(target=ProcRunner.run, args=(i,))
            proc.start()
            processes.append(proc)
        for i in range(self._ncores):
            processes[i].join()
            code = processes[i].exitcode
            self._status.success(i) if 0 == code else self._status.fail(i)
        _log.debug("run.parallel.multiprocess.end states={}".format(self._status))