Download gaf, gpad, or gpi from http://current.geneontology.org/annotations/
338917
def dnld_annotation(assc_file, prt=sys.stdout):
    """"""
    if not os.path.isfile(assc_file):
        # assc_http = "http://geneontology.org/gene-associations/"
        assc_http = "http://current.geneontology.org/annotations/"
        _, assc_base = os.path.split(assc_file)
        src = os.path.join(assc_http, "{ASSC}.gz".format(ASSC=assc_base))
        dnld_file(src, assc_file, prt, loading_bar=None)
690825
def get(datasets_identifiers, identifier_type='hid', history_id=None):
    
    history_id = history_id or os.environ['HISTORY_ID']
    # The object version of bioblend is to slow in retrieving all datasets from a history
    # fallback to the non-object path
    gi = get_galaxy_connection(history_id=history_id, obj=False)
    for dataset_identifier in datasets_identifiers:
        file_path = '/import/%s' % dataset_identifier
        log.debug('Downloading gx=%s history=%s dataset=%s', gi, history_id, dataset_identifier)
        # Cache the file requests. E.g. in the example of someone doing something
        # silly like a get() for a Galaxy file in a for-loop, wouldn't want to
        # re-download every time and add that overhead.
        if not os.path.exists(file_path):
            hc = HistoryClient(gi)
            dc = DatasetClient(gi)
            history = hc.show_history(history_id, contents=True)
            datasets = {ds[identifier_type]: ds['id'] for ds in history}
            if identifier_type == 'hid':
                dataset_identifier = int(dataset_identifier)
            dc.download_dataset(datasets[dataset_identifier], file_path=file_path, use_default_filename=False)
        else:
            log.debug('Cached, not re-downloading')

    return file_path
80608
def cli(self, prt=sys.stdout):
        """"""
        kws = self.objdoc.get_docargs(prt=None)
        if os.path.exists(kws['i']):
            obj = NCBIgeneFileReader(kws['i'])
            nts = obj.get_nts()
            if nts:
                geneid2nt = self._get_geneid2nt(nts)
                self._wrpy_ncbi_gene_nts(kws['o'], geneid2nt, prt)
        else:
            raise RuntimeError("\n{DOC}\n**ERROR: NO FILE FOUND: {NCBI}".format(
                NCBI=kws['i'], DOC=__doc__))
665948
def _chk_gene2go(assoc):
    """"""
    if not assoc:
        raise RuntimeError("NO ITEMS FOUND IN ASSOCIATIONS {A}".format(A=assoc))
    for key in assoc:
        if isinstance(key, str) and key[:3] == "GO:":
            raise Exception("ASSOCIATIONS EXPECTED TO BE gene2go, NOT go2gene: {EX}".format(
                EX=assoc.items()[:2]))
        return
683988
def gffatributes2ids(s):
    
    Name,gene_id,transcript_id,protein_id,exon_id=np.nan,np.nan,np.nan,np.nan,np.nan
    if '=' in s:
        d=dict([i.split('=') for i in s.split(';')])
        if 'Parent' in d:
            d[d['Parent'].split(':')[0]+'_id']=d['Parent'].split(':')[1]
        Name,gene_id,transcript_id,protein_id,exon_id=np.nan,np.nan,np.nan,np.nan,np.nan
        if 'Name' in d:    
            Name=d['Name']
        if 'gene_id' in d:    
            gene_id=d['gene_id']
        if 'transcript_id' in d:    
            transcript_id=d['transcript_id']
        if 'protein_id' in d:    
            protein_id=d['protein_id']
        if 'exon_id' in d:    
            exon_id=d['exon_id']
    return Name,gene_id,transcript_id,protein_id,exon_id
1313512
def _setup_genome_annotations(g, args, ann_groups):
    
    available_anns = g.get("annotations", []) + g.pop("annotations_available", [])
    anns = []
    for orig_target in args.datatarget:
        if orig_target in ann_groups:
            targets = ann_groups[orig_target]
        else:
            targets = [orig_target]
        for target in targets:
            if target in available_anns:
                anns.append(target)
    g["annotations"] = anns
    if "variation" not in args.datatarget and "validation" in g:
        del g["validation"]
    return g
751401
def _run_genotype_gvcfs_genomicsdb(genomics_db, region, out_file, data):
    
    if not utils.file_exists(out_file):
        with file_transaction(data, out_file) as tx_out_file:
            broad_runner = broad.runner_from_config(data["config"])
            params = ["-T", "GenotypeGVCFs",
                      "--variant", "gendb://%s" % genomics_db,
                      "-R", dd.get_ref_file(data),
                      "--output", tx_out_file,
                      "-L", bamprep.region_to_gatk(region)]
            params += ["-ploidy", str(ploidy.get_ploidy([data], region))]
            # Avoid slow genotyping runtimes with improved quality score calculation in GATK4
            # https://gatkforums.broadinstitute.org/gatk/discussion/11471/performance-troubleshooting-tips-for-genotypegvcfs/p1
            params += ["--use-new-qual-calculator"]
            resources = config_utils.get_resources("gatk", data["config"])
            params += [str(x) for x in resources.get("options", [])]
            cores = dd.get_cores(data)
            memscale = {"magnitude": 0.9 * cores, "direction": "increase"} if cores > 1 else None
            broad_runner.run_gatk(params, memscale=memscale)
    return vcfutils.bgzip_and_index(out_file, data["config"])
749780
def dannots2dalignbed2dannotsagg(cfg):
    
    datatmpd=cfg['datatmpd']
    
    daannotp=f'{datatmpd}/08_dannot.tsv'
    cfg['daannotp']=daannotp
    dannotsaggp=cfg['dannotsaggp']
    logging.info(basename(daannotp))
    if ((not exists(daannotp)) and (not exists(dannotsaggp))) or cfg['force']:
        gff_renamed_cols=[c+' annotation' if c in set(bed_colns).intersection(gff_colns) else c for c in gff_colns]
        dannots=pd.read_csv(cfg['annotationsbedp'],sep='\t',
                   names=bed_colns+gff_renamed_cols,
                           low_memory=False)
        dannots=del_Unnamed(dannots)

        dannots=dannots.set_index('id')
        dannots['annotations count']=1
        # separate ids from attribute columns
        dannots=lambda2cols(dannots,lambdaf=gffatributes2ids,
                    in_coln='attributes',
                to_colns=['gene name','gene id','transcript id','protein id','exon id'])
        dannots=dannots.drop(['attributes']+[c for c in gff_renamed_cols if 'annotation' in c],axis=1)
        logging.debug('or this step takes more time?')
        to_table(dannots,daannotp)
#         to_table_pqt(dannots,daannotp)
    else:
#         dannots=read_table_pqt(daannotp)
        dannots=read_table(daannotp)
        dannots=del_Unnamed(dannots)
        
    logging.info(basename(dannotsaggp))
    if not exists(dannotsaggp) or cfg['force']:
        if not 'dannots' in locals():
#             dannots=read_table_pqt(daannotp)
            dannots=pd.read_table(daannotp,low_memory=False)
        dannots=del_Unnamed(dannots)
        dannots=dannots.reset_index()
        logging.debug('aggregating the annotations')
        from rohan.dandage.io_sets import unique 
        cols2aggf={'annotations count':np.sum,
                  'type': unique,
                  'gene name': unique,
                  'gene id': unique,
                  'transcript id': unique,
                  'protein id': unique,
                  'exon id': unique}
        dannotsagg=dannots.groupby('id').agg(cols2aggf)
        dannotsagg['annotations count']=dannotsagg['annotations count']-1
        dannotsagg.loc[dannotsagg['annotations count']==0,'region']='intergenic'
        dannotsagg.loc[dannotsagg['annotations count']!=0,'region']='genic'
        logging.debug('end of the slowest step')            
        del dannots    
        dannotsagg=dannotsagg.reset_index()
#         to_table_pqt(dannotsagg,dannotsaggp)
        dannotsagg.to_csv(dannotsaggp,sep='\t')        
    return cfg
423223
def _wrpy_ncbi_gene_nts(fout_py, geneid2nt, log):
        """"""
        num_genes = len(geneid2nt)
        with open(fout_py, 'w') as ofstrm:
            docstr = "Data downloaded from NCBI Gene converted into Python namedtuples."
            ofstrm.write('"""{PYDOC}"""\n\n'.format(PYDOC=docstr))
            ofstrm.write("from collections import namedtuple\n\n")
            ofstrm.write('WRITTEN = "{DATE}"'.format(
                DATE=re.sub('-', '_', str(datetime.date.today()))))
            ofstrm.write(' # {N} items\n\n'.format(N=num_genes))
            ntd = next(iter(geneid2nt.values())) # Access one dictionary value in Python 2
            ofstrm.write("#pylint: disable=line-too-long,too-many-lines,invalid-name\n")
            ofstrm.write("{NtName} = namedtuple('{NtName}', '{FLDS}')\n\n".format(
                NtName=type(ntd).__name__, FLDS=' '.join(ntd._fields)))
            ofstrm.write("GENEID2NT = {{ # {N:,} items\n".format(N=num_genes))
            for geneid, ntd in sorted(geneid2nt.items(), key=lambda t: t[0]):
                ofstrm.write("    {GeneID} : {NT},\n".format(GeneID=geneid, NT=ntd))
            ofstrm.write("}\n")
            log.write("  {N:9} geneids WROTE: {PY}\n".format(N=num_genes, PY=fout_py))
1116495
def read_ncbi_gene2go(fin_gene2go, taxids=None, **kws):
    """"""
    obj = Gene2GoReader(fin_gene2go, taxids=taxids)
    # By default, return id2gos. User can cause go2geneids to be returned by:
    #   >>> read_ncbi_gene2go(..., go2geneids=True
    if 'taxid2asscs' not in kws:
        if len(obj.taxid2asscs) == 1:
            taxid = next(iter(obj.taxid2asscs))
            kws_ncbi = {k:v for k, v in kws.items() if k in AnnoOptions.keys_exp}
            kws_ncbi['taxid'] = taxid
            return obj.get_id2gos(**kws_ncbi)
    # Optional detailed associations split by taxid and having both ID2GOs & GO2IDs
    # e.g., taxid2asscs = defaultdict(lambda: defaultdict(lambda: defaultdict(set))
    t2asscs_ret = obj.get_taxid2asscs(taxids, **kws)
    t2asscs_usr = kws.get('taxid2asscs', defaultdict(lambda: defaultdict(lambda: defaultdict(set))))
    if 'taxid2asscs' in kws:
        obj.fill_taxid2asscs(t2asscs_usr, t2asscs_ret)
    return obj.get_id2gos_all(t2asscs_ret)
1008613
def _gids_to_genes(gids, ssm_locs, cnv_ssms, data):
    
    locs = collections.defaultdict(set)
    for gid in gids:
        cur_locs = []
        try:
            cur_locs.append(ssm_locs[gid])
        except KeyError:
            for ssm_loc in cnv_ssms.get(gid, []):
                cur_locs.append(ssm_locs[ssm_loc])
        for chrom, pos in cur_locs:
            locs[chrom].add(pos)
    genes = set([])
    with tx_tmpdir(data) as tmpdir:
        chrom_prefix = "chr" if next(ref.file_contigs(dd.get_ref_file(data))).name.startswith("chr") else ""
        loc_file = os.path.join(tmpdir, "battenberg_find_genes.bed")
        with open(loc_file, "w") as out_handle:
            for chrom in sorted(locs.keys()):
                for loc in sorted(list(locs[chrom])):
                    out_handle.write("%s%s\t%s\t%s\n" % (chrom_prefix, chrom, loc - 1, loc))
        ann_file = annotate.add_genes(loc_file, data, max_distance=10000)
        for r in pybedtools.BedTool(ann_file):
            for gene in r.name.split(","):
                if gene != ".":
                    genes.add(gene)
    return sorted(list(genes))
378567
def get_rRNA(gtf):
    
    rRNA_biotypes = ["rRNA", "Mt_rRNA", "tRNA", "MT_tRNA"]
    features = set()
    with open_gzipsafe(gtf) as in_handle:
        for line in in_handle:
            if not "gene_id" in line or not "transcript_id" in line:
                continue
            if any(x in line for x in rRNA_biotypes):
                geneid = line.split("gene_id")[1].split(" ")[1]
                geneid = _strip_non_alphanumeric(geneid)
                geneid = _strip_feature_version(geneid)
                txid = line.split("transcript_id")[1].split(" ")[1]
                txid = _strip_non_alphanumeric(txid)
                txid = _strip_feature_version(txid)
                features.add((geneid, txid))
    return features
1140196
def genestatus(args):
    
    p = OptionParser(genestatus.__doc__)
    opts, args = p.parse_args(args)

    if len(args) != 1:
        sys.exit(not p.print_help())

    idsfile, = args
    data = get_tags(idsfile)
    key = lambda x: x[0].split(".")[0]
    for gene, cc in groupby(data, key=key):
        cc = list(cc)
        tags = [x[-1] for x in cc]
        if "complete" in tags:
            tag = "complete"
        elif "partial" in tags:
            tag = "partial"
        else:
            tag = "pseudogene"
        print("\t".join((gene, tag)))
1320039
def _get_annotation(c, loci):
    """"""
    data_ann_temp = {}
    data_ann = []
    counts = Counter()
    for lid in c.loci2seq:
        # original Py 2.7 code
        #for dbi in loci[lid].db_ann.keys():
        #    data_ann_temp[dbi] = {dbi: map(lambda (x): loci[lid].db_ann[dbi].ann[x].name, loci[lid].db_ann[dbi].ann.keys())}
        # suggestion by 2to3
        for dbi in list(loci[lid].db_ann.keys()):
            data_ann_temp[dbi] = {dbi: [loci[lid].db_ann[dbi].ann[x].name for x in list(loci[lid].db_ann[dbi].ann.keys())]}
            logger.debug("_json_: data_ann_temp %s %s" % (dbi, data_ann_temp[dbi]))
            counts[dbi] += 1
        # original Py 2.7 code
        #data_ann = data_ann + map(lambda (x): data_ann_temp[x], data_ann_temp.keys())
        # suggestion by 2to3
        data_ann = data_ann + [data_ann_temp[x] for x in list(data_ann_temp.keys())]
        logger.debug("_json_: data_ann %s" % data_ann)
    counts = {k: v for k, v in counts.iteritems()}
    total_loci = sum([counts[db] for db in counts])
    valid_ann = [k for k, v in counts.iteritems() if up_threshold(v, total_loci, 0.7)]
    return data_ann, valid_ann
999600
def _clean_up_gene_id(geneid, sp, curie_map):
        
        # special case for MGI
        geneid = re.sub(r'MGI:MGI:', 'MGI:', geneid)

        # rewrite Ensembl --> ENSEMBL
        geneid = re.sub(r'Ensembl', 'ENSEMBL', geneid)

        # rewrite Gene:CELE --> WormBase
        # these are old-school cosmid identifier
        geneid = re.sub(r'Gene:CELE', 'WormBase:', geneid)
        if sp == 'CAEEL':
            if re.match(r'(Gene|ENSEMBLGenome):\w+\.\d+', geneid):
                geneid = re.sub(
                    r'(?:Gene|ENSEMBLGenome):(\w+\.\d+)',
                    r'WormBase:\1', geneid)

        if sp == 'DROME':
            if re.match(r'(ENSEMBLGenome):\w+\.\d+', geneid):
                geneid = re.sub(
                    r'(?:ENSEMBLGenome):(\w+\.\d+)', r'FlyBase:\1', geneid)

        # rewrite GeneID --> NCBIGene
        geneid = re.sub(r'GeneID', 'NCBIGene', geneid)

        # rewrite Gene:Dmel --> FlyBase
        geneid = re.sub(r'Gene:Dmel_', 'FlyBase:', geneid)
        # rewrite Gene:CG --> FlyBase:CG
        geneid = re.sub(r'Gene:CG', 'FlyBase:CG', geneid)

        # rewrite ENSEMBLGenome:FBgn --> FlyBase:FBgn
        geneid = re.sub(r'ENSEMBLGenome:FBgn', 'FlyBase:FBgn', geneid)

        # rewrite Gene:<ensembl ids> --> ENSEMBL:<id>
        geneid = re.sub(r'Gene:ENS', 'ENSEMBL:ENS', geneid)

        # rewrite Gene:<Xenbase ids> --> Xenbase:<id>
        geneid = re.sub(r'Gene:Xenbase:', 'Xenbase:', geneid)

        # TODO this would be much better done as
        # if foo not in selfcurie_map:
        # if re.match(r'(Gene|ENSEMBLGenome):', geneid) or \
        #        re.match(r'Gene_ORFName', geneid) or \
        #        re.match(r'Gene_Name', geneid):
        #    # LOG.warning(
        #    #"Found an identifier I don't know how to fix (species %s): %s",
        #    #   sp, geneid)

        pfxlcl = re.split(r':', geneid)
        pfx = pfxlcl[0]
        if pfx is None or pfx not in curie_map:
            # LOG.warning( "No curie prefix for (species %s): %s", sp, geneid)
            geneid = None
        return geneid
316987
function (xrefs, server){
    this._is_a = 'amigo2.linker';

    // TODO:BUG: With the new dispatcher, relative URLs no longer work.

    this._xrefs = xrefs;

    // Easy app base.
    this.app_base = server.app_base;
    // Internal term matcher.
    this.term_regexp = null;
    var internal_regexp_str = server.term_regexp;
    if( internal_regexp_str ){
	this.term_regexp = new RegExp(internal_regexp_str);
    }

    // Categories for different special cases (internal links).
    this.ont_category = {
	'term': true,
	'ontology_class': true,
	'annotation_class': true,
	'annotation_class_closure': true,
	'annotation_class_list': true,
	// Noctua model stuff.
	'function_class': true,
	'function_class_closure': true,
	'process_class': true,
	'process_class_closure': true,
	'location_list': true,
	'location_list_closure': true
    };
    this.bio_category = {
        'gp': true,
	'gene_product': true,
	'bioentity': true
    };
    this.ref_category = {
	'reference': true
    };
    this.model_category = {
        'model': true
    };
    this.search_category = { // not including the trivial medial_search below
        'search': true,
	'live_search': true
    };
    this.search_modifier = {
	// Possibly "dynamic".
	'gene_product': '/bioentity',
	'bioentity': '/bioentity',
	'ontology': '/ontology',
	'reference': '/reference',
	'annotation': '/annotation',
	'model': '/model',
	'family': '/family',
	'lego_unit': '/lego_unit',
	'general': '/general'
    };
    this.other_interlinks = {
	'medial_search': '/amigo/medial_search',
	'landing': '/amigo/landing',
	'tools': '/amigo/software_list',
	'schema_details': '/amigo/schema_details',
	'load_details': '/amigo/load_details',
	'browse': '/amigo/browse',
	'goose': '/goose',
	'grebe': '/grebe',
	'gannet': '/gannet',
	'repl': '/repl'	
    };
}
694644
function _prep_download_url(){

    var engine = new impl_engine(golr_response);
    //engine.method('GET');
    //engine.use_jsonp(true);
    var gman = new golr_manager(gserv, gconf, engine, 'async');
	
    gman.set_personality('annotation');
    gman.add_query_filter('document_category', 'annotation', ['*']);
	
    // gman.set_results_count(download_count);
    var field_list = [ // from GAF
	'source', // c1
	'bioentity_internal_id', // c2; not bioentity
	'bioentity_label', // c3
	'qualifier', // c4
	'annotation_class', // c5
	'reference', // c6
	'evidence_type', // c7
	'evidence_with', // c8
	'aspect', // c9
	'bioentity_name', // c10
	'synonym', // c11
	'type', // c12
	'taxon', // c13
	'date', // c14
	'assigned_by', // c15
	'annotation_extension_class', // c16
	'bioentity_isoform' // c17
    ];
    // gman.set('fl', field_list.join(','));
    // //manager.set('rows', arg_hash['rows']);
    // gman.set('csv.encapsulator', '');
    // gman.set('csv.separator', '%09');
    // gman.set('csv.header', 'false');
    // gman.set('csv.mv.separator', '|');
    // gman.set('start', _random_number(5));

    //gman.set('start', _random_number(5));
    gman.set_extra('&start=' + _random_number(4));
    //console.log(gman.get_download_url(field_list, { 'rows': lines }));
    return gman.get_download_url(field_list, { 'rows': lines });
}