Download bibtex format and parse it from EvoBib
961833
def getEvoBibAsBibtex(*keys, **kw):
    """"""
    res = []
    for key in keys:
        bib = get_url(
            "http://bibliography.lingpy.org/raw.php?key=" + key,
            log=kw.get('log')).text
        try:
            res.append('@' + bib.split('@')[1].split('</pre>')[0])
        except IndexError:  # pragma: no cover
            res.append('@misc{' + key + ',\nNote={missing source}\n\n}')
    return '\n\n'.join(res)
1041065
def read_alignments(self, input_dir, file_format, header_grep=None):
        

        compression = ['', 'gz', 'bz2']

        if file_format == 'fasta':
            extensions = ['fa', 'fas', 'fasta']

        elif file_format == 'phylip':
            extensions = ['phy']

        else:
            extensions = []

        extensions = list('.'.join([x,y]) if y else x for x,y in itertools.product(extensions, compression))

        files = fileIO.glob_by_extensions(input_dir, extensions)
        files.sort(key=SORT_KEY)
        self._input_files = files
        records = []

        if self.show_progress:
            pbar = setup_progressbar("Loading files", len(files), simple_progress=True)
            pbar.start()

        for i, f in enumerate(files):
            if f.endswith('.gz') or f.endswith('.bz2'):
                fd, tmpfile = tempfile.mkstemp()
                with fileIO.freader(f, f.endswith('.gz'), f.endswith('.bz2')) as reader,\
                     fileIO.fwriter(tmpfile) as writer:
                    for line in reader:
                        if ISPY3:
                            line = line.decode()
                        writer.write(line)
                try:
                    record = Alignment(tmpfile, file_format, True)
                except ValueError:
                    record = Alignment(tmpfile, file_format, False)
                finally:
                    os.close(fd)
                    os.unlink(tmpfile)

            else:
                try:
                    record = Alignment(f, file_format, True)
                except RuntimeError:
                    record = Alignment(f, file_format, False)

            if header_grep:
                try:
                    datatype = 'dna' if record.is_dna() else 'protein'

                    record = Alignment([(header_grep(x), y) for (x, y) in record.get_sequences()], datatype)

                except TypeError:
                    raise TypeError("Couldn't apply header_grep to header\n"
                                    "alignment number={}, name={}\n"
                                    "header_grep={}".format(i, fileIO.strip_extensions(f), header_grep))
                except RuntimeError:
                    print('RuntimeError occurred processing alignment number={}, name={}'
                          .format(i, fileIO.strip_extensions(f)))
                    raise

            record.name = (fileIO.strip_extensions(f))
            records.append(record)
            if self.show_progress:
                pbar.update(i)
        if self.show_progress:
            pbar.finish()
        return records
176848
def EBook(self, data=None, subset=None):
        """"""
        return self.factory.get_object(jssobjects.EBook, data, subset)
791187
def pdbechem_parse(download = False, filename = '/kortemmelab/shared/mirror/PDBeChem/chem_comp.xml'):
    ''''''

    xml_data = None
    if download:
        # this URL will go out of date
        try:
            resource = ['ftp.ebi.ac.uk', '/pub/databases/msd/pdbechem/chem_comp.xml']
            xml_data = get_insecure_resource(resource[0], resource[1])
        except FTPException550, e:
            colortext.error("This resource ftp://%s appears to be missing. The link may need to be updated in the script." % "".join(resource))
            raise
        except Exception, e:
            colortext.error("An error occurred downloading ftp://%s:\n%s" % ("".join(resource), str(e)))
            raise
    else:
        xml_data = read_file(filename)

    _dom = parseString(xml_data)

    main_tag = _dom.getElementsByTagName("chemCompList")
    assert(len(main_tag) == 1)
    main_tag = main_tag[0]
    entries = main_tag.getElementsByTagName("chemComp")

    parsed_dict = {}
    properties = ["id", "name", "formula", "systematicName", "stereoSmiles", "nonStereoSmiles", "InChi"]
    for e in entries:
        d = {}
        for p in properties:
            t = e.getElementsByTagName(p)
            assert(len(t) <= 1)
            if p == "id":
                assert(len(t) == 1 and t[0].childNodes)
            if len(t):
                if t[0].childNodes:
                    d[p] = t[0].firstChild.nodeValue
                else:
                    d[p] = None
            else:
                d[p] = None
        parsed_dict[d['id']] = d
    return parsed_dict
793367
def load_data(self, pdbid):
        """"""
        f = urlopen("http://projects.biotec.tu-dresden.de/plip-rest/pdb/%s?format=xml" % pdbid.lower())
        self.doc = etree.parse(f)
43344
def get_resources(genome, ref_file, data):
    
    base_dir = os.path.normpath(os.path.dirname(ref_file))
    resource_file = os.path.join(base_dir, "%s-resources.yaml" % genome.replace("-test", ""))
    if not os.path.exists(resource_file):
        raise IOError("Did not find resource file for %s: %s\n"
                      "To update bcbio_nextgen.py with genome resources for standard builds, run:\n"
                      "bcbio_nextgen.py upgrade -u skip"
                      % (genome, resource_file))
    with open(resource_file) as in_handle:
        resources = yaml.safe_load(in_handle)

    def resource_file_path(x):
        if isinstance(x, six.string_types) and os.path.exists(os.path.join(base_dir, x)):
            return os.path.normpath(os.path.join(base_dir, x))
        return x
    cleaned = utils.dictapply(resources, resource_file_path)
    return ensure_annotations(cleaned, data)
522351
def check_dependencies():
    
    available = []
    try:
        shell('ebook-convert')
        available.append('calibre')
    except OSError:
        pass
    try:
        shell('pandoc --help')
        available.append('pandoc')
    except OSError:
        pass
    if not available:
        sys.exit(error('No generator found, you cannot use md2ebook.'))
    check_dependency_epubcheck()
    return available
289257
def pdb_to_structure(filename):
    

    try:
        from Bio.PDB import PDB
    except ImportError:
        print("I can't import Biopython which is needed to handle PDB files.")
        raise
    p = PDB.PDBParser()
    structure = p.get_structure('S', filename)
    for _ in structure.get_chains():
        atoms = [np.array(atom.get_coord()) for atom in structure.get_atoms()]
    return atoms
498000
def source_start(base='', book_id='book'):
    
    repo_htm_path = "{book_id}-h/{book_id}-h.htm".format(book_id=book_id)
    possible_paths = ["book.asciidoc",
                      repo_htm_path,
                      "{}-0.txt".format(book_id),
                      "{}-8.txt".format(book_id),
                      "{}.txt".format(book_id),
                      "{}-pdf.pdf".format(book_id),
                     ]

    # return the first match

    for path in possible_paths:
        fullpath = os.path.join(base, path)
        if os.path.exists(fullpath):
            return path

    return None
138120
def download_log(currentfile=None):
    """"""

    if currentfile == None:
        return
    
    # add ".err.log" if file doesn't end with it!
    if not currentfile.endswith(".err.log"):
        currentfile=currentfile + ".err.log"
    
    list = get_base_ev3dev_cmd() + ['download','--force']
    list.append(currentfile)

    env = os.environ.copy()
    env["PYTHONUSERBASE"] = THONNY_USER_BASE
    proc = subprocess.Popen(list, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
                            universal_newlines=True, env=env)
    dlg = MySubprocessDialog(get_workbench(), proc, "Downloading log of program from EV3", autoclose=True)
    dlg.wait_window()
    if dlg.returncode == 0:
        from pathlib import Path
        home = str(Path.home())
        open_file(currentfile,home,True)
161104
def read_10xgenomics(cls, tarball_fpath: str, prefix: str,
                         use_ensembl_ids: bool = False):
        """"""

        _LOGGER.info('Reading file: %s', tarball_fpath)

        with tarfile.open(tarball_fpath, mode='r:gz') as tf:
            ti = tf.getmember('%smatrix.mtx' % prefix)
            with tf.extractfile(ti) as fh:
                mtx = scipy.io.mmread(fh)

            ti = tf.getmember('%sgenes.tsv' % prefix)
            with tf.extractfile(ti) as fh:
                wrapper = io.TextIOWrapper(fh, encoding='ascii')
                i = 1
                if use_ensembl_ids:
                    i = 0
                gene_names = \
                        [row[i] for row in csv.reader(wrapper, delimiter='\t')]

            ti = tf.getmember('%sbarcodes.tsv' % prefix)
            with tf.extractfile(ti) as fh:
                wrapper = io.TextIOWrapper(fh, encoding='ascii')
                barcodes = \
                        [row[0] for row in csv.reader(wrapper, delimiter='\t')]


            assert mtx.shape[0] == len(gene_names)
            assert mtx.shape[1] == len(barcodes)
        
        _LOGGER.info('Matrix dimensions: %s', str(mtx.shape))
        X = mtx.todense()
        matrix = cls(X=X, genes=gene_names, cells=barcodes)
        
        return matrix
821188
def parse(binary, **params):
    """"""
    binary = io.BytesIO(binary)
    collection = list()
    with tarfile.TarFile(fileobj=binary, mode='r') as tar:
        for tar_info in tar.getmembers():
            content_type, encoding = mimetypes.guess_type(tar_info.name)
            content = tar.extractfile(tar_info)
            content = content_encodings.get(encoding).decode(content)
            content = content_types.get(content_type).parse(content, **params)
            collection.apppend((tar_info.name, content))
    return collection
341093
def from_command_line():
    
    # Parse options
    parser = argparse.ArgumentParser(
        description='Convert Complete Genomics var files to gVCF format.')
    parser.add_argument(
        '-d', '--refseqdir', metavar='REFSEQDIR', required=True,
        dest='refseqdir',
        help='Directory twobit reference genomes files are stored.')
    parser.add_argument(
        '-i', '--input', metavar='INPUTVARFILE',
        dest='cgivarfile',
        help='Path to Complete Genomics var file to convert. If omitted, data '
        ' also be piped in as standard input.')
    parser.add_argument(
        '-o', '--output', metavar='OUTPUTVCFFILE',
        dest='vcfoutfile',
        help='Path to where to save output VCF file.')
    parser.add_argument(
        '-D', '--download', action='store_true', dest='downloadrefseq',
        help='Download the 2bit file from UCSC to REFSEQDIR, if needed.')
    parser.add_argument(
        '-v', '--var-only', action='store_true', dest='varonly',
        help='Only report variant lines (i.e. VCF, but not gVCF)')
    args = parser.parse_args()

    # Get local twobit file from its directory. Download and store if needed.
    twobit_path, twobit_name = get_reference_genome_file(
        args.refseqdir, build='b37')
    # Handle input
    if sys.stdin.isatty():  # false if data is piped in
        var_input = args.cgivarfile
    else:
        var_input = sys.stdin
    # Handle output
    if args.vcfoutfile:
        convert_to_file(var_input,
                        args.vcfoutfile,
                        twobit_path,
                        twobit_name,
                        args.varonly)
    else:
        for line in convert(
                cgi_input=var_input,
                twobit_ref=twobit_path,
                twobit_name=twobit_name,
                var_only=args.varonly):
            print(line)
1186553
def get_read_format(cls, source, args, kwargs):
    
    ctx = None
    if isinstance(source, FILE_LIKE):
        fileobj = source
        filepath = source.name if hasattr(source, 'name') else None
    else:
        filepath = source
        try:
            ctx = get_readable_fileobj(filepath, encoding='binary')
            fileobj = ctx.__enter__()  # pylint: disable=no-member
        except IOError:
            raise
        except Exception:  # pylint: disable=broad-except
            fileobj = None
    try:
        return get_format('read', cls, filepath, fileobj, args, kwargs)
    finally:
        if ctx is not None:
            ctx.__exit__(*sys.exc_info())
173232
def parse_apidoc(
    file_or_branch,
    from_github=False,
    save_github_version=True
) -> List['ApiEndpoint']:
    """"""
    apis = []  # type: List[ApiEndpoint]
    regex = r'(?P<group>\([^)]*\)){0,1} *(?P<type_>{[^}]*}){0,1} *'
    regex += r'(?P<field>[^ ]*) *(?P<description>.*)$'
    param_regex = re.compile(r'^@apiParam {1,}' + regex)
    success_regex = re.compile(r'^@apiSuccess {1,}' + regex)
    if from_github:
        text = download_api(file_or_branch)
        if save_github_version:
            save_apidoc(text)
    else:
        with open(file_or_branch) as f:
            text = f.read()
    for line in text.split('\n'):
        line = line.replace('\n', '')
        if line.startswith('@api '):
            if apis:
                if not apis[-1].retcode:
                    apis[-1].retcode = 200
            split_line = line.split(' ')
            assert len(split_line) >= 3
            method = split_line[1]
            uri = split_line[2]
            assert method[0] == '{'
            assert method[-1] == '}'
            method = method[1:-1]
            if not uri.startswith(API_URI_BASE):
                warnings.warn(_("Wrong api url: {}").format(uri))  # noqa: Q000
            title = ' '.join(split_line[3:])
            apis.append(ApiEndpoint(method, uri, title))
        elif line.startswith('@apiParam '):
            res = next(param_regex.finditer(line)).groupdict()
            apis[-1].add_param(**res)
        elif line.startswith('@apiSuccess '):
            res = next(success_regex.finditer(line)).groupdict()
            apis[-1].add_success(**res)
    if apis:
        if not apis[-1].retcode:
            apis[-1].retcode = 200
    return apis
849023
def bibcode_from_url(cls, url):
        
        try:
            code = url.split('/abs/')
            code = code[1].strip()
            return code
        except:
            return None
1336550
def parse(self, limit=None):
        

        files_to_download, ftp = self._get_file_list(
            self.files['anat_entity']['path'],
            self.files['anat_entity']['pattern'])
        for dlname in files_to_download:
            localfile = '/'.join((self.rawdir, dlname))
            with gzip.open(localfile, 'rt', encoding='ISO-8859-1') as fh:
                LOG.info("Processing %s", localfile)
                self._parse_gene_anatomy(fh, limit)
        return