:param file: file path
        :param limit: limit (int, optional) limit the number of rows processed
        :return: dict
330209
def _parse_gene_coordinates(file):
        
        id_map = {}
        col = ['gene_curie', 'start', 'end', 'strand', 'build']
        if os.path.exists(os.path.join(os.path.dirname(__file__), file)):
            with open(os.path.join(os.path.dirname(__file__), file)) as tsvfile:
                reader = csv.reader(tsvfile, delimiter="\t")
                for row in reader:
                    id_map[row[col.index('gene_curie')]] = {
                        'start': row[col.index('start')],
                        'end': row[col.index('end')],
                        'strand': row[col.index('strand')],
                        'build': row[col.index('build')]
                    }
        return id_map
200351
def parse(self, limit=None):
        
        if limit is not None:
            LOG.info("Only parsing first %d rows", limit)

        phenotype_file = '/'.join(
            (self.rawdir, self.files['patient_phenotypes']['file']))
        variant_file = '/'.join((self.rawdir, self.files['patient_variants']['file']))

        phenotype_file_handler = open(phenotype_file, 'r')
        variant_file_handler = open(variant_file, 'r')

        self._parse_patient_phenotypes(phenotype_file_handler, limit)
        self._parse_patient_variants(variant_file_handler)

        return
170875
def process_all_common_disease_files(self, limit=None):
        
        LOG.info("Iterating over all common disease files")
        common_file_count = 0
        total_processed = ""  # stopgap gill we fix common-disease files
        unpadded_doids = ""   # stopgap gill we fix common-disease files
        for ingest in self.files:
            if ingest[:5] == 'common':
                common_file_count += 1
                raw = self.files[ingest]['file']
                total_processed += self.process_common_disease_file(
                    raw, unpadded_doids, limit)
            if not self.test_mode and limit is not None and total_processed > limit:
                break
        LOG.info("Finished iterating over all common disease files.")
        return
239150
def getLimits(locations, current, sortResults=True, verbose=False):
    
    limit = {}
    for l in locations:
        a, b = current.common(l)
        if a is None:
            continue
        for name, value in b.items():
            f = a[name]
            if name not in limit:
                limit[name] = {}
                limit[name]['<'] = {}
                limit[name]['='] = {}
                limit[name]['>'] = {}
                if f > 0:
                    limit[name]['>'] = {0: [Location()]}
                elif f<0:
                    limit[name]['<'] = {0: [Location()]}
                else:
                    limit[name]['='] = {0: [Location()]}
            if current[name] < value - _EPSILON:
                if value not in limit[name]["<"]:
                    limit[name]["<"][value] = []
                limit[name]["<"][value].append(l)
            elif current[name] > value + _EPSILON:
                if value not in limit[name][">"]:
                    limit[name][">"][value] = []
                limit[name][">"][value].append(l)
            else:
                if value not in limit[name]["="]:
                    limit[name]["="][value] = []
                limit[name]["="][value].append(l)
    if not sortResults:
        return limit
    # now we have all the data, let's sort to the relevant values
    l = {}
    for name, lims in  limit.items():
        less = []
        more = []
        if lims[">"].keys():
            less = sorted(lims[">"].keys())
            lim_min = less[-1]
        else:
            lim_min = None
        if lims["<"].keys():
            more = sorted(lims["<"].keys())
            lim_max = more[0]
        else:
            lim_max = None
        if lim_min is None and lim_max is not None:
            # extrapolation < min
            if len(limit[name]['='])>0:
                l[name] = (None, list(limit[name]['='].keys())[0], None)
            elif len(more) > 1 and len(limit[name]['='])==0:
                # extrapolation
                l[name] = (None,  more[0], more[1])
        elif lim_min is not None and lim_max is None:
            # extrapolation < max
            if len(limit[name]['='])>0:
                # less > 0, M > 0, more = None
                # -> end of a limit
                l[name] = (None, limit[name]['='], None)
            elif len(less) > 1 and len(limit[name]['='])==0:
                # less > 0, M = None, more = None
                # extrapolation
                l[name] = (less[-2], less[-1], None)
        else:
            if len(limit[name]['=']) > 0:
                l[name] = (None, list(limit[name]['='].keys())[0], None)
            else:
                l[name] = (lim_min,  None, lim_max)
    return l
877606
def size_limit(self):
        
        limits = [
            lim for lim in current_files_rest.file_size_limiters(
                self)
            if lim.limit is not None
        ]
        return min(limits) if limits else None
662198
def limitted_dump(cursor=None, twitter=None, path='tweets.json', limit=450, rate=TWITTER_SEARCH_RATE_LIMIT, indent=-1):
    
    if not twitter:
        twitter = get_twitter()
    cursor = cursor or 'python'
    if isinstance(cursor, basestring):
        cursor = get_cursor(twitter, search=cursor)
    newline = '\n' if indent is not None else ''
    if indent < 0:
        indent = None
    # TODO: keep track of T0 for the optimal "reset" sleep duration
    with (open(path, 'w') if not isinstance(path, file) else path) as f:
        f.write('[\n')
        for i, obj in enumerate(cursor):
            f.write(json.dumps(obj, indent=indent))
            if i < limit - 1:
                f.write(',' + newline)
            else:
                break
            remaining = int(twitter.get_lastfunction_header('x-rate-limit-remaining'))
            if remaining > 0:
                sleep(1. / rate)
            else:
                sleep(15 * 60)
        f.write('\n]\n')
78821
def getToLimit(clus, limit)
      # Init.
      out = MemoryBuffer.create(limit)
      pos = 0
      cur_clus = clus

      # How many clusters fill request.
      num = limit.divmod(@bytesPerCluster)
      num_clus = num[0]; num_clus += 1 if num[1] > 0

      # Loop until done or EOF.
      while num_clus > 0

        # Find number of contiguous clusters & trim by num_clus.
        contig = countContigClusters(cur_clus)
        red_clus = num_clus > contig ? contig : num_clus

        # Get data.
        chunk = red_clus * @bytesPerCluster
        @stream.seek(clusToByte(cur_clus))
        out[pos, chunk] = @stream.read(chunk)
        pos += chunk

        # Inc current & dec number to read.
        cur_clus += (red_clus - 1); num_clus -= red_clus

        # Get next cluster & abort if end of chain.
        cur_clus = getFatEntry(cur_clus)
        break if cur_clus > CC_END_OF_CHAIN
      end

      # Return next cluster & data.
      [cur_clus, out]
    end
449628
def ReadDictionary(self, file):
        

        fil = dictfile.DictFile(file)

        state = {}
        state['vendor'] = ''

        self.defer_parse = []
        for line in fil:
            state['file'] = fil.File()
            state['line'] = fil.Line()
            line = line.split('#', 1)[0].strip()

            tokens = line.split()
            if not tokens:
                continue

            key = tokens[0].upper()
            if key == 'ATTRIBUTE':
                self.__ParseAttribute(state, tokens)
            elif key == 'VALUE':
                self.__ParseValue(state, tokens, True)
            elif key == 'VENDOR':
                self.__ParseVendor(state, tokens)
            elif key == 'BEGIN-VENDOR':
                self.__ParseBeginVendor(state, tokens)
            elif key == 'END-VENDOR':
                self.__ParseEndVendor(state, tokens)

        for state, tokens in self.defer_parse:
            key = tokens[0].upper()
            if key == 'VALUE':
                self.__ParseValue(state, tokens, False)
        self.defer_parse = []
957732
function limitPath($file)
    {
        $x = explode('/', $file);
        $l = count($x);
        return ($l>1) ? $x[$l-2] . '/' . $x[$l-1] : $file;
    }
355914
def _parse_rdf(self, file):
        
        store = Graph()
        store.parse(file)

        print len(store)
578517
def parse(self, limit=None, or_limit=1):
        
        dir_path = Path(self.rawdir)
        aeolus_file = dir_path / self.files['aeolus']['file']
        aeolus_fh = aeolus_file.open('r')
        count = 0
        for line in aeolus_fh.readlines():
            if limit is not None and count >= limit:
                break
            line = line.rstrip("\n,")
            if line != '[' and line != ']':
                self._parse_aeolus_data(document=json.loads(line),
                                        or_limit=or_limit)
                count += 1
            if count % 500 == 0:
                LOG.info("Processed %i documents", count)

        aeolus_fh.close()
        return
1330502
def parse(self, limit=None):
        
        if limit is not None:
            LOG.info("Only parsing first %d rows", limit)

        protein_paths = self._get_file_paths(self.tax_ids, 'protein_links')
        col = ['NCBI taxid', 'entrez', 'STRING']
        for taxon in protein_paths:
            ensembl = Ensembl(self.graph_type, self.are_bnodes_skized)
            string_file_path = '/'.join((
                self.rawdir, protein_paths[taxon]['file']))

            with gzip.open(string_file_path, 'rb') as reader:
                dataframe = pd.read_csv(reader, sep=r'\s+')
            p2gene_map = dict()

            if taxon in self.id_map_files:
                LOG.info("Using string provided id_map files")
                map_file = '/'.join((self.rawdir, self.id_map_files[taxon]['file']))

                with gzip.open(map_file, 'rt') as reader:
                    line = next(reader).strip()
                    if line != '# NCBI taxid / entrez / STRING':
                        LOG.error(
                            'Expected Headers:\t%s\nRecived Headers:\t%s\n', col, line)
                        exit(-1)

                    for line in reader.readlines():
                        row = line.rstrip('\n').split('\t')
                        # tax = row[col.index(''NCBI taxid')].strip()
                        gene = row[col.index('entrez')].strip()
                        prot = row[col.index('STRING')].strip()

                        genes = gene.split('|')
                        p2gene_map[prot.replace(taxon + '.', '')] = [
                            "NCBIGene:" + entrez_id for entrez_id in genes]
            else:
                LOG.info("Fetching ensembl proteins for taxon %s", taxon)
                p2gene_map = ensembl.fetch_protein_gene_map(taxon)
                for key in p2gene_map:
                    for phen, gene in enumerate(p2gene_map[key]):
                        p2gene_map[key][phen] = "ENSEMBL:{}".format(gene)

            LOG.info(
                "Finished fetching ENSP ID mappings, fetched %i proteins",
                len(p2gene_map))

            LOG.info(
                "Fetching protein protein interactions for taxon %s", taxon)

            self._process_protein_links(dataframe, p2gene_map, taxon, limit)
401529
def parse(self, limit=None):
        
        if limit is not None:
            LOG.info("Only parsing first %d rows", limit)

        LOG.info("Parsing files...")
        # pub_map = dict()
        # file_path = '/'.join((self.rawdir,
        # self.static_files['publications']['file']))
        # if os.path.exists(file_path) is True:
        #     pub_map = self._parse_publication_file(
        #         self.static_files['publications']['file']
        #     )

        if self.test_only:
            self.test_mode = True

        self.geno = Genotype(self.graph)
        self.pathway = Pathway(self.graph)

        self._parse_ctd_file(
            limit, self.files['chemical_disease_interactions']['file'])
        self._parse_ctd_file(limit, self.files['gene_pathway']['file'])
        self._parse_ctd_file(limit, self.files['gene_disease']['file'])
        self._parse_curated_chem_disease(limit)

        LOG.info("Done parsing files.")

        return
134212
def _parse_ctd_file(self, limit, file):
        
        row_count = 0
        version_pattern = re.compile(r'^# Report created: (.+)$')
        is_versioned = False
        file_path = '/'.join((self.rawdir, file))
        with gzip.open(file_path, 'rt') as tsvfile:
            reader = csv.reader(tsvfile, delimiter="\t")
            for row in reader:
                # Scan the header lines until we get the version
                # There is no official version sp we are using
                # the upload timestamp instead
                if is_versioned is False:
                    match = re.match(version_pattern, ' '.join(row))
                    if match:
                        version = re.sub(r'\s|:', '-', match.group(1))
                        # TODO convert this timestamp to a proper timestamp
                        self.dataset.setVersion(version)
                        is_versioned = True
                elif re.match(r'^#', ' '.join(row)):
                    pass
                else:
                    row_count += 1
                    if file == self.files[
                            'chemical_disease_interactions']['file']:
                        self._process_interactions(row)
                    elif file == self.files['gene_pathway']['file']:
                        self._process_pathway(row)
                    elif file == self.files['gene_disease']['file']:
                        self._process_disease2gene(row)

                if not self.test_mode and limit is not None and row_count >= limit:
                    break

        return
366396
def filer(filelist):
    
    # Initialise the dictionary
    filedict = dict()
    for seqfile in filelist:
        # Split off the file extension and remove the path from the name
        strainname = os.path.splitext(os.path.basename(seqfile))[0]
        # Populate the dictionary
        filedict[strainname] = seqfile
    return filedict
146223
def list_documents(self, limit=None):
        
        limit_str = ''
        if limit:
            try:
                limit_str = 'LIMIT {}'.format(int(limit))
            except (TypeError, ValueError):
                pass

        query = ('SELECT identifier FROM identifier_index ' + limit_str)

        for row in self.backend.library.database.connection.execute(query).fetchall():
            yield row['identifier']
1336550
def parse(self, limit=None):
        

        files_to_download, ftp = self._get_file_list(
            self.files['anat_entity']['path'],
            self.files['anat_entity']['pattern'])
        for dlname in files_to_download:
            localfile = '/'.join((self.rawdir, dlname))
            with gzip.open(localfile, 'rt', encoding='ISO-8859-1') as fh:
                LOG.info("Processing %s", localfile)
                self._parse_gene_anatomy(fh, limit)
        return