IMPC data is delivered in three separate csv files OR
        in one integrated file, each with the same file format.

        :param limit:
        :return:
436848
def parse(self, limit=None):
        
        if limit is not None:
            LOG.info("Only parsing first %s rows fo each file", str(limit))

        LOG.info("Parsing files...")

        if self.test_only:
            self.test_mode = True

        # for f in ['impc', 'euro', 'mgd', '3i']:
        for f in ['all']:
            file = '/'.join((self.rawdir, self.files[f]['file']))
            self._process_data(file, limit)

        LOG.info("Finished parsing")

        return
395503
def _format_count_file(count_file, data):
    
    COUNT_COLUMN = 5
    out_file = os.path.splitext(count_file)[0] + ".fixed.counts"
    if file_exists(out_file):
        return out_file

    df = pd.io.parsers.read_table(count_file, sep="\t", index_col=0, header=1)
    df_sub = df.ix[:, COUNT_COLUMN]
    with file_transaction(data, out_file) as tx_out_file:
        df_sub.to_csv(tx_out_file, sep="\t", index_label="id", header=False)
    return out_file
1170579
def process_catalog(self, limit=None):
        
        raw = '/'.join((self.rawdir, self.files['catalog']['file']))
        LOG.info("Processing Data from %s", raw)
        efo_ontology = RDFGraph(False, "EFO")
        LOG.info("Loading EFO ontology in separate rdf graph")
        efo_ontology.parse(self.files['efo']['url'], format='xml')
        efo_ontology.bind_all_namespaces()
        LOG.info("Finished loading EFO ontology")

        so_ontology = RDFGraph(False, "SO")
        LOG.info("Loading SO ontology in separate rdf graph")
        so_ontology.parse(self.files['so']['url'], format='xml')
        so_ontology.bind_all_namespaces()
        LOG.info("Finished loading SO ontology")

        with open(raw, 'r', encoding="iso-8859-1") as csvfile:
            filereader = csv.reader(csvfile, delimiter='\t')
            header = next(filereader, None)  # the header row
            header_len = len(header)
            LOG.info('header length:\t %i', header_len)

            for row in filereader:
                if not row:
                    pass
                else:
                    if header_len != len(row):
                        LOG.error('BadRow: %i has %i columns', filereader.line_num, row)

                    (date_added_to_catalog,
                     pubmed_num,
                     first_author,
                     pub_date,
                     journal,
                     link,
                     study_name,
                     disease_or_trait,
                     initial_sample_description,
                     replicate_sample_description,
                     region,
                     chrom_num,
                     chrom_pos,
                     reported_gene_nums,
                     mapped_gene,
                     upstream_gene_num,
                     downstream_gene_num,
                     snp_gene_nums,
                     upstream_gene_distance,
                     downstream_gene_distance,
                     strongest_snp_risk_allele,
                     snps,
                     merged,
                     snp_id_current,
                     context,
                     intergenic_flag,
                     risk_allele_frequency,
                     pvalue,
                     pvalue_mlog,
                     pvalue_text,
                     or_or_beta,
                     confidence_interval_95,
                     platform_with_snps_passing_qc,
                     cnv_flag,
                     mapped_trait,
                     mapped_trait_uri,
                     study_accession,
                     GENOTYPING_TECHNOLOGY
                    ) = row

                    if self.test_mode:
                        continue

# 06-May-2015	25917933
#   Zai CC	20-Nov-2014	J Psychiatr Res	http://europepmc.org/abstract/MED/25917933
# A genome-wide association study of suicide severity scores in bipolar disorder.
# Suicide in bipolar disorder
# 959 European ancestry individuals	NA
# 10p11.22	10	32704340	C10orf68, CCDC7, ITGB1	CCDC7
# rs7079041-A	rs7079041	0	7079041	intron	0		2E-6	5.698970

                    variant_curie, variant_type = self._get_curie_and_type_from_id(
                        strongest_snp_risk_allele)

                    if strongest_snp_risk_allele.strip() == '':
                        LOG.debug(
                            "No strongest SNP risk allele for %s:\n%s",
                            pubmed_num, str(row))
                        # still consider adding in the EFO terms
                        # for what the study measured?
                        continue

                    if variant_type == 'snp':
                        self._add_snp_to_graph(
                            variant_curie, strongest_snp_risk_allele, chrom_num,
                            chrom_pos, context, risk_allele_frequency)

                        self._add_deprecated_snp(
                            variant_curie, snp_id_current, merged, chrom_num, chrom_pos)

                        self._add_snp_gene_relation(
                            variant_curie, snp_gene_nums, upstream_gene_num,
                            downstream_gene_num)
                    elif variant_type == 'haplotype':
                        self._process_haplotype(
                            variant_curie, strongest_snp_risk_allele, chrom_num,
                            chrom_pos, context, risk_allele_frequency, mapped_gene,
                            so_ontology)
                    elif variant_type is None:
                        LOG.warning(
                            "There's a snp id i can't manage: %s",
                            strongest_snp_risk_allele)
                        continue

                    description = self._make_description(
                        disease_or_trait, initial_sample_description,
                        replicate_sample_description,
                        platform_with_snps_passing_qc, pvalue)

                    self._add_variant_trait_association(
                        variant_curie, mapped_trait_uri, efo_ontology,
                        pubmed_num, description)

                    if not self.test_mode and (
                            limit is not None and filereader.line_num > limit):
                        break

        # TODO loop through the location hash,
        # and make all snps at that location equivalent
        for l in self.id_location_map:
            snp_ids = self.id_location_map[l]
            if len(snp_ids) > 1:
                LOG.info("%s has >1 snp id: %s", l, str(snp_ids))
        return
797279
def convert_ulog2csv(ulog_file_name, messages, output, delimiter):
    

    msg_filter = messages.split(',') if messages else None

    ulog = ULog(ulog_file_name, msg_filter)
    data = ulog.data_list

    output_file_prefix = ulog_file_name
    # strip '.ulg'
    if output_file_prefix.lower().endswith('.ulg'):
        output_file_prefix = output_file_prefix[:-4]

    # write to different output path?
    if output:
        base_name = os.path.basename(output_file_prefix)
        output_file_prefix = os.path.join(output, base_name)

    for d in data:
        fmt = '{0}_{1}_{2}.csv'
        output_file_name = fmt.format(output_file_prefix, d.name, d.multi_id)
        fmt = 'Writing {0} ({1} data points)'
        # print(fmt.format(output_file_name, len(d.data['timestamp'])))
        with open(output_file_name, 'w') as csvfile:

            # use same field order as in the log, except for the timestamp
            data_keys = [f.field_name for f in d.field_data]
            data_keys.remove('timestamp')
            data_keys.insert(0, 'timestamp')  # we want timestamp at first position

            # we don't use np.savetxt, because we have multiple arrays with
            # potentially different data types. However the following is quite
            # slow...

            # write the header
            csvfile.write(delimiter.join(data_keys) + '\n')

            # write the data
            last_elem = len(data_keys)-1
            for i in range(len(d.data['timestamp'])):
                for k in range(len(data_keys)):
                    csvfile.write(str(d.data[data_keys[k]][i]))
                    if k != last_elem:
                        csvfile.write(delimiter)
                csvfile.write('\n')
700049
def get_input_sequence_files(data, default=None):
    
    if "files" not in data or data.get("files") is None:
        file1, file2 = None, None
    elif len(data["files"]) == 2:
        file1, file2 = data["files"]
    else:
        assert len(data["files"]) == 1, data["files"]
        file1, file2 = data["files"][0], None
    return file1, file2
1097049
def process_data(self, fname, tokenizer, max_size):
        
        logging.info(f'Processing data from {fname}')
        data = []
        with open(fname) as dfile:
            for idx, line in enumerate(dfile):
                if max_size and idx == max_size:
                    break
                entry = tokenizer.segment(line)
                entry = torch.tensor(entry)
                data.append(entry)
        return data
440836
def readline(self, limit=-1, delim=b'\n'):
        
        self._check_readable()
        chunks = []
        while True:
            chunk = self._buffer.get_chunk(limit, delim)
            if not chunk:
                break
            chunks.append(chunk)
            if chunk.endswith(delim):
                break
            if limit >= 0:
                limit -= len(chunk)
                if limit == 0:
                    break
        if not chunks and not self._buffer.eof and self._buffer.error:
            raise compat.saved_exc(self._buffer.error)
        return b''.join(chunks)
1336809
def distribution(self, limit=1024):
        
        res = self._qexec("%s, count(*) as __cnt" % self.name(), group="%s" % self.name(),
                          order="__cnt DESC LIMIT %d" % limit)
        dist = []
        cnt = self._table.size()
        for i, r in enumerate(res):
            dist.append(list(r) + [i, r[1] / float(cnt)])

        self._distribution = pd.DataFrame(dist, columns=["value", "cnt", "r", "fraction"])
        self._distribution.index = self._distribution.r

        return self._distribution
634899
def _merge_fastqc(samples):
    
    fastqc_list = collections.defaultdict(list)
    seen = set()
    for data in samples:
        name = dd.get_sample_name(data)
        if name in seen:
            continue
        seen.add(name)
        fns = glob.glob(os.path.join(dd.get_work_dir(data), "qc", dd.get_sample_name(data), "fastqc") + "/*")
        for fn in fns:
            if fn.endswith("tsv"):
                metric = os.path.basename(fn)
                fastqc_list[metric].append([name, fn])

    for metric in fastqc_list:
        dt_by_sample = []
        for fn in fastqc_list[metric]:
            dt = pd.read_csv(fn[1], sep="\t")
            dt['sample'] = fn[0]
            dt_by_sample.append(dt)
        dt = utils.rbind(dt_by_sample)
        dt.to_csv(metric, sep="\t", index=False, mode ='w')
    return samples
688383
def csv(self, jql, limit=1000):
        
        url = 'sr/jira.issueviews:searchrequest-csv-all-fields/temp/SearchRequest.csv?tempMax={limit}&jqlQuery={jql}'.format(
            limit=limit, jql=jql)
        return self.get(url, not_json_response=True, headers={'Accept': 'application/csv'})
178762
def limit(self, limit):
        
        if self._limit != limit:
            self.dirty = True
            self._limit = limit
        return self
449060
def datagetter(cls):
        
        with open('myfile', 'rt') as f:
            rows = [r for r in csv.reader(f)]
        dothing = lambda _: [i for i, v in enumerate(_)]
        rows = [dothing(_) for _ in rows]
        raise NotImplementedError('You need to implement this yourlself!')
        return rows
403531
def checkMultipleFiles(input):
    
    f,i,o,a=buildFileList(input)
    return len(f) > 1
1005009
def restricted_to_files(self,
                            filenames: List[str]
                            ) -> 'Spectra':
        
        tally_passing = \
            {fn: entries for (fn, entries) in self.__tally_passing.items() \
             if fn in filenames}
        tally_failing = \
            {fn: entries for (fn, entries) in self.__tally_failing.items() \
             if fn in filenames}
        return Spectra(self.__num_passing,
                       self.__num_failing,
                       tally_passing,
                       tally_failing)
770531
def __encodeMultipart(self, fields, files):
        
        boundary = mimetools.choose_boundary()
        crlf = '\r\n'

        l = []
        for k, v in fields:
            l.append('--' + boundary)
            l.append('Content-Disposition: form-data; name="%s"' % k)
            l.append('')
            l.append(v)
        for (k, f, v) in files:
            l.append('--' + boundary)
            l.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (k, f))
            l.append('Content-Type: %s' % self.__getContentType(f))
            l.append('')
            l.append(v)
        l.append('--' + boundary + '--')
        l.append('')
        body = crlf.join(l)

        return boundary, body
1238250
def interpret(self, infile):
        

        data = []

        for record in self.generate_records(infile):
            data.append(record)

        return data
1299592
def read_content(path: str, limit: Optional[int] = None) -> Iterator[List[str]]:
    
    with smart_open(path) as indata:
        for i, line in enumerate(indata):
            if limit is not None and i == limit:
                break
            yield list(get_tokens(line))