Export records as a GTFS comma-separated file
198388
def export_txt(cls, feed):
        ''''''
        objects = cls.objects.in_feed(feed)

        # If no records, return None
        if not objects.exists():
            return

        # Get the columns used in the dataset
        column_map = objects.populated_column_map()
        columns, fields = zip(*column_map)
        extra_columns = feed.meta.get(
            'extra_columns', {}).get(cls.__name__, [])

        # Get sort order
        if hasattr(cls, '_sort_order'):
            sort_fields = cls._sort_order
        else:
            sort_fields = []
            for field in fields:
                base_field = field.split('__', 1)[0]
                point_match = re_point.match(base_field)
                if point_match:
                    continue
                field_type = cls._meta.get_field(base_field)
                assert not isinstance(field_type, ManyToManyField)
                sort_fields.append(field)

        # Create CSV writer
        out = StringIO()
        csv_writer = writer(out, lineterminator='\n')

        # Write header row
        header_row = [text_type(c) for c in columns]
        header_row.extend(extra_columns)
        write_text_rows(csv_writer, [header_row])

        # Report the work to be done
        total = objects.count()
        logger.info(
            '%d %s to export...',
            total, cls._meta.verbose_name_plural)

        # Populate related items cache
        model_to_field_name = {}
        cache = {}
        for field_name in fields:
            if '__' in field_name:
                local_field_name, subfield_name = field_name.split('__', 1)
                field = cls._meta.get_field(local_field_name)
                field_type = field.related_model
                model_name = field_type.__name__
                if model_name in model_to_field_name:
                    # Already loaded this model under a different field name
                    cache[field_name] = cache[model_to_field_name[model_name]]
                else:
                    # Load all feed data for this model
                    pairs = field_type.objects.in_feed(
                        feed).values_list('id', subfield_name)
                    cache[field_name] = dict(
                        (i, text_type(x)) for i, x in pairs)
                    cache[field_name][None] = u''
                    model_to_field_name[model_name] = field_name

        # Assemble the rows, writing when we hit batch size
        count = 0
        rows = []
        for item in objects.order_by(*sort_fields).iterator():
            row = []
            for csv_name, field_name in column_map:
                obj = item
                point_match = re_point.match(field_name)
                if '__' in field_name:
                    # Return relations from cache
                    local_field_name = field_name.split('__', 1)[0]
                    field_id = getattr(obj, local_field_name + '_id')
                    row.append(cache[field_name][field_id])
                elif point_match:
                    # Get the lat or long from the point
                    name, index = point_match.groups()
                    field = getattr(obj, name)
                    row.append(field.coords[int(index)])
                else:
                    # Handle other field types
                    field = getattr(obj, field_name) if obj else ''
                    if isinstance(field, date):
                        formatted = field.strftime(u'%Y%m%d')
                        row.append(text_type(formatted))
                    elif isinstance(field, bool):
                        row.append(1 if field else 0)
                    elif field is None:
                        row.append(u'')
                    else:
                        row.append(text_type(field))
            for col in extra_columns:
                row.append(obj.extra_data.get(col, u''))
            rows.append(row)
            if len(rows) % batch_size == 0:  # pragma: no cover
                write_text_rows(csv_writer, rows)
                count += len(rows)
                logger.info(
                    "Exported %d %s",
                    count, cls._meta.verbose_name_plural)
                rows = []

        # Write rows smaller than batch size
        write_text_rows(csv_writer, rows)
        return out.getvalue()
948401
def _convert_file_records(self, paths):
        
        ret = []
        for path in paths:
            # path = self.fs.remove_prefix(path, self.prefix)  # Remove bucket prefix from paths
            if os.path.basename(path) == self.fs.dir_keep_file:
                continue
            type_ = self.guess_type(path, allow_directory=True)
            if type_ == "notebook":
                ret.append(self._notebook_model_from_path(path, False))
            elif type_ == "file":
                ret.append(self._file_model_from_path(path, False, None))
            elif type_ == "directory":
                ret.append(self._directory_model_from_path(path, False))
            else:
                self.do_error("Unknown file type %s for file '%s'" % (type_, path), 500)
        return ret
141753
def _export(self, collection):
        
        fieldnames = collection[0].keys()
        faux_csv = ','.join(fieldnames) + '\r\n'
        for resource in collection:
            faux_csv += ','.join((str(x) for x in resource.values())) + '\r\n'
        response = make_response(faux_csv)
        response.mimetype = 'text/csv'
        return response
803897
def export(self, storage_client, overwrite=True):
    
        '''
            
        '''
        
        title = '%s.export' % self.__class__.__name__
        
    # validate storage client
        method_list = [ 'save', 'load', 'list', 'export', 'delete', 'remove', '_import', 'collection_name' ]
        for method in method_list:
            if not getattr(storage_client, method, None):
                from labpack.parsing.grammar import join_words
                raise ValueError('%s(storage_client=...) must be a client object with %s methods.' % (title, join_words(method_list)))
    
    # define copy record function
        def _copy_record(_record, _storage_client):
            record_key = _record['key']
            record_data, record_metadata = self.s3.read_record(self.bucket_name, record_key)
            encryption = ''
            if 'encryption' in record_metadata['metadata'].keys():
                encryption = record_metadata['metadata']['encryption']
            last_modified = 0.0
            if 'last_modified' in record_metadata['metadata'].keys():
                try:
                    last_modified = float(record_metadata['metadata']['last_modified'])
                except:
                    pass
            outcome = _storage_client._import(record_key, record_data, overwrite=overwrite, encryption=encryption, last_modified=last_modified)
            return outcome
            
    # retrieve list of records in bucket
        count = 0
        skipped = 0
        record_list, next_key = self.s3.list_records(self.bucket_name)
        for record in record_list:
            outcome = _copy_record(record, storage_client)
            if outcome:
                count += 1
            else:
                skipped += 1
        
    # continue through bucket
        if next_key:
            while next_key:
                record_list, next_key = self.s3.list_records(self.bucket_name, starting_key=next_key)
                for record in record_list:
                    outcome = _copy_record(record, storage_client)
                    if outcome:
                        count += 1
                    else:
                        skipped += 1
    
    # report outcome
        from os import path
        plural = ''
        skip_insert = ''
        new_root, new_folder = path.split(storage_client.collection_folder)
        if count != 1:
            plural = 's'
        if skipped > 0:
            skip_plural = ''
            if skipped > 1:
                skip_plural = 's'
            skip_insert = ' %s record%s skipped to avoid overwrite.' % (str(skipped), skip_plural)
        exit_msg = '%s record%s exported to %s.%s' % (str(count), plural, new_folder, skip_insert)
        return exit_msg
430545
def dump2sqlite(records, output_file):
    """"""
    results_keys = list(records.results[0].keys())
    pad_data = []

    for key in REQUIRED_KEYS:
        if key not in results_keys:
            results_keys.append(key)
            pad_data.append("")

    conn = sqlite3.connect(os.path.expanduser(output_file), detect_types=sqlite3.PARSE_DECLTYPES)

    # in each row there needs to be data for every column
    # last column is current time
    pad_data.append(datetime.datetime.utcnow())

    to_db = [list(row.values()) + pad_data for row in records.results]

    cur = conn.cursor()
    cur.execute(
        "CREATE TABLE testcases ({},sqltime TIMESTAMP)".format(
            ",".join("{} TEXT".format(key) for key in results_keys)
        )
    )
    cur.executemany(
        "INSERT INTO testcases VALUES ({},?)".format(",".join(["?"] * len(results_keys))), to_db
    )

    if records.testrun:
        cur.execute("CREATE TABLE testrun (testrun TEXT)")
        cur.execute("INSERT INTO testrun VALUES (?)", (records.testrun,))

    conn.commit()
    conn.close()

    logger.info("Data written to '%s'", output_file)
343133
def export(self, timestamp=None, info={}):
        
        tval = tuple(time.localtime()) if timestamp is None else timestamp
        tstamp = time.strftime(self.timestamp_format, tval)

        info = dict(info, timestamp=tstamp)
        export_name = self._format(self.export_name, info)
        files = [((self._format(base, info), ext), val)
                 for ((base, ext), val) in self._files.items()]
        root = os.path.abspath(self.root)
        # Make directory and populate if multiple files and not packed
        if len(self) > 1 and not self.pack:
            self._directory_archive(export_name, files, root)
        elif len(files) == 1:
            self._single_file_archive(export_name, files, root)
        elif self.archive_format == 'zip':
            self._zip_archive(export_name, files, root)
        elif self.archive_format == 'tar':
            self._tar_archive(export_name, files, root)
        if self.flush_archive:
            self._files = OrderedDict()
134228
function exportCSV(params, cb) {
  var resourcePath = config.addURIParams("/appforms/submissions/export", params);
  var method = "POST";
  var data = params.queryParams;

  params.resourcePath = resourcePath;
  params.method = method;
  params.data = data;

  params.fileRequest = true;

  mbaasRequest.app(params, cb);
}
187029
def get_gtf():
    
    size, md5, url = GTF
    full_gtf = os.path.join(args.data_dir, os.path.basename(url))
    subset_gtf = os.path.join(
        args.data_dir,
        os.path.basename(url).replace('.gtf.gz', '_%s.gtf' % CHROM))

    if not _up_to_date(md5, subset_gtf):
        download(url, full_gtf)
        cmds = [
            'zcat', '<',
            full_gtf,
            '|', 'awk -F "\\t" \'{if ($1 == "%s") print $0}\''
            % CHROM.replace('chr', ''),
            '|', 'awk \'{print "chr"$0}\'', '>', subset_gtf]

        logged_command(cmds)
9541
def DumpAsCSV (self, separator=",", file=sys.stdout):
                """"""
                for row in range(1, self.maxRow + 1):
                        sep = ""
                        for column in range(1, self.maxColumn + 1):
                                file.write("%s\"%s\"" % (sep, self.GetCellValue(column, row, "")))
                                sep = separator
                        file.write("\n")
588819
def save_csvf(data: list, fieldnames: Sequence[str], fpath: str, encoding: str, with_header: bool = False,
              crlf: bool = False, tsv: bool = False) -> str:
    
    with codecs.open(fpath, mode='w', encoding=encoding) as f:
        f.write(dump_csv(data, fieldnames, with_header=with_header, crlf=crlf, tsv=tsv))
        return fpath
414881
def csv(self, client_id):
        """"""
        logging.info('Exporting CSV file [{}]'.format(client_id))
        query = (
            db.session.query(Query)
            .filter_by(client_id=client_id)
            .one()
        )

        rejected_tables = security_manager.rejected_datasources(
            query.sql, query.database, query.schema)
        if rejected_tables:
            flash(
                security_manager.get_table_access_error_msg('{}'.format(rejected_tables)))
            return redirect('/')
        blob = None
        if results_backend and query.results_key:
            logging.info(
                'Fetching CSV from results backend '
                '[{}]'.format(query.results_key))
            blob = results_backend.get(query.results_key)
        if blob:
            logging.info('Decompressing')
            json_payload = utils.zlib_decompress_to_string(blob)
            obj = json.loads(json_payload)
            columns = [c['name'] for c in obj['columns']]
            df = pd.DataFrame.from_records(obj['data'], columns=columns)
            logging.info('Using pandas to convert to CSV')
            csv = df.to_csv(index=False, **config.get('CSV_EXPORT'))
        else:
            logging.info('Running a query to turn into CSV')
            sql = query.select_sql or query.executed_sql
            df = query.database.get_df(sql, query.schema)
            # TODO(bkyryliuk): add compression=gzip for big files.
            csv = df.to_csv(index=False, **config.get('CSV_EXPORT'))
        response = Response(csv, mimetype='text/csv')
        response.headers['Content-Disposition'] = f'attachment; filename={query.name}.csv'
        logging.info('Ready to return response')
        return response
846036
function exportArchiveToFilesystem (opts, cb) {
  return maybe(cb, async function () {
    assert(opts && typeof opts === 'object', 'opts object is required')

    // core arguments, dstPath and srcArchive
    var srcArchive = opts.srcArchive
    var dstPath = opts.dstPath
    assert(srcArchive && typeof srcArchive === 'object', 'srcArchive is required')
    assert(dstPath && typeof dstPath === 'string', 'dstPath is required')

    // options
    var srcPath = typeof opts.srcPath === 'string' ? opts.srcPath : '/'
    var overwriteExisting = opts.overwriteExisting === true
    var skipUndownloadedFiles = opts.skipUndownloadedFiles === true
    var ignore = Array.isArray(opts.ignore) ? opts.ignore : DEFAULT_IGNORE

    // abort if nonempty and not overwriting existing
    if (!overwriteExisting) {
      let files
      try {
        files = await fse.readdir(dstPath)
      } catch (e) {
        // target probably doesnt exist, continue and let ensureDirectory handle it
      }
      if (files && files.length > 0) {
        throw new DestDirectoryNotEmpty()
      }
    }

    const statThenExport = async function (srcPath, dstPath) {
      // apply ignore filter
      if (ignore && match(ignore, srcPath)) {
        return
      }

      // export by type
      var srcStat = await stat(srcArchive, srcPath)
      if (srcStat.isFile()) {
        await exportFile(srcPath, srcStat, dstPath)
      } else if (srcStat.isDirectory()) {
        await exportDirectory(srcPath, dstPath)
      }
    }

    const exportFile = async function (srcPath, srcStat, dstPath) {
      // skip undownloaded files
      if (skipUndownloadedFiles && srcStat.downloaded < srcStat.blocks) {
        return
      }

      // fetch dest stats
      var dstFileStats = null
      try {
        dstFileStats = await fse.stat(dstPath)
      } catch (e) {}

      // track the stats
      stats.fileCount++
      stats.totalSize += srcStat.size || 0
      if (dstFileStats) {
        if (dstFileStats.isDirectory()) {
          // delete the directory-tree
          await fse.remove(dstPath)
          stats.addedFiles.push(dstPath)
        } else {
          stats.updatedFiles.push(dstPath)
        }
      } else {
        stats.addedFiles.push(dstPath)
      }

      // write the file
      return new Promise((resolve, reject) => {
        pump(
          srcArchive.createReadStream(srcPath),
          fse.createWriteStream(dstPath),
          err => {
            if (err) reject(err)
            else resolve()
          }
        )
      })
    }

    const exportDirectory = async function (srcPath, dstPath) {
      // make sure the destination folder exists
      await fse.ensureDir(dstPath)

      // list the directory
      var fileNames = await readdir(srcArchive, srcPath)

      // recurse into each
      var promises = fileNames.map(name => {
        return statThenExport(path.join(srcPath, name), path.join(dstPath, name))
      })
      await Promise.all(promises)
    }

    // recursively export
    var stats = { addedFiles: [], updatedFiles: [], skipCount: 0, fileCount: 0, totalSize: 0 }
    await statThenExport(srcPath, dstPath)
    return stats
  })
}
1138896
def tx2genefile(gtf, out_file=None, data=None, tsv=True, keep_version=False):
    
    if tsv:
        extension = ".tsv"
        sep = "\t"
    else:
        extension = ".csv"
        sep = ","
    if file_exists(out_file):
        return out_file
    with file_transaction(data, out_file) as tx_out_file:
        with open(tx_out_file, "w") as out_handle:
            for k, v in tx2genedict(gtf, keep_version).items():
                out_handle.write(sep.join([k, v]) + "\n")
    logger.info("tx2gene file %s created from %s." % (out_file, gtf))
    return out_file
237564
def tx2genefile(gtf, out_file=None):
    
    installed_tx2gene = os.path.join(os.path.dirname(gtf), "tx2gene.csv")
    if file_exists(installed_tx2gene):
        return installed_tx2gene
    if file_exists(out_file):
        return out_file
    with file_transaction(out_file) as tx_out_file:
        with open(tx_out_file, "w") as out_handle:
            for k, v in transcript_to_gene(gtf).items():
                out_handle.write(",".join([k, v]) + "\n")
    return out_file
1122576
def export(self, out_file: str, layer: str = None, format: str = "tab") -> None:
		
		if format != "tab":
			raise NotImplementedError("Only 'tab' is supported")

		with open(out_file, "w") as f:
			# Emit column attributes
			for ca in self.col_attrs.keys():
				for ra in self.row_attrs.keys():
					f.write("\t")
				f.write(ca + "\t")
				for v in self.col_attrs[ca]:
					f.write(str(v) + "\t")
				f.write("\n")

			# Emit row attribute names
			for ra in self.row_attrs.keys():
				f.write(ra + "\t")
			f.write("\t")
			for v in range(self.shape[1]):
				f.write("\t")
			f.write("\n")

			# Emit row attr values and matrix values
			for row in range(self.shape[0]):
				for ra in self.row_attrs.keys():
					f.write(str(self.row_attrs[ra][row]) + "\t")
				f.write("\t")

				if layer is None:
					for v in self[row, :]:
						f.write(str(v) + "\t")
				else:
					for v in self.layers[layer][row, :]:
						f.write(str(v) + "\t")
				f.write("\n")
1331298
def to_vcf(in_tsv, data):
    
    call_convert = {"Amp": "DUP", "Del": "DEL"}
    out_file = "%s.vcf" % utils.splitext_plus(in_tsv)[0]
    if not utils.file_uptodate(out_file, in_tsv):
        with file_transaction(data, out_file) as tx_out_file:
            with open(in_tsv) as in_handle:
                with open(tx_out_file, "w") as out_handle:
                    out_handle.write(VCF_HEADER + "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t%s\n"
                                     % (dd.get_sample_name(data)))
                    header = in_handle.readline().split("\t")
                    for cur in (dict(zip(header, l.split("\t"))) for l in in_handle):
                        if cur["Amp_Del"] in call_convert:
                            svtype = call_convert[cur["Amp_Del"]]
                            info = "SVTYPE=%s;END=%s;SVLEN=%s;FOLD_CHANGE_LOG=%s;PROBES=%s;GENE=%s" % (
                                svtype, cur["End"], int(cur["End"]) - int(cur["Start"]),
                                cur["Log2ratio"], cur["Ab_Seg"], cur["Gene"])
                            out_handle.write("\t".join([cur["Chr"], cur["Start"], ".", "N", "<%s>" % (svtype),
                                                        ".", ".", info, "GT", "1/1"]) + "\n")
    return vcfutils.sort_by_ref(out_file, data)
118261
def prt_tsv_dat(prt, data_nts, **kws):
    """"""
    sep = "\t" if 'sep' not in kws else kws['sep']
    fld2fmt = None if 'fld2fmt' not in kws else kws['fld2fmt']
    if 'sort_by' in kws:
        data_nts = sorted(data_nts, key=kws['sort_by'])
    prt_if = kws['prt_if'] if 'prt_if' in kws else None
    prt_flds = kws['prt_flds'] if 'prt_flds' in kws else data_nts[0]._fields
    items = 0
    for nt_data_row in data_nts:
        if prt_if is None or prt_if(nt_data_row):
            if fld2fmt is not None:
                row_fld_vals = [(fld, getattr(nt_data_row, fld)) for fld in prt_flds]
                row_vals = _fmt_fields(row_fld_vals, fld2fmt)
            else:
                row_vals = [getattr(nt_data_row, fld) for fld in prt_flds]
            prt.write("{}\n".format(sep.join(str(d) for d in row_vals)))
            items += 1
    return items