Runs bcp FORMAT command to create a format file that will assist in creating the bulk data file
427523
def make(self, cmd_args, db_args):
        '''
        
        '''
        with NamedTemporaryFile(delete=True) as f:
            format_file = f.name + '.bcp-format'
        format_args = cmd_args + ['format', NULL_FILE, '-c', '-f', format_file, '-t,'] + db_args
        _run_cmd(format_args)
        self.load(format_file)
        return format_file
991339
def _prep_callable_bed(in_file, work_dir, stats, data):
    
    out_file = os.path.join(work_dir, "%s-merge.bed.gz" % utils.splitext_plus(os.path.basename(in_file))[0])
    gsort = config_utils.get_program("gsort", data)
    if not utils.file_uptodate(out_file, in_file):
        with file_transaction(data, out_file) as tx_out_file:
            fai_file = ref.fasta_idx(dd.get_ref_file(data))
            cmd = ("{gsort} {in_file} {fai_file} | bedtools merge -i - -d {stats[merge_size]} | "
                   "bgzip -c > {tx_out_file}")
            do.run(cmd.format(**locals()), "Prepare SV callable BED regions")
    return vcfutils.bgzip_and_index(out_file, data["config"])
7393
def create_load_string(upload_dir, groups=None, organism=None):
    
    libraryline = 'library(bcbioRNASeq)'
    load_template = Template(
        ('bcb <- bcbioRNASeq(uploadDir="$upload_dir",'
         'interestingGroups=$groups,'
         'organism="$organism")'))
    load_noorganism_template = Template(
        ('bcb <- bcbioRNASeq(uploadDir="$upload_dir",'
         'interestingGroups=$groups,'
         'organism=NULL)'))
    flatline = 'flat <- flatFiles(bcb)'
    saveline = 'saveData(bcb, flat, dir="data")'
    if groups:
        groups = _list2Rlist(groups)
    else:
        groups = _quotestring("sampleName")
    if organism:
        load_bcbio = load_template.substitute(
            upload_dir=upload_dir, groups=groups, organism=organism)
    else:
        load_bcbio = load_noorganism_template.substitute(upload_dir=upload_dir,
                                                         groups=groups)
    return ";\n".join([libraryline, load_bcbio, flatline, saveline])
1207518
def create_tables(name, runSetResults, rows, rowsDiff, outputPath, outputFilePattern, options):
    '''
    
    '''

    # get common folder of sourcefiles
    common_prefix = os.path.commonprefix([r.filename for r in rows]) # maybe with parts of filename
    common_prefix = common_prefix[: common_prefix.rfind('/') + 1] # only foldername
    for row in rows:
        Row.set_relative_path(row, common_prefix, outputPath)

    # compute nice name of each run set for displaying
    firstBenchmarkName = Util.prettylist(runSetResults[0].attributes['benchmarkname'])
    allBenchmarkNamesEqual = all(Util.prettylist(r.attributes['benchmarkname']) == firstBenchmarkName for r in runSetResults)
    for r in runSetResults:
        if not r.attributes['name']:
            r.attributes['niceName'] = r.attributes['benchmarkname']
        elif allBenchmarkNamesEqual:
            r.attributes['niceName'] = r.attributes['name']
        else:
            r.attributes['niceName'] = [Util.prettylist(r.attributes['benchmarkname']) + '.' + Util.prettylist(r.attributes['name'])]

    template_values = lambda: None # dummy object as dict replacement for simpler syntax
    template_values.head = get_table_head(runSetResults, common_prefix)
    template_values.run_sets = [runSetResult.attributes for runSetResult in runSetResults]
    template_values.columns = [[column for column in runSet.columns] for runSet in runSetResults]
    template_values.columnTitles =\
        [[column.format_title() for column in runSet.columns] for runSet in runSetResults]

    template_values.relevant_id_columns = select_relevant_id_columns(rows)
    template_values.count_id_columns = template_values.relevant_id_columns.count(True)

    template_values.lib_url = options.lib_url
    template_values.base_dir = outputPath
    template_values.href_base = os.path.dirname(options.xmltablefile) if options.xmltablefile else None
    template_values.version = __version__

    futures = []

    def write_table(table_type, title, rows, use_local_summary):
        # calculate statistics if necessary
        if not options.format == ['csv']:
            local_summary = get_summary(runSetResults) if use_local_summary else None
            stats, stats_columns = get_stats(rows, local_summary, options.correct_only)
        else:
            stats = stats_columns = None

        for template_format in (options.format or TEMPLATE_FORMATS):
            if outputFilePattern == '-':
                outfile = None
                logging.info('Writing %s to stdout...', template_format.upper().ljust(4))
            else:
                outfile = os.path.join(outputPath, outputFilePattern.format(name=name, type=table_type, ext=template_format))
                logging.info('Writing %s into %s ...', template_format.upper().ljust(4), outfile)

            this_template_values = dict(title=title, body=rows, foot=stats, foot_columns=stats_columns)
            this_template_values.update(template_values.__dict__)

            futures.append(parallel.submit(
                write_table_in_format,
                template_format, outfile, this_template_values,
                options.show_table and template_format == 'html',
                ))

    # write normal tables
    write_table("table", name, rows,
                use_local_summary=(not options.correct_only and not options.common))

    # write difference tables
    if rowsDiff:
        write_table("diff", name + " differences", rowsDiff, use_local_summary=False)

    return futures
1107169
def bulk_create(self, objs, *args, **kwargs):
        """"""
        if hasattr(self.model, 'save_prep'):
            # Method from AbstractBaseModel. If the model class doesn't
            # subclass AbstractBaseModel, then don't call this.
            self.model.save_prep(instance_or_instances=objs)

        return super(CommonManager, self).bulk_create(objs=objs,
                                                      *args,
                                                      **kwargs)
181662
def execute(self, conn, run_num=-1, logical_file_name="", block_name="", dataset="", trans=False):
        
        sql = self.sql
        binds = {}
	if logical_file_name and "%" not in logical_file_name:
	    sql += """ inner join %sFILES FILES on FILES.FILE_ID = FL.FILE_ID
		    WHERE FILES.LOGICAL_FILE_NAME = :logical_file_name"""%(self.owner)
	    binds["logical_file_name"] = logical_file_name
	elif block_name and "%" not in block_name:
            sql += """ inner join %sFILES FILES on FILES.FILE_ID = FL.FILE_ID
		    inner join %sBLOCKS BLOCKS on BLOCKS.BLOCK_ID = FILES.BLOCK_ID
		    WHERE BLOCKS.BLOCK_NAME = :block_name """%(self.owner, self.owner)
            binds["block_name"] =  block_name
	elif dataset and "%" not in dataset:
	    sql += """ inner join %sFILES FILES on FILES.FILE_ID = FL.FILE_ID
	    inner join %sDATASETS DATASETS on DATASETS.DATASET_ID = FILES.DATASET_ID
	    WHERE DATASETS.DATASET = :dataset """%(self.owner, self.owner)
	    binds["dataset"] = dataset
	else:
	    pass
        
	if run_num != -1:
            andorwhere = ("WHERE", "AND")["WHERE" in sql]
            run_list = []
            wheresql_run_list = ''
            wheresql_run_range = ''
            #
            for r in parseRunRange(run_num):
                if isinstance(r, basestring) or isinstance(r, int) or isinstance(r, long):
                    run_list.append(str(r))
                if isinstance(r, run_tuple):
                    if r[0] == r[1]:
                        dbsExceptionHandler('dbsException-invalid-input', "DBS run_num range must be apart at least by 1.", self.logger.exception)
                    wheresql_run_range = " FL.RUN_NUM between :minrun and :maxrun "
                    binds.update({"minrun":r[0]})
                    binds.update({"maxrun":r[1]})
            # 
            if run_list:
                wheresql_run_list = " fl.RUN_NUM in (SELECT TOKEN FROM TOKEN_GENERATOR) "
                run_generator, run_binds = create_token_generator(run_list)
                sql =  "{run_generator}".format(run_generator=run_generator) + sql
                binds.update(run_binds)

            if wheresql_run_range and wheresql_run_list:
                sql += " %s (" %andorwhere    + wheresql_run_range + " or " +  wheresql_run_list + " )"
            elif wheresql_run_range and not wheresql_run_list:
                sql += " %s " %andorwhere  + wheresql_run_range
            elif not wheresql_run_range and wheresql_run_list:
                sql += " %s " %andorwhere  + wheresql_run_list
        #self.logger.debug(sql)
	cursors = self.dbi.processData(sql, binds, conn, transaction=trans, returnCursor=True)
        result=[]
        for c in cursors:
            result.extend(self.formatCursor(c, size=100))
        return result
1222920
def bulkImport_json(self, filename, onDuplicate="error", formatType="auto", **params) :
        """"""

        url = "%s/import" % self.database.URL
        params["onDuplicate"] = onDuplicate
        params["collection"] = self.name
        params["type"] = formatType
        with open(filename) as f:
            data = f.read()
            r = self.connection.session.post(URL, params = params, data = data)

            try :
                errorMessage = "At least: %d errors. The first one is: '%s'\n\n more in <this_exception>.data" % (len(data), data[0]["errorMessage"])
            except KeyError:
                raise UpdateError(data['errorMessage'], data)
513611
public function createBulkCommand($config = [])
    {
        $this->open();
        $config['db'] = $this;
        $command = new BulkCommand($config);

        return $command;
    }
1204592
def batch(args):
    
    from jcvi.apps.grid import MakeManager

    p = OptionParser(batch.__doc__)
    opts, args = p.parse_args(args)

    if len(args) < 2:
        sys.exit(not p.print_help())

    cdsfile = args[0]
    anchors = args[1:]
    workdirs = [".".join(op.basename(x).split(".")[:2]) for x in anchors]
    for wd in workdirs:
        mkdir(wd)

    mm = MakeManager()
    for wd, ac in zip(workdirs, anchors):
        pairscdsfile = wd + ".cds.fasta"
        cmd = "python -m jcvi.apps.ks prepare {} {} -o {}".\
                format(ac, cdsfile, pairscdsfile)
        mm.add((ac, cdsfile), pairscdsfile, cmd)
        ksfile = wd + ".ks"
        cmd = "python -m jcvi.apps.ks calc {} -o {} --workdir {}".\
                format(pairscdsfile, ksfile, wd)
        mm.add(pairscdsfile, ksfile, cmd)
    mm.write()
63713
def parse_bulk_create(prs, conn):
    
    prs_create = prs.add_parser(
        'bulk_create', help='create bulk records of specific zone')
    set_option(prs_create, 'infile')
    conn_options(prs_create, conn)
    prs_create.add_argument('--domain', action='store',
                            help='create records with specify zone')
    prs_create.set_defaults(func=create)
769390
def LoadFile(f, table_name, conn):
  """"""
  reader = csv.reader(f)
  header = next(reader)

  columns = []
  for n in header:
    n = n.replace(' ', '')
    n = n.replace('-', '_')
    columns.append(n)

  create_columns = []
  column_types = {}
  for n in columns:
    if n in column_types:
      create_columns.append("%s %s" % (n, column_types[n]))
    else:
      create_columns.append("%s INTEGER" % (n))

  c = conn.cursor()
  try:
    c.execute("CREATE TABLE %s (%s)" % (table_name, ",".join(create_columns)))
  except sqlite.OperationalError:
    # Likely table exists
    print("table %s already exists?" % (table_name))
    for create_column in create_columns:
      try:
        c.execute("ALTER TABLE %s ADD COLUMN %s" % (table_name, create_column))
      except sqlite.OperationalError:
        # Likely it already exists
        print("column %s already exists in %s?" % (create_column, table_name))

  placeholders = ",".join(["?"] * len(columns))
  insert_values = "INSERT INTO %s (%s) VALUES (%s)" % (table_name, ",".join(columns), placeholders)

  #c.execute("BEGIN TRANSACTION;")
  for row in reader:
    if row:
      if len(row) < len(columns):
        row.extend([None] * (len(columns) - len(row)))
      c.execute(insert_values, row)
  #c.execute("END TRANSACTION;")
  conn.commit()
1073847
def _setup_output_file(self, output_filename, args, write_header=True):
        """"""
        # write command line into outputFile
        # (without environment variables, they are documented by benchexec)
        try:
            output_file = open(output_filename, 'w') # override existing file
        except IOError as e:
            sys.exit(e)

        if write_header:
            output_file.write(' '.join(map(util.escape_string_shell, self._build_cmdline(args)))
                              + '\n\n\n' + '-' * 80 + '\n\n\n')
            output_file.flush()

        return output_file
1242888
def create(args):
    
    # for PUT HTTP method
    action = True

    if ((args.__dict__.get('domain') and args.__dict__.get('name')
         and args.__dict__.get('rtype') and args.__dict__.get('content'))):
        # for create sub-command

        domain = args.domain
        o = JSONConverter(domain)

        name, rtype, content, ttl, priority = get_record_params(args)
        record_dict = o.set_record(name, rtype, content, ttl, priority)

        json = set_json(domain, action, record=record_dict)

    else:
        # for bulk_create sub-command
        if args.__dict__.get('domain'):
            domain = args.domain
        else:
            domain = check_infile(args.infile)
        json = set_json(domain, action, filename=args.infile)

    password = get_password(args)
    token = connect.get_token(args.username, password, args.server)
    processing.create_records(args.server, token, domain, json)

    if args.auto_update_soa == 'True':
        update_soa_serial(args)
262801
def _prep_cmd(cmd, tx_out_file):
    
    cmd = " ".join(cmd) if isinstance(cmd, (list, tuple)) else cmd
    return "export TMPDIR=%s && %s" % (os.path.dirname(tx_out_file), cmd)
1202639
def _batched_write_command(
        namespace, operation, command, docs, check_keys, opts, ctx):
    
    buf = StringIO()

    # Save space for message length and request id
    buf.write(_ZERO_64)
    # responseTo, opCode
    buf.write(b"\x00\x00\x00\x00\xd4\x07\x00\x00")

    # Write OP_QUERY write command
    to_send, length = _batched_write_command_impl(
        namespace, operation, command, docs, check_keys, opts, ctx, buf)

    # Header - request id and message length
    buf.seek(4)
    request_id = _randint()
    buf.write(_pack_int(request_id))
    buf.seek(0)
    buf.write(_pack_int(length))

    return request_id, buf.getvalue(), to_send
752546
def main():
    """"""
    desc = 'Generate files to benchmark'
    parser = argparse.ArgumentParser(description=desc)
    parser.add_argument(
        '--src',
        dest='src_dir',
        default='src',
        help='The directory containing the templates'
    )
    parser.add_argument(
        '--out',
        dest='out_dir',
        default='generated',
        help='The output directory'
    )
    parser.add_argument(
        '--seed',
        dest='seed',
        default='13',
        help='The random seed (to ensure consistent regeneration)'
    )

    args = parser.parse_args()

    random.seed(int(args.seed))

    mkdir_p(args.out_dir)

    for template in templates_in(args.src_dir):
        modes = template.modes()

        n_range = template.range()
        for n_value in n_range:
            base = template.instantiate(n_value)
            for mode in modes:
                write_file(
                    os.path.join(
                        args.out_dir,
                        out_filename(template, n_value, mode)
                    ),
                    mode.convert_from(base)
                )
        write_file(
            os.path.join(args.out_dir, '{0}.json'.format(template.name)),
            json.dumps({
                'files': {
                    n: {
                        m.identifier: out_filename(template, n, m)
                        for m in modes
                    } for n in n_range
                },
                'name': template.name,
                'x_axis_label': template.property('x_axis_label'),
                'desc': template.property('desc'),
                'modes': {m.identifier: m.description() for m in modes}
            })
        )
732019
def to_file(format, *args)
      options = extract_options!(args)
      datasource = args[0]
      path = options[:path] || File.join(Dir.tmpdir, "beardley_#{Time.now.to_i.to_s(36)}_#{rand(100_000_000_000).to_s(36)}.#{format}")
      if format == :pdf
        _JasperPrint                 = Rjb.import('net.sf.jasperreports.engine.JasperPrint')
        _JasperExportManager         = Rjb.import('net.sf.jasperreports.engine.JasperExportManager')
        _JasperExportManager._invoke('exportReportToPdfFile', 'Lnet.sf.jasperreports.engine.JasperPrint;Ljava.lang.String;', prepare(datasource), Rjb.import('java.lang.String').new(path.to_s))
      elsif Beardley.exporters[format]
        exporter = Beardley.with_warnings { Rjb.import(Beardley.exporters[format]) }.new
        _JRExporterParameter = Rjb.import('net.sf.jasperreports.engine.JRExporterParameter')
        exporter.setParameter(_JRExporterParameter.JASPER_PRINT, prepare(datasource))
        exporter.setParameter(_JRExporterParameter.OUTPUT_FILE_NAME, path.to_s)
        exporter.exportReport
      else
        raise "Invalid export format: #{format.inspect}"
      end
      path
    end