Runs a federated import job on BigQuery for the data in the output path in addition to calling
the delegate's commitJob.
335853
@Override
  public void commitJob(JobContext context) throws IOException {
    super.commitJob(context);

    // Get the destination configuration information.
    Configuration conf = context.getConfiguration();
    TableReference destTable = BigQueryOutputConfiguration.getTableReference(conf);
    String destProjectId = BigQueryOutputConfiguration.getProjectId(conf);
    Optional<BigQueryTableSchema> destSchema = BigQueryOutputConfiguration.getTableSchema(conf);
    BigQueryFileFormat outputFileFormat = BigQueryOutputConfiguration.getFileFormat(conf);
    List<String> sourceUris = getOutputFileURIs();

    getBigQueryHelper()
        .importFederatedFromGcs(
            destProjectId,
            destTable,
            destSchema.isPresent() ? destSchema.get().get() : null,
            outputFileFormat,
            sourceUris);
  }
254173
@Override
  public final void commitJob(org.apache.hadoop.mapreduce.JobContext context
                               ) throws IOException {
    commitJob((JobContext) context);
  }
314997
def _batch_load(project, workspace, headerline, entity_data, chunk_size=500):
    


    if fcconfig.verbosity:
        print("Batching " + str(len(entity_data)) + " updates to Firecloud...")

    # Parse the entity type from the first cell, e.g. "entity:sample_id"
    # First check that the header is valid
    if not _valid_headerline(headerline):
        eprint("Invalid loadfile header:\n" + headerline)
        return 1

    update_type = "membership" if headerline.startswith("membership") else "entitie"
    etype = headerline.split('\t')[0].split(':')[1].replace("_id", "")

    # Split entity_data into chunks
    total = int(len(entity_data) / chunk_size) + 1
    batch = 0
    for i in range(0, len(entity_data), chunk_size):
        batch += 1
        if fcconfig.verbosity:
            print("Updating {0} {1}s {2}-{3}, batch {4}/{5}".format(
                etype, update_type, i+1, min(i+chunk_size, len(entity_data)),
                batch, total))
        this_data = headerline + '\n' + '\n'.join(entity_data[i:i+chunk_size])

        # Now push the entity data to firecloud
        r = fapi.upload_entities(project, workspace, this_data)
        fapi._check_response_code(r, 200)

    return 0
1307011
public static void export(OutputStream os, List<JobDef> jobDefList, DbConn cnx) throws JqmXmlException
    {
        // Argument tests
        if (os == null)
        {
            throw new IllegalArgumentException("output stream cannot be null");
        }
        if (jobDefList == null || jobDefList.isEmpty())
        {
            throw new IllegalArgumentException("job def list cannot be null or empty");
        }
        if (cnx == null)
        {
            throw new IllegalArgumentException("database connection cannot be null");
        }

        Collections.sort(jobDefList, new Comparator<JobDef>()
        {
            @Override
            public int compare(JobDef o1, JobDef o2)
            {
                return o1.getJarPath().compareTo(o2.getJarPath());
            }
        });

        // Create XML document
        Element root = new Element("jqm");
        Document document = new Document(root);

        Element jobDefinitions = null;
        String currentJarPath = null;
        Set<Cl> cls = new HashSet<>();

        for (JobDef j : jobDefList)
        {
            if (currentJarPath == null || !j.getJarPath().equals(currentJarPath))
            {
                currentJarPath = j.getJarPath();
                Element jar = new Element("jar");
                addTextElementToParentElement(jar, "path", currentJarPath);
                addTextElementToParentElement(jar, "pathType", j.getPathType().toString());
                jobDefinitions = new Element("jobdefinitions");
                jar.addContent(jobDefinitions);
                root.addContent(jar);
            }
            Element jobDefinition = getJobDefinitionElement(j, cnx);
            jobDefinitions.addContent(jobDefinition);

            if (j.getClassLoader(cnx) != null)
            {
                cls.add(j.getClassLoader());
            }
        }

        for (Cl cl : cls)
        {
            root.addContent(getClElement(cl));
        }

        // Done: output XML to stream.
        try
        {
            XMLOutputter out = new XMLOutputter(Format.getPrettyFormat());
            out.output(document, os);
        }
        catch (java.io.IOException e)
        {
            throw new JqmXmlException("Could not output XML to stream", e);
        }
    }
621297
def run_job(clusterID, packageID, args = {})
      query = args.inject({}) {|v, kv|
        v.merge(
          {
            "job[variables][#{kv.first}]" => kv.last
          })
      }
      query = query.merge(
        {
          'job[cluster_id]' => clusterID,
          'job[package_id]' => packageID
        })

      request(
        :expects => 201,
        :method => :post,
        :path => "/#{account_id}/api/jobs",
        :query => query
      )
    end
628604
public AnalysisResultFuture run() {
        try {
            // the injection manager is job scoped
            final InjectionManager injectionManager = _configuration.getInjectionManager(_job);

            final LifeCycleHelper rowProcessingLifeCycleHelper = new LifeCycleHelper(injectionManager,
                    new ReferenceDataActivationManager(), _includeNonDistributedTasks);

            final RowProcessingPublishers publishers = new RowProcessingPublishers(_job, _analysisListener,
                    _taskRunner, rowProcessingLifeCycleHelper, _sourceColumnFinder);

            final AnalysisJobMetrics analysisJobMetrics = publishers.getAnalysisJobMetrics();

            // A task listener that will register either succesfull executions
            // or unexpected errors (which will be delegated to the
            // errorListener)
            JobCompletionTaskListener jobCompletionTaskListener = new JobCompletionTaskListener(analysisJobMetrics,
                    _analysisListener, 1);

            _analysisListener.jobBegin(_job, analysisJobMetrics);

            validateSingleTableInput(_transformerJobs);
            validateSingleTableInput(_filterJobs);
            validateSingleTableInput(_analyzerJobs);

            // at this point we are done validating the job, it will run.
            scheduleRowProcessing(publishers, rowProcessingLifeCycleHelper, jobCompletionTaskListener,
                    analysisJobMetrics);

            return new AnalysisResultFutureImpl(_resultQueue, jobCompletionTaskListener, _errorAware);
        } catch (RuntimeException e) {
            _analysisListener.errorUknown(_job, e);
            throw e;
        }

    }
528728
private void importQueue(
            final ITenant tenant,
            final Map<PortalDataKey, Set<BucketTuple>> queue,
            final StringBuilder importReport)
            throws Exception {

        final StandardEvaluationContext ctx = new StandardEvaluationContext();
        ctx.setRootObject(new RootObjectImpl(tenant));
        IDataTemplatingStrategy templating = new SpELDataTemplatingStrategy(portalSpELService, ctx);
        Document doc = null;
        try {
            for (PortalDataKey pdk : dataKeyImportOrder) {
                Set<BucketTuple> bucket = queue.get(pdk);
                if (bucket != null) {
                    log.debug(
                            "Importing the specified PortalDataKey tenant '{}':  {}",
                            tenant.getName(),
                            pdk.getName());
                    for (BucketTuple tuple : bucket) {
                        doc = tuple.getDocument();
                        Source data =
                                templating.processTemplates(
                                        doc, tuple.getResource().getURL().toString());
                        dataHandlerService.importData(data, pdk);
                        importReport.append(createImportReportLineItem(pdk, tuple));
                    }
                }
            }

        } catch (Exception e) {
            log.error(
                    "Failed to process the specified template document:\n{}",
                    (doc != null ? doc.asXML() : "null"),
                    e);
            throw e;
        }
    }
303068
def add_job_to_db(self, key, job):
        """"""
        job_msg = self.registry.deep_encode(job)
        prov = prov_key(job_msg)

        def set_link(duplicate_id):
            self.cur.execute(
                'update "jobs" set "link" = ?, "status" = ? where "id" = ?',
                (duplicate_id, Status.DUPLICATE, key))

        with self.lock:
            self.cur.execute(
                'select * from "jobs" where "prov" = ? '
                'and (("result" is not null) or '
                '("session" = ? and "link" is null))',
                (prov, self.session))
            rec = self.cur.fetchone()
            rec = JobEntry(*rec) if rec is not None else None

            self.cur.execute(
                'update "jobs" set "prov" = ?, "version" = ?, "function" = ?, '
                '"arguments" = ?, "status" = ? where "id" = ?',
                (prov, job_msg['data']['hints'].get('version'),
                 json.dumps(job_msg['data']['function']),
                 json.dumps(job_msg['data']['arguments']),
                 Status.WAITING,
                 key))

            if not rec:
                # no duplicate found, go on
                return 'initialized', None

            set_link(rec.id)

            if rec.result is not None and rec.status == Status.WORKFLOW:
                # the found duplicate returned a workflow
                if rec.link is not None:
                    # link is set, so result is fully realized
                    self.cur.execute(
                        'select * from "jobs" where "id" = ?',
                        (rec.link,))
                    rec = self.cur.fetchone()
                    assert rec is not None, "database integrity violation"
                    rec = JobEntry(*rec)

                else:
                    # link is not set, the result is still waited upon
                    assert rec.session == self.session, \
                        "database integrity violation"
                    self.attached[rec.id].append(key)
                    return 'attached', None

            if rec.result is not None:
                # result is found! return it
                result_value = self.registry.from_json(rec.result, deref=True)
                result = ResultMessage(
                    key, 'retrieved', result_value, None)
                return 'retrieved', result

            if rec.session == self.session:
                # still waiting for result, attach
                self.attached[rec.id].append(key)
                return 'attached', None
727957
function handleArrayResult(m, workerIdx) {
	var job = jobs[m.context.jobID],
		partition = m.context.partition,
		subResult = m.result,
		result = [],
		i;
	
	job.result[partition] = subResult;

	// Increase callback count.
	job.cbCount++;
	
	// When all workers are finished return result
	if(job.cbCount === workers.length) {
		for (i = 0; i < job.result.length; i++) {
			result = result.concat(job.result[i]);
		};
		
		// Cle
		job.cb(null, result);
	}

	// Worker is finished.
	workerJobCount[workerIdx]--;
	dispatchWorkItems();
}
1002816
private Callable<Boolean> coalesceTruncationSnapshotPlan(String file_path, String pathType, String file_nonce, long txnId,
                                                             Map<Integer, Long> partitionTransactionIds,
                                                             SystemProcedureExecutionContext context,
                                                             VoltTable result,
                                                             ExtensibleSnapshotDigestData extraSnapshotData,
                                                             SiteTracker tracker,
                                                             HashinatorSnapshotData hashinatorData,
                                                             long timestamp,
                                                             int newPartitionCount)
    {
        final NativeSnapshotWritePlan plan = new NativeSnapshotWritePlan();
        final Callable<Boolean> deferredTruncationSetup =
                plan.createSetupInternal(file_path, pathType, file_nonce, txnId, partitionTransactionIds,
                        new SnapshotRequestConfig(newPartitionCount, context.getDatabase()), context, result,
                        extraSnapshotData, tracker, hashinatorData, timestamp);
        m_taskListsForHSIds.putAll(plan.m_taskListsForHSIds);

        return new Callable<Boolean>() {
            @Override
            public Boolean call() throws Exception
            {
                final Boolean retval = deferredTruncationSetup.call();
                m_targets.addAll(plan.m_targets);
                return retval;
            }
        };
    }
1014648
private void internalUpdateJobStateAndResult(final JobInstance job) {
        transactionTemplate.execute(new TransactionCallbackWithoutResult() {
            protected void doInTransactionWithoutResult(TransactionStatus status) {
                jobInstanceDao.updateStateAndResult(job);
                if (job.isCompleted()) {
                    buildPropertiesService.saveCruiseProperties(job);
                }
            }
        });
    }
543517
def run_ec2_import(self, config_file_location, description, region='us-east-1'):
        
        import_cmd = "aws ec2 import-image --description '{}' --profile '{}' --region '{}' --output 'json'" \
                     " --disk-containers file://{}"\
            .format(description, self.aws_project, region, config_file_location)
        try:
            res = subprocess.check_output(shlex.split(import_cmd), stderr=subprocess.STDOUT)
        except subprocess.CalledProcessError as e:
            print "Error importing to ec2"
            print "output: {}".format(e.output)
            sys.exit(5)

        print "got res: {}".format(res)
        res_json = json.loads(res)
        task_running, import_id = self.check_task_status_and_id(res_json)
        return import_id
489149
function runQuery(connection, query, options, done) {
  options = _.isObject(options) ? options :
    connection.QueryTypes[options] ? { type: connection.QueryTypes[options] } : {};

  connection.query(query, options)
    .then(function(results) {
      done(null, results);
    })
    .catch(function(error) {
      if (error) {
        output.error("Query error.  SQL has been written to .tables.debug.  Use a smaller batchSize to more easily find the issue.");
        output.error(error);
        fs.writeFileSync(".tables.debug", query);
      }

      done(error);
    });
}
585566
public static Job createSubmittableJob(Configuration conf, String[] args)
  throws IOException {
    conf.setIfUnset("hbase.client.connection.impl",
      BigtableConfiguration.getConnectionClass().getName());
    conf.setIfUnset(BigtableOptionsFactory.BIGTABLE_RPC_TIMEOUT_MS_KEY, "60000");
    conf.setBoolean(TableInputFormat.SHUFFLE_MAPS, true);

    String tableName = args[0];
    Path outputDir = new Path(args[1]);
    Job job = Job.getInstance(conf, NAME + "_" + tableName);
    job.setJobName(NAME + "_" + tableName);
    job.setJarByClass(Export.class);
    // Set optional scan parameters
    Scan s = getConfiguredScanForJob(conf, args);
    TableMapReduceUtil.initTableMapperJob(tableName, s, IdentityTableMapper.class, ImmutableBytesWritable.class, Result.class, job, false);
    // No reducers.  Just write straight to output files.
    job.setNumReduceTasks(0);
    job.setOutputFormatClass(SequenceFileOutputFormat.class);
    job.setOutputKeyClass(ImmutableBytesWritable.class);
    job.setOutputValueClass(Result.class);
    FileOutputFormat.setOutputPath(job, outputDir); // job conf doesn't contain the conf so doesn't have a default fs.
    return job;
  }
1191911
def execute(self, conn, logical_file_name='', block_name='', run_num=-1, validFileOnly=0, migration=False):
        
	sql = ""
	wheresql = ""
	lfn_generator = ""
	run_generator = ""
        if logical_file_name and not isinstance(logical_file_name, list):
            binds = {'logical_file_name': logical_file_name}
            if int(validFileOnly) == 0:
		if migration:   #migration always call with single file and include all files no matter valid or not.
		    sql = self.sql + """ FROM {owner}FILE_LUMIS FL 
				     JOIN {owner}FILES F ON F.FILE_ID = FL.FILE_ID 
				     WHERE F.LOGICAL_FILE_NAME = :logical_file_name 
				  """.format(owner=self.owner)
		else:
		    sql = self.sql + """ , F.LOGICAL_FILE_NAME as LOGICAL_FILE_NAME FROM {owner}FILE_LUMIS FL
				    JOIN {owner}FILES F ON F.FILE_ID = FL.FILE_ID 
                                     WHERE F.LOGICAL_FILE_NAME = :logical_file_name 
                                  """.format(owner=self.owner)
            else:
                sql = self.sql + """ , F.LOGICAL_FILE_NAME as LOGICAL_FILE_NAME FROM {owner}FILE_LUMIS FL 
				     JOIN {owner}FILES F ON F.FILE_ID = FL.FILE_ID
				     JOIN {owner}DATASETS D ON  D.DATASET_ID = F.DATASET_ID
				     JOIN {owner}DATASET_ACCESS_TYPES DT ON  DT.DATASET_ACCESS_TYPE_ID = D.DATASET_ACCESS_TYPE_ID		
				     WHERE F.IS_FILE_VALID = 1 AND F.LOGICAL_FILE_NAME = :logical_file_name 
				     AND DT.DATASET_ACCESS_TYPE in ('VALID', 'PRODUCTION') 
				 """.format(owner=self.owner)
        elif logical_file_name and isinstance(logical_file_name, list):
	    sql = self.sql + """ , F.LOGICAL_FILE_NAME as LOGICAL_FILE_NAME FROM {owner}FILE_LUMIS FL JOIN {owner}FILES F ON F.FILE_ID = FL.FILE_ID """.format(owner=self.owner)	
            lfn_generator, binds = create_token_generator(logical_file_name)
            if int(validFileOnly) == 0:
                wheresql = "WHERE F.LOGICAL_FILE_NAME in (SELECT TOKEN FROM TOKEN_GENERATOR)"
            else:
		sql = sql + """ JOIN {owner}DATASETS D ON  D.DATASET_ID = F.DATASET_ID 
				JOIN {owner}DATASET_ACCESS_TYPES DT ON  DT.DATASET_ACCESS_TYPE_ID = D.DATASET_ACCESS_TYPE_ID
		            """.format(owner=self.owner)			
                wheresql = """ WHERE F.IS_FILE_VALID = 1 AND F.LOGICAL_FILE_NAME in (SELECT TOKEN FROM TOKEN_GENERATOR) 
			       AND DT.DATASET_ACCESS_TYPE in ('VALID', 'PRODUCTION')
			   """
            sql = "{lfn_generator} {sql} {wheresql}".format(lfn_generator=lfn_generator, sql=sql, wheresql=wheresql)
        elif block_name:
            binds = {'block_name': block_name}
            if int(validFileOnly) == 0:
                sql = self.sql + """ , F.LOGICAL_FILE_NAME as LOGICAL_FILE_NAME FROM {owner}FILE_LUMIS FL JOIN {owner}FILES F ON F.FILE_ID = FL.FILE_ID  
				     JOIN {owner}BLOCKS B ON B.BLOCK_ID = F.BLOCK_ID  
				     WHERE B.BLOCK_NAME = :block_name""".format(owner=self.owner)
            else:
                sql = self.sql + """ , F.LOGICAL_FILE_NAME as LOGICAL_FILE_NAME FROM {owner}FILE_LUMIS FL JOIN {owner}FILES F ON F.FILE_ID = FL.FILE_ID 
				     JOIN {owner}DATASETS D ON  D.DATASET_ID = F.DATASET_ID 
				     JOIN {owner}DATASET_ACCESS_TYPES DT ON  DT.DATASET_ACCESS_TYPE_ID = D.DATASET_ACCESS_TYPE_ID 
				     JOIN {owner}BLOCKS B ON B.BLOCK_ID = F.BLOCK_ID
				     WHERE F.IS_FILE_VALID = 1 AND B.BLOCK_NAME = :block_name 
				     AND DT.DATASET_ACCESS_TYPE in ('VALID', 'PRODUCTION')	
				""".format(owner=self.owner)
        else:
                dbsExceptionHandler('dbsException-invalid-input2', "FileLumi/List: Either logocal_file_name or block_name must be provided.", self.logger.exception, "FileLumi/List: Either logocal_file_name or block_name must be provided.")
          #
        if run_num != -1:
            run_list = []
            wheresql_run_list=''
            wheresql_run_range=''
            for r in parseRunRange(run_num):
                if isinstance(r, basestring) or isinstance(r, int) or isinstance(r, long) or isinstance(r, str):
                    run_list.append(str(r))
                if isinstance(r, run_tuple):
                    if r[0] == r[1]:
                        dbsExceptionHandler('dbsException-invalid-input2', "DBS run range must be apart at least by 1.",
			self.logger.exception, "DBS run range must be apart at least by 1.")
                    wheresql_run_range = " FL.RUN_NUM between :minrun and :maxrun "
                    binds.update({"minrun":r[0]})
                    binds.update({"maxrun":r[1]})
            #
            if run_list:
		if len(run_list) == 1:
		    wheresql_run_list = " fl.RUN_NUM = :single_run "
		    binds.update({"single_run": long(run_list[0])})

		else:
		    wheresql_run_list = " fl.RUN_NUM in (SELECT TOKEN FROM TOKEN_GENERATOR) "
                    run_generator, run_binds = create_token_generator(run_list)
                    sql =  "{run_generator}".format(run_generator=run_generator) + sql
                    binds.update(run_binds)

            if wheresql_run_range and wheresql_run_list:
                sql += " and (" + wheresql_run_range + " or " +  wheresql_run_list + " )"
            elif wheresql_run_range and not wheresql_run_list:
                sql += " and " + wheresql_run_range
            elif not wheresql_run_range and wheresql_run_list:
                sql += " and " + wheresql_run_list
        self.logger.debug(sql) 
	self.logger.debug(binds)
	if run_generator and lfn_generator:
		dbsExceptionHandler('dbsException-invalid-input2', "listFileLumiArray support single list of lfn or run_num. ",
			self.logger.exception, "listFileLumiArray support single list of lfn or run_num. ")
        cursors = self.dbi.processData(sql, binds, conn, transaction=False, returnCursor=True)
        result=[]
	file_run_lumi={}
        event_ct=False
        for i in cursors:
            result.extend(self.formatCursor(i, size=100))
        #for migration, we need flat format to load the data into another DB.
        #self.logger.error(result) 
        if migration:
            #YG 09/2015. 
	    for item in result:
		yield item
	else:
            if result and result[0]['event_count']: 
                event_ct = True
	    for i in result:
		r = i['run_num']
		f = i['logical_file_name']
                if event_ct:
                    file_run_lumi.setdefault((f, r), []).append([i['lumi_section_num'], i['event_count']])
                else:
                    file_run_lumi.setdefault((f, r), []).append(i['lumi_section_num'])
	    for k, v in file_run_lumi.iteritems():
                if event_ct:
                    lumi=[]
                    event=[]
                    for le in v:
                        lumi.append(le[0])
                        event.append(le[1])
                    yield {'logical_file_name':k[0], 'run_num':k[1], 'lumi_section_num':lumi, 'event_count':event}
                else:
                    yield {'logical_file_name':k[0], 'run_num':k[1], 'lumi_section_num':v}
        del file_run_lumi
        del result
389708
def process_import_job_from_file(list, file_path, report_email = nil, postback_url = nil, options = {})
      data = options
      data['list'] = list
      data['file'] = file_path
      process_job(:import, data, report_email, postback_url, 'file')
    end
118758
def run_job(key, node):
    """"""
    try:
        result = node.apply()
        return ResultMessage(key, 'done', result, None)

    except Exception as exc:
        return ResultMessage(key, 'error', None, exc)