Adds yarn runtime definitions to the builder.
@param yarnJobSubmissionParams Yarn job submission parameters
@param jobSubmissionParameters Generic job submission parameters
@param builder The multi runtime builder
185014
private void addYarnRuntimeDefinition(
          final AvroYarnJobSubmissionParameters yarnJobSubmissionParams,
          final AvroJobSubmissionParameters jobSubmissionParameters,
          final MultiRuntimeDefinitionBuilder builder) {
    // create and serialize yarn configuration if defined
    final Configuration yarnDriverConfiguration =
            createYarnConfiguration(yarnJobSubmissionParams, jobSubmissionParameters);

    // add yarn runtime to the builder
    builder.addRuntime(yarnDriverConfiguration, RuntimeIdentifier.RUNTIME_NAME);
  }
1040245
public void addJobEntry(BackupJob job)
   {
      // jobEntries
      try
      {
         JobEntryInfo info = new JobEntryInfo();
         info.setDate(Calendar.getInstance());
         info.setType(job.getType());
         info.setState(job.getState());
         info.setURL(job.getStorageURL());

         logWriter.write(info, config);
      }
      catch (IOException e)
      {
         logger.error("Can't add job", e);
      }
      catch (XMLStreamException e)
      {
         logger.error("Can't add job", e);
      }
      catch (BackupOperationException e)
      {
         logger.error("Can't add job", e);
      }
   }
44655
void addWorker(final VortexWorkerManager vortexWorkerManager) {
    lock.lock();
    try {
      if (!terminated) {
        if (!removedBeforeAddedWorkers.contains(vortexWorkerManager.getId())) {
          this.runningWorkers.put(vortexWorkerManager.getId(), vortexWorkerManager);
          this.schedulingPolicy.workerAdded(vortexWorkerManager);
          this.workerAggregateFunctionMap.put(vortexWorkerManager.getId(), new HashSet<Integer>());

          // Notify (possibly) waiting scheduler
          noWorkerOrResource.signal();
        }
      } else {
        // Terminate the worker
        vortexWorkerManager.terminate();
      }
    } finally {
      lock.unlock();
    }
  }
1269847
public synchronized void addJob(JobInProgress job) {
    String poolName = getPoolName(job);
    LOG.info("Adding job " + job.getJobID() + " to pool " + poolName +
             ", originally from pool " +
             job.getJobConf().get(EXPLICIT_POOL_PROPERTY));
    getPool(poolName).addJob(job);
  }
565907
private void buildSubJobBatchWorkUnits() {

        List<Flow> flows = this.split.getFlows();

        splitFlowWorkUnits = new ArrayList<BatchSplitFlowWorkUnit>();

        for (Flow flow : flows) {
            // 1. First, we build the subjob JSLJob model for flows in split
            JSLJob splitFlowJSLJob = ParallelStepBuilder.buildFlowInSplitSubJob(runtimeWorkUnitExecution.getTopLevelInstanceId(), runtimeWorkUnitExecution.getWorkUnitJobContext(),
                                                                                this.split, flow);
            subJobs.add(splitFlowJSLJob);

            // 2. Next, we build a (persisted) execution and a work unit (thread) around it.
            SplitFlowConfig splitFlowConfig = new SplitFlowConfig(runtimeWorkUnitExecution.getTopLevelNameInstanceExecutionInfo(), split.getId(), flow.getId(), runtimeWorkUnitExecution.getCorrelationId());
            BatchSplitFlowWorkUnit workUnit = getBatchKernelService().createSplitFlowWorkUnit(splitFlowConfig, splitFlowJSLJob, completedWorkQueue);
            splitFlowWorkUnits.add(workUnit);
        }
    }
263632
public void setJobRunIds(java.util.Collection<String> jobRunIds) {
        if (jobRunIds == null) {
            this.jobRunIds = null;
            return;
        }

        this.jobRunIds = new java.util.ArrayList<String>(jobRunIds);
    }
839453
public void setJobDefinitions(java.util.Collection<JobDefinition> jobDefinitions) {
        if (jobDefinitions == null) {
            this.jobDefinitions = null;
            return;
        }

        this.jobDefinitions = new java.util.ArrayList<JobDefinition>(jobDefinitions);
    }
1110831
protected void initJobExecutor() {
    if (jobExecutor == null) {
      jobExecutor = new DefaultJobExecutor();
    }

    jobHandlers = new HashMap<String, JobHandler>();
    TimerExecuteNestedActivityJobHandler timerExecuteNestedActivityJobHandler = new TimerExecuteNestedActivityJobHandler();
    jobHandlers.put(timerExecuteNestedActivityJobHandler.getType(), timerExecuteNestedActivityJobHandler);

    TimerCatchIntermediateEventJobHandler timerCatchIntermediateEvent = new TimerCatchIntermediateEventJobHandler();
    jobHandlers.put(timerCatchIntermediateEvent.getType(), timerCatchIntermediateEvent);

    TimerStartEventJobHandler timerStartEvent = new TimerStartEventJobHandler();
    jobHandlers.put(timerStartEvent.getType(), timerStartEvent);

    TimerStartEventSubprocessJobHandler timerStartEventSubprocess = new TimerStartEventSubprocessJobHandler();
    jobHandlers.put(timerStartEventSubprocess.getType(), timerStartEventSubprocess);

    AsyncContinuationJobHandler asyncContinuationJobHandler = new AsyncContinuationJobHandler();
    jobHandlers.put(asyncContinuationJobHandler.getType(), asyncContinuationJobHandler);

    ProcessEventJobHandler processEventJobHandler = new ProcessEventJobHandler();
    jobHandlers.put(processEventJobHandler.getType(), processEventJobHandler);

    TimerSuspendProcessDefinitionHandler suspendProcessDefinitionHandler = new TimerSuspendProcessDefinitionHandler();
    jobHandlers.put(suspendProcessDefinitionHandler.getType(), suspendProcessDefinitionHandler);

    TimerActivateProcessDefinitionHandler activateProcessDefinitionHandler = new TimerActivateProcessDefinitionHandler();
    jobHandlers.put(activateProcessDefinitionHandler.getType(), activateProcessDefinitionHandler);

    TimerSuspendJobDefinitionHandler suspendJobDefinitionHandler = new TimerSuspendJobDefinitionHandler();
    jobHandlers.put(suspendJobDefinitionHandler.getType(), suspendJobDefinitionHandler);

    TimerActivateJobDefinitionHandler activateJobDefinitionHandler = new TimerActivateJobDefinitionHandler();
    jobHandlers.put(activateJobDefinitionHandler.getType(), activateJobDefinitionHandler);

    BatchSeedJobHandler batchSeedJobHandler = new BatchSeedJobHandler();
    jobHandlers.put(batchSeedJobHandler.getType(), batchSeedJobHandler);

    BatchMonitorJobHandler batchMonitorJobHandler = new BatchMonitorJobHandler();
    jobHandlers.put(batchMonitorJobHandler.getType(), batchMonitorJobHandler);

    HistoryCleanupJobHandler historyCleanupJobHandler = new HistoryCleanupJobHandler();
    jobHandlers.put(historyCleanupJobHandler.getType(), historyCleanupJobHandler);

    for (JobHandler batchHandler : batchHandlers.values()) {
      jobHandlers.put(batchHandler.getType(), batchHandler);
    }

    // if we have custom job handlers, register them
    if (getCustomJobHandlers() != null) {
      for (JobHandler customJobHandler : getCustomJobHandlers()) {
        jobHandlers.put(customJobHandler.getType(), customJobHandler);
      }
    }

    jobExecutor.setAutoActivate(jobExecutorActivate);

    if (jobExecutor.getRejectedJobsHandler() == null) {
      if (customRejectedJobsHandler != null) {
        jobExecutor.setRejectedJobsHandler(customRejectedJobsHandler);
      } else {
        jobExecutor.setRejectedJobsHandler(new NotifyAcquisitionRejectedJobsHandler());
      }
    }

  }
338105
@Override
    public void call(Subscriber<? super Job<K, V>> subscriber) {

        log.debug("onSubscribe()");
        if (subscriber.isUnsubscribed()) {
            return;
        }

        String subscriberId = getClass().getSimpleName() + "-" + id + "-" + subscriberIds.incrementAndGet();
        subscriber.onStart();

        try {
            Scheduler.Worker worker = scheduler.createWorker();

            GetJobsAction<K, V> getJobsAction = new GetJobsAction<K, V>(disqueConnectionSupplier, subscriberId, subscriber,
                    jobLocalityTracking, getJobsArgs);

            actions.add(getJobsAction);
            Subscription subscription = worker.schedulePeriodically(getJobsAction, 0, 10, TimeUnit.MILLISECONDS);
            getJobsAction.setSelfSubscription(subscription);

            if (improveLocalityTimeUnit != null && improveLocalityInterval > 0 && reconnectTrigger == null) {
                reconnectTrigger = worker.schedulePeriodically(new Action0() {
                    @Override
                    public void call() {
                        switchNodes();
                    }
                }, improveLocalityInterval, improveLocalityInterval, improveLocalityTimeUnit);
            }
        } catch (Exception e) {
            if (log.isDebugEnabled()) {
                log.debug("QueueListener.call caught an exception: {}", e.getMessage(), e);
            }
            subscriber.onError(e);
        }
    }
811801
public void setJobs(java.util.Collection<JobDetail> jobs) {
        if (jobs == null) {
            this.jobs = null;
            return;
        }

        this.jobs = new java.util.ArrayList<JobDetail>(jobs);
    }
811279
void preInitializeJob(JobInProgress job) {
    JobConf jobConf = job.getJobConf();
    
    // Compute number of slots required to run a single map/reduce task
    int slotsPerMap = 1;
    int slotsPerReduce = 1;
    if (memoryMatcher.isSchedulingBasedOnMemEnabled()) {
      slotsPerMap = jobConf.computeNumSlotsPerMap(getMemSizeForMapSlot());
     slotsPerReduce = 
       jobConf.computeNumSlotsPerReduce(getMemSizeForReduceSlot());
    }
    job.setNumSlotsPerMap(slotsPerMap);
    job.setNumSlotsPerReduce(slotsPerReduce);
  }
622506
@Override
	public CompletableFuture<JobSubmissionResult> submitJob(@Nonnull JobGraph jobGraph) {
		// we have to enable queued scheduling because slot will be allocated lazily
		jobGraph.setAllowQueuedScheduling(true);

		CompletableFuture<java.nio.file.Path> jobGraphFileFuture = CompletableFuture.supplyAsync(() -> {
			try {
				final java.nio.file.Path jobGraphFile = Files.createTempFile("flink-jobgraph", ".bin");
				try (ObjectOutputStream objectOut = new ObjectOutputStream(Files.newOutputStream(jobGraphFile))) {
					objectOut.writeObject(jobGraph);
				}
				return jobGraphFile;
			} catch (IOException e) {
				throw new CompletionException(new FlinkException("Failed to serialize JobGraph.", e));
			}
		}, executorService);

		CompletableFuture<Tuple2<JobSubmitRequestBody, Collection<FileUpload>>> requestFuture = jobGraphFileFuture.thenApply(jobGraphFile -> {
			List<String> jarFileNames = new ArrayList<>(8);
			List<JobSubmitRequestBody.DistributedCacheFile> artifactFileNames = new ArrayList<>(8);
			Collection<FileUpload> filesToUpload = new ArrayList<>(8);

			filesToUpload.add(new FileUpload(jobGraphFile, RestConstants.CONTENT_TYPE_BINARY));

			for (Path jar : jobGraph.getUserJars()) {
				jarFileNames.add(jar.getName());
				filesToUpload.add(new FileUpload(Paths.get(jar.toUri()), RestConstants.CONTENT_TYPE_JAR));
			}

			for (Map.Entry<String, DistributedCache.DistributedCacheEntry> artifacts : jobGraph.getUserArtifacts().entrySet()) {
				artifactFileNames.add(new JobSubmitRequestBody.DistributedCacheFile(artifacts.getKey(), new Path(artifacts.getValue().filePath).getName()));
				filesToUpload.add(new FileUpload(Paths.get(artifacts.getValue().filePath), RestConstants.CONTENT_TYPE_BINARY));
			}

			final JobSubmitRequestBody requestBody = new JobSubmitRequestBody(
				jobGraphFile.getFileName().toString(),
				jarFileNames,
				artifactFileNames);

			return Tuple2.of(requestBody, Collections.unmodifiableCollection(filesToUpload));
		});

		final CompletableFuture<JobSubmitResponseBody> submissionFuture = requestFuture.thenCompose(
			requestAndFileUploads -> sendRetriableRequest(
				JobSubmitHeaders.getInstance(),
				EmptyMessageParameters.getInstance(),
				requestAndFileUploads.f0,
				requestAndFileUploads.f1,
				isConnectionProblemOrServiceUnavailable())
		);

		submissionFuture
			.thenCombine(jobGraphFileFuture, (ignored, jobGraphFile) -> jobGraphFile)
			.thenAccept(jobGraphFile -> {
			try {
				Files.delete(jobGraphFile);
			} catch (IOException e) {
				log.warn("Could not delete temporary file {}.", jobGraphFile, e);
			}
		});

		return submissionFuture
			.thenApply(
				(JobSubmitResponseBody jobSubmitResponseBody) -> new JobSubmissionResult(jobGraph.getJobID()))
			.exceptionally(
				(Throwable throwable) -> {
					throw new CompletionException(new JobSubmissionException(jobGraph.getJobID(), "Failed to submit JobGraph.", ExceptionUtils.stripCompletionException(throwable)));
				});
	}
898995
public void run(String name, Config config, Builder builder) {
    BuilderImpl bldr = (BuilderImpl) builder;
    TopologyBuilder topologyBuilder = bldr.build();
    try {
      HeronSubmitter.submitTopology(name, config.getHeronConfig(),
                                    topologyBuilder.createTopology());
    } catch (AlreadyAliveException | InvalidTopologyException e) {
      e.printStackTrace();
    }
  }
139275
@Override
  public void onNext(final JobSubmissionEvent jobSubmissionEvent) {
    LOG.log(Level.FINEST, "Submitting job: {0}", jobSubmissionEvent);

    try {
      this.applicationId = createApplicationId(jobSubmissionEvent);
      final String folderName = this.azureBatchFileNames.getStorageJobFolder(this.applicationId);

      LOG.log(Level.FINE, "Creating a job folder on Azure at: {0}.", folderName);
      final URI jobFolderURL = this.azureStorageClient.getJobSubmissionFolderUri(folderName);

      LOG.log(Level.FINE, "Getting a shared access signature for {0}.", folderName);
      final String storageContainerSAS = this.azureStorageClient.createContainerSharedAccessSignature();

      LOG.log(Level.FINE, "Assembling Configuration for the Driver.");
      final Configuration driverConfiguration = makeDriverConfiguration(jobSubmissionEvent, this.applicationId,
          jobFolderURL);

      LOG.log(Level.FINE, "Making Job JAR.");
      final File jobSubmissionJarFile =
          this.jobJarMaker.createJobSubmissionJAR(jobSubmissionEvent, driverConfiguration);

      LOG.log(Level.FINE, "Uploading Job JAR to Azure.");
      final URI jobJarSasUri = this.azureStorageClient.uploadFile(folderName, jobSubmissionJarFile);

      LOG.log(Level.FINE, "Assembling application submission.");
      final String command = this.launchCommandBuilder.buildDriverCommand(jobSubmissionEvent);

      this.azureBatchHelper.submitJob(getApplicationId(), storageContainerSAS, jobJarSasUri, command);

    } catch (final IOException e) {
      LOG.log(Level.SEVERE, "Error submitting Azure Batch request: {0}", e);
      throw new RuntimeException(e);
    }
  }
60756
public JobType addJobNotificationSubscriptionType(
            JobNotificationSubscriptionType jobNotificationSubscription) {
        if (this.jobNotificationSubscriptionTypes == null) {
            this.jobNotificationSubscriptionTypes = new ArrayList<JobNotificationSubscriptionType>();
        }
        this.jobNotificationSubscriptionTypes.add(jobNotificationSubscription);
        return this;
    }
1338173
public RuntimeJobExecution createJobStartExecution(final WSJobInstance jobInstance,
                                                       IJobXMLSource jslSource,
                                                       Properties jobParameters,
                                                       long executionId) throws JobStartException {

        // TODO - redesign to avoid cast?
        final JobInstanceEntity jobInstanceImpl = (JobInstanceEntity) jobInstance;
        long instanceId = jobInstance.getInstanceId();

        ModelNavigator<JSLJob> navigator = createFirstExecution(jobInstanceImpl, jslSource, jobParameters);
        JobExecutionEntity jobExec = null;
        try {
            jobExec = getPersistenceManagerService().getJobExecutionMostRecent(instanceId);
            // Check to make sure the executionId belongs to the most recent execution.
            // If not, a restart may have occurred. So, fail the start.
            // Also check to make sure a stop did not come in while the start was on the queue.
            BatchStatus currentBatchStatus = jobExec.getBatchStatus();
            if (jobExec.getExecutionId() != executionId ||
                currentBatchStatus.equals(BatchStatus.STOPPING) ||
                currentBatchStatus.equals(BatchStatus.STOPPED)) {
                throw new JobStartException();
            }

        } catch (IllegalStateException ie) {
            // If no execution exists, request came from old dispatch path.  
            jobExec = getPersistenceManagerService().createJobExecution(instanceId, jobParameters, new Date());
            BatchEventsPublisher eventsPublisher = batchKernelImpl.getBatchEventsPublisher();
            if (eventsPublisher != null) {
                String correlationId = getCorrelationId(jobParameters);
                eventsPublisher.publishJobExecutionEvent(jobExec, BatchEventsPublisher.TOPIC_EXECUTION_STARTING, correlationId);
            }
        }

        TopLevelNameInstanceExecutionInfo topLevelInfo = new TopLevelNameInstanceExecutionInfo(jobInstanceImpl.getJobName(), instanceId, jobExec.getExecutionId());

        return new RuntimeJobExecution(topLevelInfo, jobParameters, navigator);
    }
889331
public void setSuccessfulSubmissions(java.util.Collection<BatchStopJobRunSuccessfulSubmission> successfulSubmissions) {
        if (successfulSubmissions == null) {
            this.successfulSubmissions = null;
            return;
        }

        this.successfulSubmissions = new java.util.ArrayList<BatchStopJobRunSuccessfulSubmission>(successfulSubmissions);
    }