Applies any partition-specific configuration to the job builder before
building it.

@param jobBuilder
@param partitionNumber
329986
private void configureComponentsBeforeBuilding(final AnalysisJobBuilder jobBuilder, final int partitionNumber) {
        // update datastores and resource properties to point to node-specific
        // targets if possible. This way parallel writing to files on HDFS does
        // not cause any inconsistencies because each node is writing to a
        // separate file.
        for (final ComponentBuilder cb : jobBuilder.getComponentBuilders()) {
            // find any datastore properties that point to HDFS files
            final Set<ConfiguredPropertyDescriptor> targetDatastoreProperties =
                    cb.getDescriptor().getConfiguredPropertiesByType(UpdateableDatastore.class, false);
            for (final ConfiguredPropertyDescriptor targetDatastoreProperty : targetDatastoreProperties) {
                final Object datastoreObject = cb.getConfiguredProperty(targetDatastoreProperty);
                if (datastoreObject instanceof ResourceDatastore) {
                    final ResourceDatastore resourceDatastore = (ResourceDatastore) datastoreObject;
                    final Resource resource = resourceDatastore.getResource();
                    final Resource replacementResource = createReplacementResource(resource, partitionNumber);
                    if (replacementResource != null) {
                        final ResourceDatastore replacementDatastore =
                                createReplacementDatastore(cb, resourceDatastore, replacementResource);
                        if (replacementDatastore != null) {
                            cb.setConfiguredProperty(targetDatastoreProperty, replacementDatastore);
                        }
                    }
                }
            }

            final Set<ConfiguredPropertyDescriptor> resourceProperties =
                    cb.getDescriptor().getConfiguredPropertiesByType(Resource.class, false);
            for (final ConfiguredPropertyDescriptor resourceProperty : resourceProperties) {
                final Resource resource = (Resource) cb.getConfiguredProperty(resourceProperty);
                final Resource replacementResource = createReplacementResource(resource, partitionNumber);
                if (replacementResource != null) {
                    cb.setConfiguredProperty(resourceProperty, replacementResource);
                }
            }

            // special handlings of specific component types are handled here
            if (cb.getComponentInstance() instanceof CreateCsvFileAnalyzer) {
                if (partitionNumber > 0) {
                    // ensure header is only created once
                    cb.setConfiguredProperty(CreateCsvFileAnalyzer.PROPERTY_INCLUDE_HEADER, false);
                }
            }
        }

        // recursively apply this function also on output data stream jobs
        final List<AnalysisJobBuilder> children = jobBuilder.getConsumedOutputDataStreamsJobBuilders();
        for (final AnalysisJobBuilder childJobBuilder : children) {
            configureComponentsBeforeBuilding(childJobBuilder, partitionNumber);
        }
    }
1288869
@Override
  public Configuration getDriverConfiguration(final URI jobFolder,
                                              final String clientRemoteId,
                                              final String jobId,
                                              final Configuration applicationConfiguration) {
    ConfigurationModuleBuilder driverConfigurationBuilder = AzureBatchDriverConfiguration.CONF.getBuilder()
        .bindImplementation(CommandBuilder.class, this.commandBuilder.getClass());

    // If using docker containers, then use a different set of bindings
    if (this.containerRegistryProvider.isValid()) {
      driverConfigurationBuilder = driverConfigurationBuilder
          .bindImplementation(LocalAddressProvider.class, ContainerBasedLocalAddressProvider.class)
          .bindImplementation(TcpPortProvider.class, SetTcpPortProvider.class);
    }

    final Configuration driverConfiguration = driverConfigurationBuilder.build()
        .set(AzureBatchDriverConfiguration.JOB_IDENTIFIER, jobId)
        .set(AzureBatchDriverConfiguration.CLIENT_REMOTE_IDENTIFIER, clientRemoteId)
        .set(AzureBatchDriverConfiguration.JVM_HEAP_SLACK, this.jvmSlack)
        .set(AzureBatchDriverConfiguration.RUNTIME_NAME, RuntimeIdentifier.RUNTIME_NAME)
        .set(AzureBatchDriverConfiguration.AZURE_BATCH_ACCOUNT_URI, this.azureBatchAccountUri)
        .set(AzureBatchDriverConfiguration.AZURE_BATCH_ACCOUNT_NAME, this.azureBatchAccountName)
        .set(AzureBatchDriverConfiguration.AZURE_BATCH_POOL_ID, this.azureBatchPoolId)
        .set(AzureBatchDriverConfiguration.AZURE_STORAGE_ACCOUNT_NAME, this.azureStorageAccountName)
        .set(AzureBatchDriverConfiguration.AZURE_STORAGE_CONTAINER_NAME, this.azureStorageContainerName)
        .set(AzureBatchDriverConfiguration.CONTAINER_REGISTRY_SERVER,
            this.containerRegistryProvider.getContainerRegistryServer())
        .set(AzureBatchDriverConfiguration.CONTAINER_REGISTRY_USERNAME,
            this.containerRegistryProvider.getContainerRegistryUsername())
        .set(AzureBatchDriverConfiguration.CONTAINER_REGISTRY_PASSWORD,
            this.containerRegistryProvider.getContainerRegistryPassword())
        .set(AzureBatchDriverConfiguration.CONTAINER_IMAGE_NAME,
            this.containerRegistryProvider.getContainerImageName())
        .setMultiple(AzureBatchDriverConfiguration.TCP_PORT_SET, this.tcpPortSet)
        .build();
    return Configurations.merge(driverConfiguration, applicationConfiguration);
  }
719080
public void setJobTemplates(java.util.Collection<JobTemplate> jobTemplates) {
        if (jobTemplates == null) {
            this.jobTemplates = null;
            return;
        }

        this.jobTemplates = new java.util.ArrayList<JobTemplate>(jobTemplates);
    }
49279
public void setJobSummaryList(java.util.Collection<JobSummary> jobSummaryList) {
        if (jobSummaryList == null) {
            this.jobSummaryList = null;
            return;
        }

        this.jobSummaryList = new java.util.ArrayList<JobSummary>(jobSummaryList);
    }
1044076
private void setUseNewAPI() throws IOException {
    int numReduces = conf.getNumReduceTasks();
    String oldMapperClass = "mapred.mapper.class";
    String oldReduceClass = "mapred.reducer.class";
    conf.setBooleanIfUnset("mapred.mapper.new-api",
                           conf.get(oldMapperClass) == null);
    if (conf.getUseNewMapper()) {
      String mode = "new map API";
      ensureNotSet("mapred.input.format.class", mode);
      ensureNotSet(oldMapperClass, mode);
      if (numReduces != 0) {
        ensureNotSet("mapred.partitioner.class", mode);
       } else {
        ensureNotSet("mapred.output.format.class", mode);
      }
    } else {
      String mode = "map compatability";
      ensureNotSet(JobContext.INPUT_FORMAT_CLASS_ATTR, mode);
      ensureNotSet(JobContext.MAP_CLASS_ATTR, mode);
      if (numReduces != 0) {
        ensureNotSet(JobContext.PARTITIONER_CLASS_ATTR, mode);
       } else {
        ensureNotSet(JobContext.OUTPUT_FORMAT_CLASS_ATTR, mode);
      }
    }
    if (numReduces != 0) {
      conf.setBooleanIfUnset("mapred.reducer.new-api",
                             conf.get(oldReduceClass) == null);
      if (conf.getUseNewReducer()) {
        String mode = "new reduce API";
        ensureNotSet("mapred.output.format.class", mode);
        ensureNotSet(oldReduceClass, mode);
      } else {
        String mode = "reduce compatability";
        ensureNotSet(JobContext.OUTPUT_FORMAT_CLASS_ATTR, mode);
        ensureNotSet(JobContext.REDUCE_CLASS_ATTR, mode);
      }
    }
  }
609113
@Override
	public void triggerPartitionProducerStateCheck(
		JobID jobId,
		final IntermediateDataSetID intermediateDataSetId,
		final ResultPartitionID resultPartitionId) {

		CompletableFuture<ExecutionState> futurePartitionState =
			partitionProducerStateChecker.requestPartitionProducerState(
				jobId,
				intermediateDataSetId,
				resultPartitionId);

		futurePartitionState.whenCompleteAsync(
			(ExecutionState executionState, Throwable throwable) -> {
				try {
					if (executionState != null) {
						onPartitionStateUpdate(
							intermediateDataSetId,
							resultPartitionId,
							executionState);
					} else if (throwable instanceof TimeoutException) {
						// our request timed out, assume we're still running and try again
						onPartitionStateUpdate(
							intermediateDataSetId,
							resultPartitionId,
							ExecutionState.RUNNING);
					} else if (throwable instanceof PartitionProducerDisposedException) {
						String msg = String.format("Producer %s of partition %s disposed. Cancelling execution.",
							resultPartitionId.getProducerId(), resultPartitionId.getPartitionId());
						LOG.info(msg, throwable);
						cancelExecution();
					} else {
						failExternally(throwable);
					}
				} catch (IOException | InterruptedException e) {
					failExternally(e);
				}
			},
			executor);
	}
797835
private JobConfig.Builder createJob(List<WorkUnit> workUnits) throws IOException {
    Map<String, TaskConfig> taskConfigMap = Maps.newHashMap();

    try (ParallelRunner stateSerDeRunner = new ParallelRunner(this.stateSerDeRunnerThreads, this.fs)) {
      int multiTaskIdSequence = 0;
      for (WorkUnit workUnit : workUnits) {
        if (workUnit instanceof MultiWorkUnit) {
          workUnit.setId(JobLauncherUtils.newMultiTaskId(this.jobContext.getJobId(), multiTaskIdSequence++));
        }
        addWorkUnit(workUnit, stateSerDeRunner, taskConfigMap);
      }

      Path jobStateFilePath;

      // write the job.state using the state store if present, otherwise serialize directly to the file
      if (this.stateStores.haveJobStateStore()) {
        jobStateFilePath = GobblinClusterUtils.getJobStateFilePath(true, this.appWorkDir, this.jobContext.getJobId());
        this.stateStores.getJobStateStore().put(jobStateFilePath.getParent().getName(), jobStateFilePath.getName(),
            this.jobContext.getJobState());
      } else {
        jobStateFilePath = GobblinClusterUtils.getJobStateFilePath(false, this.appWorkDir, this.jobContext.getJobId());
        SerializationUtils.serializeState(this.fs, jobStateFilePath, this.jobContext.getJobState());
      }

      LOGGER.debug("GobblinHelixJobLauncher.createJob: jobStateFilePath {}, jobState {} jobProperties {}",
          jobStateFilePath, this.jobContext.getJobState().toString(), this.jobContext.getJobState().getProperties());
    }

    JobConfig.Builder jobConfigBuilder = new JobConfig.Builder();

    // Helix task attempts = retries + 1 (fallback to general task retry for backward compatibility)
    jobConfigBuilder.setMaxAttemptsPerTask(this.jobContext.getJobState().getPropAsInt(
        GobblinClusterConfigurationKeys.HELIX_TASK_MAX_ATTEMPTS_KEY,
        this.jobContext.getJobState().getPropAsInt(
            ConfigurationKeys.MAX_TASK_RETRIES_KEY,
            ConfigurationKeys.DEFAULT_MAX_TASK_RETRIES)) + 1);

    // Helix task timeout (fallback to general task timeout for backward compatibility)
    jobConfigBuilder.setTimeoutPerTask(this.jobContext.getJobState().getPropAsLong(
        GobblinClusterConfigurationKeys.HELIX_TASK_TIMEOUT_SECONDS,
        this.jobContext.getJobState().getPropAsLong(
            ConfigurationKeys.TASK_TIMEOUT_SECONDS,
            ConfigurationKeys.DEFAULT_TASK_TIMEOUT_SECONDS)) * 1000);

    jobConfigBuilder.setFailureThreshold(workUnits.size());
    jobConfigBuilder.addTaskConfigMap(taskConfigMap).setCommand(GobblinTaskRunner.GOBBLIN_TASK_FACTORY_NAME);
    jobConfigBuilder.setNumConcurrentTasksPerInstance(ConfigUtils.getInt(jobConfig,
        GobblinClusterConfigurationKeys.HELIX_CLUSTER_TASK_CONCURRENCY,
        GobblinClusterConfigurationKeys.HELIX_CLUSTER_TASK_CONCURRENCY_DEFAULT));

    if (this.jobConfig.hasPath(GobblinClusterConfigurationKeys.HELIX_JOB_TAG_KEY)) {
      String jobTag = this.jobConfig.getString(GobblinClusterConfigurationKeys.HELIX_JOB_TAG_KEY);
      log.info("Job {} has tags associated : {}", this.jobContext.getJobId(), jobTag);
      jobConfigBuilder.setInstanceGroupTag(jobTag);
    }

    if (this.jobConfig.hasPath(GobblinClusterConfigurationKeys.HELIX_JOB_TYPE_KEY)) {
      String jobType = this.jobConfig.getString(this.jobConfig.getString(GobblinClusterConfigurationKeys.HELIX_JOB_TYPE_KEY));
      log.info("Job {} has types associated : {}", this.jobContext.getJobId(), jobType);
      jobConfigBuilder.setJobType(jobType);
    }

    if (Task.getExecutionModel(ConfigUtils.configToState(jobConfig)).equals(ExecutionModel.STREAMING)) {
      jobConfigBuilder.setRebalanceRunningTask(true);
    }

    jobConfigBuilder.setExpiry(this.jobContext.getJobState().getPropAsLong(
        GobblinClusterConfigurationKeys.HELIX_WORKFLOW_EXPIRY_TIME_SECONDS, GobblinClusterConfigurationKeys.DEFAULT_HELIX_WORKFLOW_EXPIRY_TIME_SECONDS));

    return jobConfigBuilder;
  }
88367
public void removeJobType(final Class<?> jobType) {
        if (jobType == null) {
            throw new IllegalArgumentException("jobType must not be null");
        }
        this.jobTypes.values().remove(jobType);
    }
1138240
public void setStoreLocation(String location, Job job) throws IOException
    {
        conf = HadoopCompat.getConfiguration(job);
        setLocationFromUri(location);

        if (username != null && password != null)
            ConfigHelper.setOutputKeyspaceUserNameAndPassword(conf, username, password);
        if (splitSize > 0)
            ConfigHelper.setInputSplitSize(conf, splitSize);
        if (partitionerClass!= null)
            ConfigHelper.setOutputPartitioner(conf, partitionerClass);
        if (rpcPort != null)
        {
            ConfigHelper.setOutputRpcPort(conf, rpcPort);
            ConfigHelper.setInputRpcPort(conf, rpcPort);
        }
        if (initHostAddress != null)
        {
            ConfigHelper.setOutputInitialAddress(conf, initHostAddress);
            ConfigHelper.setInputInitialAddress(conf, initHostAddress);
        }

        ConfigHelper.setOutputColumnFamily(conf, keyspace, column_family);
        CqlConfigHelper.setOutputCql(conf, outputQuery);

        setConnectionInformation();

        if (ConfigHelper.getOutputRpcPort(conf) == 0)
            throw new IOException("PIG_OUTPUT_RPC_PORT or PIG_RPC_PORT environment variable not set");
        if (ConfigHelper.getOutputInitialAddress(conf) == null)
            throw new IOException("PIG_OUTPUT_INITIAL_ADDRESS or PIG_INITIAL_ADDRESS environment variable not set");
        if (ConfigHelper.getOutputPartitioner(conf) == null)
            throw new IOException("PIG_OUTPUT_PARTITIONER or PIG_PARTITIONER environment variable not set");

        initSchema(storeSignature);
    }
1315424
public static Builder builder(URI catalogURI, Properties jobProps) {
    String name = JobState.getJobNameFromProps(jobProps);
    String group = JobState.getJobGroupFromProps(jobProps);
    if (null == group) {
      group = "default";
    }
    try {
      URI jobURI = new URI(catalogURI.getScheme(), catalogURI.getAuthority(),
          "/" + group + "/" + name, null);
      Builder builder = new Builder(jobURI).withConfigAsProperties(jobProps);
      String descr = JobState.getJobDescriptionFromProps(jobProps);
      if (null != descr) {
        builder.withDescription(descr);
      }

      return builder;
    } catch (URISyntaxException e) {
      throw new RuntimeException("Unable to create a JobSpec URI: " + e, e);
    }
  }
56927
public void configure(JobConf job) {
    shards = Shard.getIndexShards(new IndexUpdateConfiguration(job));
    map = new HashMap<Shard, Integer>();
    for (int i = 0; i < shards.length; i++) {
      map.put(shards[i], i);
    }
  }
501856
@Override
  public List<? extends ProducerJob> partitionJobs() {
    UrlTrieNode root = _jobNode.getRight();
    if (isOperatorEquals() || root.getSize() == 1) {
      //Either at an Equals-Node or a Leaf-Node, both of which actually has actual size 1.
      return super.partitionJobs();
    } else {
      if (_groupSize <= 1) {
        throw new RuntimeException("This is impossible. When group size is 1, the operator must be equals");
      }

      UrlTrie trie = new UrlTrie(getPage(), root);
      int gs = Math.min(root.getSize(), _groupSize);

      UrlTriePrefixGrouper grouper = new UrlTriePrefixGrouper(trie, (int) Math.ceil(gs / 2.0));

      List<TrieBasedProducerJob> jobs = new ArrayList<>();
      while (grouper.hasNext()) {
        jobs.add(new TrieBasedProducerJob(_startDate, _endDate, grouper.next(), grouper.getGroupSize()));
      }
      return jobs;
    }
  }
102291
private void writeObject(ObjectOutputStream out) throws IOException {
		super.write(out);
		out.writeUTF(mapredOutputFormat.getClass().getName());
		jobConf.write(out);
	}
799650
function jobBuildNotFoundError(name, buildNumber) {
  return function (result, cb) {
    cb(new Error(text.__('Job %s build %d does not exist', name, buildNumber)));
  };
}
1228137
private void submitJobToHelix(JobConfig.Builder jobConfigBuilder) throws Exception {
    HelixUtils.submitJobToWorkFlow(jobConfigBuilder,
        this.helixWorkFlowName,
        this.jobContext.getJobId(),
        this.helixTaskDriver,
        this.helixManager,
        this.workFlowExpiryTimeSeconds);
  }
264340
public void setJobFlowIds(java.util.Collection<String> jobFlowIds) {
        if (jobFlowIds == null) {
            this.jobFlowIds = null;
            return;
        }

        this.jobFlowIds = new com.amazonaws.internal.SdkInternalList<String>(jobFlowIds);
    }
1107632
public static JSLJob buildPartitionLevelJSLJob(long topLevelJobExecutionId, Properties jobProperties, Step step, int partitionInstance) {

        ObjectFactory jslFactory = new ObjectFactory();
        JSLJob subJob = jslFactory.createJSLJob();
        
        // Uses the top-level job execution id.   Note the RI used to use the INSTANCE id, so this is 
        // a big change.
        String subJobId = generateSubJobId(topLevelJobExecutionId, step.getId(), partitionInstance);
        subJob.setId(subJobId);
        
        //Copy all job properties (from parent JobContext) to partitioned step threads
        subJob.setProperties(CloneUtility.javaPropsTojslProperties(jobProperties));

        // Add one step to job
        Step newStep = jslFactory.createStep();
        
        //set id
        newStep.setId(step.getId());
        
        /***
         * deep copy all fields in a step
         */
        // Not used at partition level
        //newStep.setAllowStartIfComplete(step.getAllowStartIfComplete());
        
        if (step.getBatchlet() != null){
        	newStep.setBatchlet(CloneUtility.cloneBatchlet(step.getBatchlet()));
        }
        
        if (step.getChunk() != null) {
        	newStep.setChunk(CloneUtility.cloneChunk(step.getChunk()));
        }
        
        // Do not copy next attribute and control elements.  Transitioning should ONLY
        // take place on the main thread.
        
        //Do not add step listeners, only call them on parent thread.

        //Add partition artifacts and set instances to 1 as the base case 
        Partition partition = step.getPartition();
        if (partition != null) {
        	if (partition.getCollector() != null) {
        		
        		Partition basePartition = jslFactory.createPartition();
        		
        		PartitionPlan partitionPlan = jslFactory.createPartitionPlan();
        		partitionPlan.setPartitions(null);
        		basePartition.setPlan(partitionPlan);
        		
        		basePartition.setCollector(partition.getCollector());
        		newStep.setPartition(basePartition);
                	
        	}
        }
        
        // Not used at partition level
        //newStep.setStartLimit(step.getStartLimit());
        newStep.setProperties(CloneUtility.cloneJSLProperties(step.getProperties()));
        
        // Don't try to only clone based on type (e.g. ChunkListener vs. StepListener).
        // We don't know the type at the model level, and a given artifact could implement more
        // than one listener interface (e.g. ChunkListener AND StepListener).
        newStep.setListeners(CloneUtility.cloneListeners(step.getListeners()));       
        
        //Add Step properties, need to be careful here to remember the right precedence
        
        subJob.getExecutionElements().add(newStep);


        return subJob;
    }